H-A-492: record a version's digests under the builder's own bound

A takeover or an end cancelling a build's context cancelled the record of its
reservation's digests eight times in a day, each leaving a passed build
unreleased. The record now runs under the builder's own bound (one minute,
not cancelled with the build), and a done build whose digests are not
recorded is recorded from its done key at the builder's next look and at its
start (before H-A-484's start check holds it against the released versions),
said each time.

Builds: design 47 H2 H-A-492
This commit is contained in:
2026-11-22 20:17:14 +01:00
parent ffa58e51c4
commit 21daeb8893
6 changed files with 174 additions and 3 deletions
@@ -259,6 +259,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
b.Reserved = reserved
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
reserved.withdraw = b.Withdraw
reserved.recordAgain = b.RecordAgain // a done build's digests its end did not record are recorded at the start (H-A-492)
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
// A state bound later numbers the builds looked at without it.
@@ -57,6 +57,9 @@ type reservations struct {
logf func(string, ...any)
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
withdraw func(module, fingerprint, why string)
// recordAgain records the digests of a done build of module at fp its end did not record, saying so (design 47 H2 H-A-492);
// nil records none.
recordAgain func(ctx context.Context, module, fp, when string) (string, error)
// call asks a seat's verb: the git seat's tag (design 47 G §370).
call toolCaller
// pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands
@@ -363,12 +366,20 @@ func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionRelea
// holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one
// said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended.
// A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released.
// Before that, a standing reservation with no digests recorded whose build key is done has them recorded from the done key
// (design 47 H2 H-A-492), so it is held against the released versions too.
func (rv *reservations) releasedAtStart(ctx context.Context) {
all, err := rv.r.All(ctx)
if err != nil {
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
return
}
if rv.recordedAtStart(ctx, all) {
if all, err = rv.r.All(ctx); err != nil {
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
return
}
}
read := map[string][]string{}
for _, v := range builtOf(all) {
if versions.NotStanding(v, true) != "" {
@@ -392,6 +403,26 @@ func (rv *reservations) releasedAtStart(ctx context.Context) {
}
}
// recordedAtStart records the digests of every standing reservation that has none recorded and whose build key is done (design 47
// H2 H-A-492): a record its build's end lost. It answers whether one was recorded; one that cannot be is said.
func (rv *reservations) recordedAtStart(ctx context.Context, all []versions.ReservedVersion) bool {
if rv.recordAgain == nil {
return false
}
recorded := false
for _, v := range all {
if !v.BuiltAt.IsZero() || v.Fingerprint == "" || versions.NotStanding(v, true) != "" {
continue
}
version, err := rv.recordAgain(ctx, v.Module, v.Fingerprint, "at the builder's start")
if err != nil {
rv.logf("%v", err)
}
recorded = recorded || version != ""
}
return recorded
}
// published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told
// before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told
// nothing.
@@ -219,6 +219,23 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
// stands.
refuse = ""
tags = nil
// Design 47 H2 H-A-492: at the start, a standing reservation with no digests recorded whose build key is done has them
// recorded (the builder's RecordAgain), and is then held against the released versions as a built one is.
if _, err := rv.Reserve(ctx, "rec", "0.1", []string{"rec:bin"}, versions.Source{Repository: "o/q", Commit: "abc", Fingerprint: "fpR"}); err != nil {
t.Fatal(err)
}
var recordedAt []string
rv.recordAgain = func(ctx context.Context, module, fp, when string) (string, error) {
recordedAt = append(recordedAt, module+" "+fp+" "+when)
_, err := rv.r.Built(ctx, module, "0.1.0", "rec:bin", "sha256:bb", nil) // its done key's digest
return "0.1.0", err
}
catalog = []meshcatalog.VersionEntry{{Module: "rec", Version: "0.1.0"}}
rv.releasedAtStart(ctx)
if _, standing, _ := rv.r.Of(ctx, "rec", "0.1.0"); standing || !slices.Equal(recordedAt, []string{"rec fpR at the builder's start"}) {
t.Fatalf("at the start, rec 0.1.0 unrecorded and released: standing %v, recorded %q; want it recorded once at the start and released", standing, recordedAt)
}
catalog, rv.recordAgain = nil, nil
var ended []string
rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) }
rv.releasedAtStart(ctx) // built nothing yet: nothing taken
@@ -122,6 +122,53 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
return out
}
// recordBound bounds the record of a reservation's digests (design 47 H2 H-A-492): the builder's own bound, never the build's
// context.
const recordBound = time.Minute
// recordContext is the context a reservation's digests are recorded under: ctx's values with no cancel of its own, bounded by
// recordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record.
func recordContext(ctx context.Context) (context.Context, context.CancelFunc) {
return context.WithTimeout(context.WithoutCancel(ctx), recordBound)
}
// RecordAgain records the digests of fp's done build under its standing reservation when its end did not record them (design 47
// H2 H-A-492): at the builder's look of the key and at its start (when says which), said each time. It answers the version
// recorded, "" when nothing was to be recorded: the key is not done, no reservation stands for fp, or it is recorded already.
func (b *Builder) RecordAgain(ctx context.Context, module, fp, when string) (string, error) {
b.init()
if b.Reserved == nil {
return "", nil
}
digests := b.doneDigests(ctx, module, fp)
if digests == nil {
return "", nil
}
res, ok, err := b.Reserved.ReservedFor(ctx, module, fp)
if err != nil {
return "", fmt.Errorf("%s at %s: whether its done build's digests are recorded cannot be read %s: %w", module, fp, when, err)
}
if !ok || !res.BuiltAt.IsZero() || len(res.Artifacts) == 0 || versions.NotStanding(res, true) != "" {
return "", nil
}
refs := map[string]string{}
for ref := range res.Artifacts {
_, artifact, _ := strings.Cut(ref, ":")
d, have := digests[artifact]
if !have {
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded, and the done key holds no digest of %s to record %s", module, fp, res.Version, ref, when)
}
refs[ref] = d
}
rec, stop := recordContext(ctx)
defer stop()
if err := b.Reserved.Built(rec, module, res.Version, refs, nil); err != nil {
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded %s either: %w", module, fp, res.Version, when, err)
}
b.logf("%s at %s: the digests of %s were not recorded at its build's end: recorded %s from the done key", module, fp, res.Version, when)
return res.Version, nil
}
// PackageNamed looks at a module again when the indexer's packages state names a name for one of its packages
// (versions.NamedModule): a done trunk build that left the package unnumbered, its name not known at its claim, is
// numbered then with the done build's digests, and version.numbered says them for its build module's late publish.
@@ -32,6 +32,8 @@ type fakeReservations struct {
next string // the version Reserve answers when set, else the series' .5
fail error // the reservation cannot be made (the catalog's versions cannot be read)
tookBy map[string]versions.ReservedVersion // by fingerprint, the reservation that took its series over (H-A-244)
held map[string]versions.ReservedVersion // by fingerprint, the reservation ReservedFor answers as it stands (H-A-492)
lost error // the record of the digests fails, as a store refusing it would
}
func (f *fakeReservations) TakenOver(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
@@ -46,6 +48,9 @@ func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, e
func (f *fakeReservations) Filled() bool { return f.filled }
func (f *fakeReservations) ReservedFor(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
if v, ok := f.held[fp]; ok {
return v, true, nil
}
for _, s := range f.sources {
if s.Fingerprint == fp {
return versions.ReservedVersion{Version: "held"}, true, nil
@@ -65,7 +70,13 @@ func (f *fakeReservations) Reserve(_ context.Context, module, series string, art
return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil
}
func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
func (f *fakeReservations) Built(ctx context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
if err := ctx.Err(); err != nil {
return err // the state's write runs under ctx: a cancelled one records nothing
}
if f.lost != nil {
return f.lost
}
f.deps = deps
f.built = map[string]string{version: ""}
for ref, d := range digests {
@@ -153,6 +164,59 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
}
}
// TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound: design 47 H2 H-A-492 — the record of a reservation's digests after a
// build ends runs under the builder's own bound, so a build context cancelled at its end still records; a done build whose
// digests are not recorded is recorded at the builder's next look, said.
func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) {
manifest := catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
t.Run("a build's context cancelled at its end still records", func(t *testing.T) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = manifest
res := &fakeReservations{filled: true}
r.b.Reserved = res
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r.b.Ended = func(context.Context, string) { cancel() } // a takeover or the builder's end, once the key is written
r.b.Module(ctx, "m", moduleValue(t, "modules/m"))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
t.Fatalf("build = %+v, want done", v)
}
if res.built["m:tool"] != digest {
t.Errorf("recorded %v, want the digest of m:tool under 0.1.5 though the build's context ended", res.built)
}
})
t.Run("a done key whose digests are not recorded is recorded at the next look", func(t *testing.T) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = manifest
res := &fakeReservations{filled: true, lost: errors.New("context canceled")}
r.b.Reserved = res
var journal []string
r.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone || res.built != nil {
t.Fatalf("build %s recorded %v, want done and nothing recorded", v.State, res.built)
}
res.lost = nil
res.held = map[string]versions.ReservedVersion{fp1: {Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": ""}}}
r.consider(t, "modules/m")
if res.built["m:tool"] != digest || len(res.built) != 2 {
t.Errorf("recorded %v at the next look, want the done key's digest of m:tool under 0.1.5", res.built)
}
if !slices.ContainsFunc(journal, func(l string) bool {
return strings.Contains(l, "0.1.5 were not recorded") && strings.Contains(l, "recorded at a look")
}) {
t.Errorf("journal %q, want a line saying 0.1.5's digests were recorded at a look", journal)
}
// Recorded once: a reservation whose build ended recorded is left.
res.built = nil
res.held[fp1] = versions.ReservedVersion{Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": digest}, BuiltAt: time.Unix(1, 0)}
r.consider(t, "modules/m")
if res.built != nil {
t.Errorf("a recorded reservation was recorded again: %v", res.built)
}
})
}
// TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through
// reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why.
func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) {
@@ -726,6 +726,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
}
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
if trunk && b.originOf(key).First && planErr == nil && !nothing && !reserving {
// A done build whose digests its end did not record is recorded at the next look (design 47 H2 H-A-492).
if _, err := b.RecordAgain(ctx, module, m.Fp2, "at a look"); err != nil {
b.logf("%v", err)
}
}
if reserving && b.waitFill(module, m) {
end("is not built yet: wait-held, the catalog has not said versions-filled")
return
@@ -859,8 +865,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
for _, a := range result.Artifacts {
digests[module+":"+a.Name] = a.Digest
}
if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil {
b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err)
// Under the builder's own bound, never the build's context: a takeover or an end that cancels it no longer leaves a passed
// build unrecorded, so unreleased (design 47 H2 H-A-492).
rec, stop := recordContext(ctx)
err := b.Reserved.Built(rec, module, reserved.Version, digests, b.takeDependencies(key))
stop()
if err != nil {
b.logf("%s: the digests of %s are not recorded: %v; recorded again at the builder's next look or start", cl.Key, reserved.Version, err)
}
}
if result.State == builds.BuildFailed {