From dcf5b7fa8f9131c0fced56e5784b00b0ea257e82 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 01:36:32 +0100 Subject: [PATCH] Keep a reservation for its fingerprint until released; withdraw a taken-over build (H-A-225) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a builder restart, the run already building a module carried its reserved number while a re-run of the same fingerprint reserved the next, so its publish was refused "has no version reserved or released" (main.61: the builder 0.1.111 vs 0.1.112, mesh-grants 0.1.52 vs 0.1.54, and others reserved twice within seconds). A take-over by another fingerprint dropped the older reservation but let the older build run on and publish a number that no longer existed. Reserve now answers the standing reservation of the same fingerprint while it is unreleased and above the catalog's highest, a build again or a re-run alike (A-490b amended so). A take-over keeps the dropped version in Withdrawn and withdraws the older build (ended superseded, ADR 0664 §4); a late publish of it is refused as withdrawn, naming the fingerprint that took it over. Builds: design 47 H H-A-225; G A-490b as amended; ADR 0664 §4 --- .../cmd/mesh-delivery-builder/main.go | 2 ++ .../cmd/mesh-delivery-builder/reserve.go | 7 ++++ .../cmd/mesh-delivery-builder/reserve_test.go | 11 +++++- .../internal/build/reserve_test.go | 14 ++++---- .../internal/build/run.go | 17 +++++++++ .../internal/build/run_test.go | 31 ++++++++++++++++ .../internal/versions/reserve.go | 35 ++++++++++++++++++- .../internal/versions/reserve_test.go | 26 ++++++++++---- .../internal/versions/reservedname_test.go | 14 ++++++++ 9 files changed, 141 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 60097089..e09db687 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -244,6 +244,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p) } b.Reserved = reserved + // A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225). + reserved.withdraw = b.Withdraw // The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it. reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps) // A state bound later numbers the builds looked at without it. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 8923151c..4f1ad99b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -50,6 +50,8 @@ type reservations struct { bound time.Duration wake func(context.Context) logf func(string, ...any) + // withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none. + withdraw func(module, fingerprint, why string) // call asks a seat's verb: the git seat's tag (design 47 G §370). call toolCaller // say says package-published on the seat mesh-builds; nil says none. @@ -92,6 +94,11 @@ func (rv *reservations) took(ctx context.Context, older, newer versions.Reserved } rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series)) rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module) + if older.Fingerprint != newer.Fingerprint && rv.withdraw != nil { + // The older build is ended superseded, as the picture's supersession ends one (H-A-225, ADR 0664 §4): nothing publishes + // under the dropped number. + rv.withdraw(older.Module, older.Fingerprint, "its reservation "+older.Version+" was taken over by the build of "+newer.Fingerprint+", which reserves "+newer.Version) + } } var _ build.Reservations = (*reservations)(nil) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 11075cbd..6051e802 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -266,7 +266,8 @@ func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing } // TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo: design 47 G A-455 — a build that reserves while the series is -// reserved by an older build never released drops the older one's bound and finding and says so in the builder's journal. +// reserved by an older build never released drops the older one's bound and finding and says so in the builder's journal; and +// H-A-225 — the older build is withdrawn, so nothing publishes under the dropped number, while a re-run of the newer keeps it. func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T) { ctx := context.Background() st := &memFindings{vals: map[string][]byte{}} @@ -279,6 +280,8 @@ func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T) rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return bounds }, found, nil, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }) rv.filled.Store(true) + var withdrawn []string + rv.withdraw = func(module, fp, why string) { withdrawn = append(withdrawn, module+" "+fp+": "+why) } older, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-old"}) if err != nil { t.Fatal(err) @@ -305,6 +308,12 @@ func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T) if got, ok, _ := rv.ReservedFor(ctx, "m", "fp-new"); !ok || got.Version != "0.1.1" { t.Errorf("reserved for fp-new: %+v %v", got, ok) } + if len(withdrawn) != 1 || withdrawn[0] != "m fp-old: its reservation 0.1.0 was taken over by the build of fp-new, which reserves 0.1.1" { + t.Errorf("withdrawn %q; want the build of fp-old withdrawn once", withdrawn) + } + if again, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-new"}); err != nil || again.Version != "0.1.1" || len(withdrawn) != 1 { + t.Errorf("a re-run of fp-new reserved %q (%v), withdrew %q; want 0.1.1 kept, nothing withdrawn", again.Version, err, withdrawn) + } } // ansKV records the value versionActed writes back to a reasks key. diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index dca68ef6..bec4ae13 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -362,10 +362,10 @@ func (m reserveMap) Delete(_ context.Context, key string) error { return nil } -// TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber: design 47 G A-490b, at the build, through the real -// reserver — a done key whose own fingerprint has an unreleased 0.1.30 standing, the catalog's highest 0.1.16, is built again -// under 0.1.17 (never 0.1.31), the standing reservation taken over. -func TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber(t *testing.T) { +// TestABuildAgainKeepsItsOwnStandingNumber: design 47 G A-490b as H-A-225 amends it, at the build, through the real reserver — +// a done key whose own fingerprint has an unreleased 0.1.30 standing, the catalog's highest 0.1.16, is built again under 0.1.30 +// (never 0.1.31, never a new number): the run building carries the standing number until it is released. +func TestABuildAgainKeepsItsOwnStandingNumber(t *testing.T) { r := newRig(t, goBundle) r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { @@ -381,8 +381,8 @@ func TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber(t *te r.b.Reserved = real got := r.b.BuildAgain(context.Background(), "m", fp1, "its publish was refused", "shanks.lead", 2*time.Second) r.b.Wait() - cur, ok, err := real.Of(context.Background(), "m", "0.1.17") - if got.Refused != "" || err != nil || !ok || cur.Fingerprint != fp1 { - t.Fatalf("answer %+v; reservation of 0.1.17 = %+v (%v, %v); want built again under 0.1.17, the catalog's highest 0.1.16 plus one", got, cur, ok, err) + cur, ok, err := real.Of(context.Background(), "m", "") + if got.Refused != "" || err != nil || !ok || cur.Fingerprint != fp1 || cur.Version != "0.1.30" { + t.Fatalf("answer %+v; reservation = %+v (%v, %v); want built again under its standing 0.1.30", got, cur, ok, err) } } diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 86d65f6c..d62ca36c 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -266,6 +266,23 @@ func (b *Builder) EntryGone(branch string, first bool, module string) { } } +// Withdraw ends the running build of module's fingerprint, superseded, with why (design 47 H H-A-225, ADR 0664 §4): its key +// fails "withdrawn" and its asks are withdrawn from the build seats. A fingerprint with no build running is left. +func (b *Builder) Withdraw(module, fingerprint, why string) { + key, err := builds.BuilderBuildKey(module, fingerprint) + if err != nil { + return + } + b.init() + b.mu.Lock() + withdraw := b.builds[key] + b.mu.Unlock() + if withdraw != nil { + b.logf("%s at %s: the build is withdrawn: %s", module, fingerprint, why) + withdraw(buildrun.Withdrawn{Why: why}) + } +} + func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.ModulesByModuleValue, e []meshcatalog.Edge) { b.mu.Lock() hk := o.Branch + "\x00" + key diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index b611511d..4ccd5a30 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -327,6 +327,37 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 } } +// H-A-225 (ADR 0664 §4): a build withdrawn while it runs (its reservation taken over by another fingerprint) ends failed +// "withdrawn" with why, so it never publishes under the dropped number; a fingerprint with no build running is left. +func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) { + r := newRig(t, goBundle) + asked, release := make(chan struct{}), make(chan struct{}) + answer := r.answer + r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { + close(asked) + <-release + return answer(a) + } + r.b.Withdraw("m", fp1, "nothing runs yet") // left: no build of it runs + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", + Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)}) + if err != nil { + t.Fatal(err) + } + r.b.Module(context.Background(), "m", data) + select { + case <-asked: + case <-time.After(5 * time.Second): + t.Fatal("no ask was put") + } + r.b.Withdraw("m", fp1, "its reservation 0.1.3 was taken over by the build of src2:b, which reserves 0.1.4") + close(release) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "withdrawn: its reservation 0.1.3 was taken over") { + t.Fatalf("the key = %+v, want failed withdrawn naming the take-over", v) + } +} + // 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key // before it is put, and the builder calls no node; the step a node wrote for it makes the key done naming its reference. func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 635805af..da6a8a51 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "maps" + "slices" "sort" "strconv" "strings" @@ -74,6 +76,9 @@ type ReservedVersion struct { // Kept are the registry seats' published heard while the build had not yet recorded every digest and whose digest marked // no artifact then: kept as heard, they mark their artifact when the build records its digests (design 47 G A-415). Kept []KeptPublished `json:"kept-published,omitempty"` + // Withdrawn are the versions of the series a take-over by another fingerprint withdrew, each with the fingerprint whose + // build it was (design 47 H H-A-225): a late publish of one is refused as withdrawn. Kept while the series' key stands. + Withdrawn map[string]string `json:"withdrawn,omitempty"` } // KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415). @@ -154,7 +159,10 @@ func SplitVersion(version string) (series string, patch int, err error) { // Reserve reserves the next version of module's series for a build that starts, for artifacts (their names), and answers // it. The next patch is the one after the higher of the highest released and the one reserved, 0 when there is neither. A -// conflicting write (another build reserved first) is read again and the next taken. +// conflicting write (another build reserved first) is read again and the next taken. A reservation standing for the same +// fingerprint whose version is not released and is above the catalog's highest is kept and answered as it stands, a build again or a re-run after a restart +// alike (design 47 H H-A-225, which amends G A-490b): the run already building carries that number. A reservation another +// fingerprint takes over is withdrawn: its version is kept in Withdrawn, and Took is told so the older build is ended. func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts []string, src Source) (ReservedVersion, error) { if r.Filled != nil && !r.Filled() { return ReservedVersion{}, ErrNotFilled @@ -175,6 +183,11 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts if err != nil { return ReservedVersion{}, err } + if found && cur.Fingerprint == src.Fingerprint && cur.Fingerprint != "" && !slices.Contains(released, cur.Version) { + if _, p, err := SplitVersion(cur.Version); err == nil && p > high { + return cur, nil // kept until released (H-A-225); one at or below the catalog's highest can never be released + } + } next := high if found && !(src.Rebuild && cur.Fingerprint == src.Fingerprint) { if _, p, err := SplitVersion(cur.Version); err == nil && p > next { @@ -189,6 +202,13 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts for _, a := range artifacts { out.Artifacts[a] = "" } + if found && cur.Fingerprint != src.Fingerprint { + out.Withdrawn = maps.Clone(cur.Withdrawn) + if out.Withdrawn == nil { + out.Withdrawn = map[string]string{} + } + out.Withdrawn[cur.Version] = cur.Fingerprint + } raw, _ := json.Marshal(out) if found { _, err = r.Store.Update(ctx, key, raw, rev) @@ -711,11 +731,24 @@ func AnswerReservedByName(ctx context.Context, r Reader, carriers func(context.C } return out, nil } + if series, _, err := SplitVersion(version); err == nil { + if cur, _, found, err := rs.get(ctx, ReservedKey(module, series)); err == nil && found && hasKey(cur.Withdrawn, version) { + // A late publish of a build a take-over withdrew (H-A-225): refused as what it is. + return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) version %s was withdrawn: the build of %s it was reserved for was taken over by %s, which reserves %s", + name, module, version, cur.Withdrawn[version], cur.Fingerprint, cur.Version)}, nil + } + } return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no version %s reserved or released", name, module, version)}, nil } return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no reserved version", name, module)}, nil } +// hasKey says whether m holds k. +func hasKey(m map[string]string, k string) bool { + _, ok := m[k] + return ok +} + // NoEntryCarries opens the answer of reserved given a name no entry carries. const NoEntryCarries = "no entry carries the name " diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 3c1bed16..099e0859 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -221,18 +221,21 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) { } } -// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces -// its fingerprint's standing reservation (and tells Took) and numbers above the catalog's highest alone, whether the standing number -// is below it or above it; another fingerprint's standing reservation stays a floor. +// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b as H-A-225 amends it — a build again +// of a done key, or a re-run after a restart, keeps its fingerprint's standing reservation while it is unreleased and above the +// catalog's highest; one at or below it is replaced (Took told) by the next above the catalog's highest alone; another +// fingerprint's standing reservation stays a floor and is withdrawn. func TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas(t *testing.T) { ctx := context.Background() for _, c := range []struct { name, standing, standingFp string want string + rebuild, kept bool }{ - {name: "standing below the catalog's highest", standing: "0.1.0", standingFp: "fp", want: "0.1.17"}, - {name: "standing above the catalog's highest", standing: "0.1.30", standingFp: "fp", want: "0.1.17"}, - {name: "another fingerprint's standing number stays a floor", standing: "0.1.30", standingFp: "other", want: "0.1.31"}, + {name: "standing below the catalog's highest", standing: "0.1.0", standingFp: "fp", want: "0.1.17", rebuild: true}, + {name: "H-A-225: standing above the catalog's highest is kept by a build again", standing: "0.1.30", standingFp: "fp", want: "0.1.30", rebuild: true, kept: true}, + {name: "H-A-225: a re-run after a restart keeps its standing number", standing: "0.1.111", standingFp: "fp", want: "0.1.111", kept: true}, + {name: "another fingerprint's standing number stays a floor", standing: "0.1.30", standingFp: "other", want: "0.1.31", rebuild: true}, } { t.Run(c.name, func(t *testing.T) { store := &reserveMem{mem: newMem()} @@ -246,10 +249,19 @@ func TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas(t * r.Took = func(_ context.Context, older, newer ReservedVersion) { took = append(took, older.Version+">"+newer.Version) } - got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "fp", Rebuild: true}) + got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "fp", Rebuild: c.rebuild}) + if c.kept { + if err != nil || got.Version != c.want || len(took) != 0 || got.Artifacts["m:a"] != "d" { + t.Fatalf("reserved %+v (%v), took %v; want the standing %s kept as it stands", got, err, took, c.want) + } + return + } if err != nil || got.Version != c.want || len(took) != 1 || took[0] != c.standing+">"+c.want { t.Fatalf("reserved %q (%v), took %v; want %s replacing %s", got.Version, err, took, c.want, c.standing) } + if w := got.Withdrawn[c.standing]; (c.standingFp != "fp") != (w == c.standingFp) { + t.Errorf("withdrawn %v; want %s withdrawn only when another fingerprint stood", got.Withdrawn, c.standing) + } }) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reservedname_test.go b/modules/mesh-delivery-builder/internal/versions/reservedname_test.go index 27896ed6..c947d8eb 100644 --- a/modules/mesh-delivery-builder/internal/versions/reservedname_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reservedname_test.go @@ -67,6 +67,20 @@ func TestReservedByNameAnswersFromTheViewAndTheCatalog(t *testing.T) { t.Errorf("%s = %+v, want error naming %q alone", name, got, c.want) } } + // H-A-225: another fingerprint taking the series over withdraws the older's version; a late publish of it is refused as + // withdrawn, naming the fingerprint that took it over, never as a version that was never reserved. + if _, err := r.Reserve(ctx, "mesh-sdk", "0.1", []string{"mesh-sdk:ts"}, Source{Fingerprint: "src2:a"}); err != nil { + t.Fatal(err) + } + if _, err := r.Reserve(ctx, "mesh-sdk", "0.1", []string{"mesh-sdk:ts"}, Source{Fingerprint: "src2:b"}); err != nil { + t.Fatal(err) + } + if got := ask("@novox/mesh-sdk", "0.1.5"); !strings.Contains(got.Error, "version 0.1.5 was withdrawn: the build of src2:a it was reserved for was taken over by src2:b, which reserves 0.1.6") { + t.Errorf("a late publish of a withdrawn version = %+v, want refused as withdrawn", got) + } + if got := ask("@novox/mesh-sdk", "9.9.9"); !strings.Contains(got.Error, "reserved or released") { + t.Errorf("a version never reserved = %+v, want refused as neither reserved nor released", got) + } if _, err := AnswerReservedByName(ctx, rd, Carriers(packagesMem(nil)), released, "x", ""); err == nil { t.Error("an unreadable view was answered as if no entry carried the name") }