Merge pull request 'Design 47 H1: the builder keeps a reservation for its fingerprint and withdraws a taken-over build (H-A-225, H-A-226)' (#303) from fix/47-H1-builder-reservation-kept into main

This commit was merged in pull request #303.
This commit is contained in:
2026-11-21 00:41:48 +00:00
9 changed files with 179 additions and 18 deletions
@@ -244,6 +244,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
}
b.Reserved = reserved
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
reserved.withdraw = b.Withdraw
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
// A state bound later numbers the builds looked at without it.
@@ -50,6 +50,8 @@ type reservations struct {
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
withdraw func(module, fingerprint, why string)
// call asks a seat's verb: the git seat's tag (design 47 G §370).
call toolCaller
// say says package-published on the seat mesh-builds; nil says none.
@@ -92,6 +94,11 @@ func (rv *reservations) took(ctx context.Context, older, newer versions.Reserved
}
rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series))
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module)
if older.Fingerprint != newer.Fingerprint && rv.withdraw != nil {
// The older build is ended superseded, as the picture's supersession ends one (H-A-225, ADR 0664 §4): nothing publishes
// under the dropped number.
rv.withdraw(older.Module, older.Fingerprint, "its reservation "+older.Version+" was taken over by the build of "+newer.Fingerprint+", which reserves "+newer.Version)
}
}
var _ build.Reservations = (*reservations)(nil)
@@ -266,7 +266,8 @@ func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing
}
// TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo: design 47 G A-455 — a build that reserves while the series is
// reserved by an older build never released drops the older one's bound and finding and says so in the builder's journal.
// reserved by an older build never released drops the older one's bound and finding and says so in the builder's journal; and
// H-A-225 — the older build is withdrawn, so nothing publishes under the dropped number, while a re-run of the newer keeps it.
func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T) {
ctx := context.Background()
st := &memFindings{vals: map[string][]byte{}}
@@ -279,6 +280,8 @@ func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T)
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return bounds }, found,
nil, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
rv.filled.Store(true)
var withdrawn []string
rv.withdraw = func(module, fp, why string) { withdrawn = append(withdrawn, module+" "+fp+": "+why) }
older, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-old"})
if err != nil {
t.Fatal(err)
@@ -305,6 +308,12 @@ func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T)
if got, ok, _ := rv.ReservedFor(ctx, "m", "fp-new"); !ok || got.Version != "0.1.1" {
t.Errorf("reserved for fp-new: %+v %v", got, ok)
}
if len(withdrawn) != 1 || withdrawn[0] != "m fp-old: its reservation 0.1.0 was taken over by the build of fp-new, which reserves 0.1.1" {
t.Errorf("withdrawn %q; want the build of fp-old withdrawn once", withdrawn)
}
if again, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-new"}); err != nil || again.Version != "0.1.1" || len(withdrawn) != 1 {
t.Errorf("a re-run of fp-new reserved %q (%v), withdrew %q; want 0.1.1 kept, nothing withdrawn", again.Version, err, withdrawn)
}
}
// ansKV records the value versionActed writes back to a reasks key.
@@ -362,10 +362,10 @@ func (m reserveMap) Delete(_ context.Context, key string) error {
return nil
}
// TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber: design 47 G A-490b, at the build, through the real
// reserver — a done key whose own fingerprint has an unreleased 0.1.30 standing, the catalog's highest 0.1.16, is built again
// under 0.1.17 (never 0.1.31), the standing reservation taken over.
func TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber(t *testing.T) {
// TestABuildAgainKeepsItsOwnStandingNumber: design 47 G A-490b as H-A-225 amends it, at the build, through the real reserver —
// a done key whose own fingerprint has an unreleased 0.1.30 standing, the catalog's highest 0.1.16, is built again under 0.1.30
// (never 0.1.31, never a new number): the run building carries the standing number until it is released.
func TestABuildAgainKeepsItsOwnStandingNumber(t *testing.T) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone {
@@ -381,8 +381,8 @@ func TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber(t *te
r.b.Reserved = real
got := r.b.BuildAgain(context.Background(), "m", fp1, "its publish was refused", "shanks.lead", 2*time.Second)
r.b.Wait()
cur, ok, err := real.Of(context.Background(), "m", "0.1.17")
if got.Refused != "" || err != nil || !ok || cur.Fingerprint != fp1 {
t.Fatalf("answer %+v; reservation of 0.1.17 = %+v (%v, %v); want built again under 0.1.17, the catalog's highest 0.1.16 plus one", got, cur, ok, err)
cur, ok, err := real.Of(context.Background(), "m", "")
if got.Refused != "" || err != nil || !ok || cur.Fingerprint != fp1 || cur.Version != "0.1.30" {
t.Fatalf("answer %+v; reservation = %+v (%v, %v); want built again under its standing 0.1.30", got, cur, ok, err)
}
}
@@ -266,6 +266,23 @@ func (b *Builder) EntryGone(branch string, first bool, module string) {
}
}
// Withdraw ends the running build of module's fingerprint, superseded, with why (design 47 H H-A-225, ADR 0664 §4): its key
// fails "withdrawn" and its asks are withdrawn from the build seats. A fingerprint with no build running is left.
func (b *Builder) Withdraw(module, fingerprint, why string) {
key, err := builds.BuilderBuildKey(module, fingerprint)
if err != nil {
return
}
b.init()
b.mu.Lock()
withdraw := b.builds[key]
b.mu.Unlock()
if withdraw != nil {
b.logf("%s at %s: the build is withdrawn: %s", module, fingerprint, why)
withdraw(buildrun.Withdrawn{Why: why})
}
}
func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.ModulesByModuleValue, e []meshcatalog.Edge) {
b.mu.Lock()
hk := o.Branch + "\x00" + key
@@ -707,8 +724,9 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
var reserved versions.ReservedVersion
hasReserved := false
if rebuild && cl.Build.State == builds.BuildBuilding {
// Reserved afresh, above the highest the catalog's versions answers, taking over the older unreleased reservation of this
// fingerprint (A-455), never its standing number kept (A-490b); the build's own publish then runs.
// Reserved as A-490b amended by H-A-225 has it: this fingerprint's standing reservation is kept while it is unreleased
// and above the highest the catalog's versions answers, else the next above that highest is taken (A-455); the build's
// own publish then runs.
if b.Reserved.Filled() {
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), true)
}
@@ -327,6 +327,37 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
}
}
// H-A-225 (ADR 0664 §4): a build withdrawn while it runs (its reservation taken over by another fingerprint) ends failed
// "withdrawn" with why, so it never publishes under the dropped number; a fingerprint with no build running is left.
func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
r := newRig(t, goBundle)
asked, release := make(chan struct{}), make(chan struct{})
answer := r.answer
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
close(asked)
<-release
return answer(a)
}
r.b.Withdraw("m", fp1, "nothing runs yet") // left: no build of it runs
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)})
if err != nil {
t.Fatal(err)
}
r.b.Module(context.Background(), "m", data)
select {
case <-asked:
case <-time.After(5 * time.Second):
t.Fatal("no ask was put")
}
r.b.Withdraw("m", fp1, "its reservation 0.1.3 was taken over by the build of src2:b, which reserves 0.1.4")
close(release)
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "withdrawn: its reservation 0.1.3 was taken over") {
t.Fatalf("the key = %+v, want failed withdrawn naming the take-over", v)
}
}
// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key
// before it is put, and the builder calls no node; the step a node wrote for it makes the key done naming its reference.
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
@@ -5,6 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"sort"
"strconv"
"strings"
@@ -74,6 +76,9 @@ type ReservedVersion struct {
// Kept are the registry seats' published heard while the build had not yet recorded every digest and whose digest marked
// no artifact then: kept as heard, they mark their artifact when the build records its digests (design 47 G A-415).
Kept []KeptPublished `json:"kept-published,omitempty"`
// Withdrawn are the versions of the series a take-over by another fingerprint withdrew, each with the fingerprint whose
// build it was (design 47 H H-A-225): a late publish of one is refused as withdrawn. Kept while the series' key stands.
Withdrawn map[string]string `json:"withdrawn,omitempty"`
}
// KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415).
@@ -154,7 +159,10 @@ func SplitVersion(version string) (series string, patch int, err error) {
// Reserve reserves the next version of module's series for a build that starts, for artifacts (their names), and answers
// it. The next patch is the one after the higher of the highest released and the one reserved, 0 when there is neither. A
// conflicting write (another build reserved first) is read again and the next taken.
// conflicting write (another build reserved first) is read again and the next taken. A reservation standing for the same
// fingerprint whose version is not released and is above the catalog's highest is kept and answered as it stands, a build again or a re-run after a restart
// alike (design 47 H H-A-225, which amends G A-490b): the run already building carries that number. A reservation another
// fingerprint takes over is withdrawn: its version is kept in Withdrawn, and Took is told so the older build is ended.
func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts []string, src Source) (ReservedVersion, error) {
if r.Filled != nil && !r.Filled() {
return ReservedVersion{}, ErrNotFilled
@@ -175,6 +183,11 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
if err != nil {
return ReservedVersion{}, err
}
if found && cur.Fingerprint == src.Fingerprint && cur.Fingerprint != "" && !slices.Contains(released, cur.Version) {
if _, p, err := SplitVersion(cur.Version); err == nil && p > high {
return cur, nil // kept until released (H-A-225); one at or below the catalog's highest can never be released
}
}
next := high
if found && !(src.Rebuild && cur.Fingerprint == src.Fingerprint) {
if _, p, err := SplitVersion(cur.Version); err == nil && p > next {
@@ -189,6 +202,23 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
for _, a := range artifacts {
out.Artifacts[a] = ""
}
if found {
out.Withdrawn = maps.Clone(cur.Withdrawn)
if cur.Fingerprint != src.Fingerprint {
if out.Withdrawn == nil {
out.Withdrawn = map[string]string{}
}
out.Withdrawn[cur.Version] = cur.Fingerprint // a dropped number stays a gap (H-A-226)
}
// One at or below the catalog's highest released is refused by the plain rule anyway: the map never grows without end.
maps.DeleteFunc(out.Withdrawn, func(v, _ string) bool {
_, p, err := SplitVersion(v)
return err != nil || p <= high
})
if len(out.Withdrawn) == 0 {
out.Withdrawn = nil
}
}
raw, _ := json.Marshal(out)
if found {
_, err = r.Store.Update(ctx, key, raw, rev)
@@ -711,11 +741,24 @@ func AnswerReservedByName(ctx context.Context, r Reader, carriers func(context.C
}
return out, nil
}
if series, _, err := SplitVersion(version); err == nil {
if cur, _, found, err := rs.get(ctx, ReservedKey(module, series)); err == nil && found && hasKey(cur.Withdrawn, version) {
// A late publish of a build a take-over withdrew (H-A-225): refused as what it is.
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) version %s was withdrawn: the build of %s it was reserved for was taken over by %s, which reserves %s",
name, module, version, cur.Withdrawn[version], cur.Fingerprint, cur.Version)}, nil
}
}
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no version %s reserved or released", name, module, version)}, nil
}
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no reserved version", name, module)}, nil
}
// hasKey says whether m holds k.
func hasKey(m map[string]string, k string) bool {
_, ok := m[k]
return ok
}
// NoEntryCarries opens the answer of reserved given a name no entry carries.
const NoEntryCarries = "no entry carries the name "
@@ -221,18 +221,46 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
}
}
// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces
// its fingerprint's standing reservation (and tells Took) and numbers above the catalog's highest alone, whether the standing number
// is below it or above it; another fingerprint's standing reservation stays a floor.
// TestTheWithdrawnVersionsAreDroppedOnceReleasedPast: design 47 H H-A-225, H-A-226 — a dropped number stays a gap, kept in
// the series key's Withdrawn while it is above the catalog's highest released; at each write those at or below it go, so the
// map never grows without end.
func TestTheWithdrawnVersionsAreDroppedOnceReleasedPast(t *testing.T) {
ctx := context.Background()
store := &reserveMem{mem: newMem()}
highest := "0.1.3"
r := &Reserver{Store: store, Filled: func() bool { return true },
ReleasedOf: func(context.Context, string) ([]string, error) { return []string{highest}, nil }}
for _, fp := range []string{"src2:a", "src2:b", "src2:c"} {
if _, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp}); err != nil {
t.Fatal(err)
}
}
cur, _, _, _ := r.get(ctx, ReservedKey("m", "0.1"))
if cur.Version != "0.1.6" || len(cur.Withdrawn) != 2 || cur.Withdrawn["0.1.4"] != "src2:a" || cur.Withdrawn["0.1.5"] != "src2:b" {
t.Fatalf("reservation %+v; want 0.1.6 with 0.1.4 and 0.1.5 withdrawn", cur)
}
highest = "0.1.5" // the catalog released past both withdrawn numbers but the newest
got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "src2:d"})
if err != nil || got.Version != "0.1.7" || len(got.Withdrawn) != 1 || got.Withdrawn["0.1.6"] != "src2:c" {
t.Errorf("reserved %+v (%v); want 0.1.7 keeping only 0.1.6 withdrawn", got, err)
}
}
// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b as H-A-225 amends it — a build again
// of a done key, or a re-run after a restart, keeps its fingerprint's standing reservation while it is unreleased and above the
// catalog's highest; one at or below it is replaced (Took told) by the next above the catalog's highest alone; another
// fingerprint's standing reservation stays a floor and is withdrawn.
func TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas(t *testing.T) {
ctx := context.Background()
for _, c := range []struct {
name, standing, standingFp string
want string
rebuild, kept bool
}{
{name: "standing below the catalog's highest", standing: "0.1.0", standingFp: "fp", want: "0.1.17"},
{name: "standing above the catalog's highest", standing: "0.1.30", standingFp: "fp", want: "0.1.17"},
{name: "another fingerprint's standing number stays a floor", standing: "0.1.30", standingFp: "other", want: "0.1.31"},
{name: "standing below the catalog's highest", standing: "0.1.0", standingFp: "fp", want: "0.1.17", rebuild: true},
{name: "H-A-225: standing above the catalog's highest is kept by a build again", standing: "0.1.30", standingFp: "fp", want: "0.1.30", rebuild: true, kept: true},
{name: "H-A-225: a re-run after a restart keeps its standing number", standing: "0.1.111", standingFp: "fp", want: "0.1.111", kept: true},
{name: "another fingerprint's standing number stays a floor", standing: "0.1.30", standingFp: "other", want: "0.1.31", rebuild: true},
} {
t.Run(c.name, func(t *testing.T) {
store := &reserveMem{mem: newMem()}
@@ -246,10 +274,19 @@ func TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas(t *
r.Took = func(_ context.Context, older, newer ReservedVersion) {
took = append(took, older.Version+">"+newer.Version)
}
got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "fp", Rebuild: true})
got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "fp", Rebuild: c.rebuild})
if c.kept {
if err != nil || got.Version != c.want || len(took) != 0 || got.Artifacts["m:a"] != "d" {
t.Fatalf("reserved %+v (%v), took %v; want the standing %s kept as it stands", got, err, took, c.want)
}
return
}
if err != nil || got.Version != c.want || len(took) != 1 || took[0] != c.standing+">"+c.want {
t.Fatalf("reserved %q (%v), took %v; want %s replacing %s", got.Version, err, took, c.want, c.standing)
}
if w := got.Withdrawn[c.standing]; (c.standingFp != "fp") != (w == c.standingFp) {
t.Errorf("withdrawn %v; want %s withdrawn only when another fingerprint stood", got.Withdrawn, c.standing)
}
})
}
}
@@ -67,6 +67,20 @@ func TestReservedByNameAnswersFromTheViewAndTheCatalog(t *testing.T) {
t.Errorf("%s = %+v, want error naming %q alone", name, got, c.want)
}
}
// H-A-225: another fingerprint taking the series over withdraws the older's version; a late publish of it is refused as
// withdrawn, naming the fingerprint that took it over, never as a version that was never reserved.
if _, err := r.Reserve(ctx, "mesh-sdk", "0.1", []string{"mesh-sdk:ts"}, Source{Fingerprint: "src2:a"}); err != nil {
t.Fatal(err)
}
if _, err := r.Reserve(ctx, "mesh-sdk", "0.1", []string{"mesh-sdk:ts"}, Source{Fingerprint: "src2:b"}); err != nil {
t.Fatal(err)
}
if got := ask("@novox/mesh-sdk", "0.1.5"); !strings.Contains(got.Error, "version 0.1.5 was withdrawn: the build of src2:a it was reserved for was taken over by src2:b, which reserves 0.1.6") {
t.Errorf("a late publish of a withdrawn version = %+v, want refused as withdrawn", got)
}
if got := ask("@novox/mesh-sdk", "9.9.9"); !strings.Contains(got.Error, "reserved or released") {
t.Errorf("a version never reserved = %+v, want refused as neither reserved nor released", got)
}
if _, err := AnswerReservedByName(ctx, rd, Carriers(packagesMem(nil)), released, "x", ""); err == nil {
t.Error("an unreadable view was answered as if no entry carried the name")
}