From 5628ca2919e978725466a8c5b776723279e41af8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 15:56:04 +0100 Subject: [PATCH] Builder: a build whose number a registry holds elsewhere is built again, never moved Every artifact carries its version from its build (a Go module zip's paths, a program's stamped version, an image's and a bundle's build environment), so no digest survives a change of number: the move that kept digests would have published a binary saying 0.1.37 under another number. The try, on a number a registry holds at another digest, withdraws it on the reservation (a gap) and raises version-held-elsewhere naming both digests, then asks the build of the same fingerprint again through the build-again path of a done key whose version is not released. Its reserve never takes a withdrawn number again, skips every number a registry holds, and the build records fresh digests and runs its own publish; the takeover ends the old number's publish item and bound. Builds: design 47 H H-A-275, H-A-279, H-A-226 --- .../cmd/mesh-delivery-builder/main.go | 13 ++++ .../cmd/mesh-delivery-builder/reserve.go | 38 +++++------ .../cmd/mesh-delivery-builder/reserve_test.go | 36 +++++++---- .../internal/versions/reserve.go | 63 ++++++------------- .../internal/versions/reserve_test.go | 28 ++++----- 5 files changed, 86 insertions(+), 92 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 81098837..bd8a8fe3 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -266,6 +266,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro reserved.r.HeldNumbers = func(ctx context.Context, module, fp string, refs []string) ([]string, error) { return republish.HeldNumbers(ctx, module, fp, refs, registryVersionsVia(calls.Call)) } + // A done build whose number a registry holds elsewhere is built again at the next free number (H-A-279), on the builder's context. + reserved.rebuild = func(_ context.Context, module, fp, why string) error { + res := b.BuildAgain(ctx, module, fp, why, "the builder", buildAgainBound) + switch { + case res.Refused != "": + return errors.New(res.Refused) + case res.Failed != "": + return errors.New(res.Failed) + case res.Pending != "": + logger.Printf("%s at %s is built again: %s", module, fp, res.Pending) + } + return nil + } reserved.r.HeldElsewhere = func(ctx context.Context, module string, skipped []string) { reserved.heldElsewhere(ctx, module, "the registries hold "+strings.Join(skipped, ", ")+", so the reserve skipped them") } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 52892f61..1e3bbd28 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -54,6 +54,8 @@ type reservations struct { withdraw func(module, fingerprint, why string) // call asks a seat's verb: the git seat's tag (design 47 G ยง370). call toolCaller + // rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none. + rebuild func(ctx context.Context, module, fp, why string) error // say says package-published on the seat mesh-builds; nil says none. say func(context.Context, meshbuilds.PackagePublished) error } @@ -428,34 +430,26 @@ func (rv *reservations) heldElsewhere(ctx context.Context, module, why string) { needs: "builder_reserved shows the reservation and its withdrawn numbers; the registry seat's versions shows what it holds."}) } -// movedElsewhere moves a done build whose number a registry holds at another digest to the next free number (design 47 H -// H-A-275): the held numbers are read again, the reservation moved (the old number withdrawn, a gap), its old publish item and -// bound ended as a takeover's are, the finding raised, and its bound and publish item asked under the new number. +// movedElsewhere takes a done build whose number a registry holds at another digest off that number (design 47 H H-A-275, +// H-A-279): every artifact carries its version from its build, so no digest survives a move. The number is withdrawn as a gap +// (H-A-226), the finding raised naming both digests, and the build of the same fingerprint asked again (the build-again path of a +// done key whose version is not released, A-490): its reserve takes the next number no registry holds, it records fresh digests +// and its own publish runs; the takeover of the withdrawn reservation ends its publish item and bound. func (rv *reservations) movedElsewhere(ctx context.Context, v versions.ReservedVersion, j versions.Judged) error { - if rv.r.HeldNumbers == nil { - return errors.New("the registries' versions are not read") + if rv.rebuild == nil { + return errors.New("no build again is wired") } - refs := make([]string, 0, len(v.Artifacts)) - for ref := range v.Artifacts { - refs = append(refs, ref) - } - held, err := rv.r.HeldNumbers(ctx, v.Module, v.Fingerprint, refs) - if err != nil { - return err - } - old, moved, ok, err := rv.r.Move(ctx, v.Module, v.Version, held) - if err != nil || !ok { + if _, ok, err := rv.r.Withdraw(ctx, v.Module, v.Version); err != nil || !ok { + if err == nil { + err = fmt.Errorf("no reservation of %s stands at %s", v.Module, v.Version) + } return err } var at []string for _, h := range j.Elsewhere { at = append(at, fmt.Sprintf("%s holds %s %s at %s, built %s", h.Seat, h.Name, h.Version, h.Digest, v.Artifacts[h.Ref])) } - rv.took(ctx, old, moved) // the same fingerprint: its old item and bound end, nothing is withdrawn from a running build - rv.heldElsewhere(ctx, v.Module, fmt.Sprintf("the build of %s is moved from %s to %s: %s", v.Fingerprint, old.Version, moved.Version, strings.Join(at, "; "))) - rv.arm(ctx, moved) - if rv.built != nil { - return rv.built(ctx, moved.Module, moved.Version) - } - return nil + why := fmt.Sprintf("%s is withdrawn and the build of %s is built again at the next free number: %s", v.Version, v.Fingerprint, strings.Join(at, "; ")) + rv.heldElsewhere(ctx, v.Module, why) + return rv.rebuild(ctx, v.Module, v.Fingerprint, why) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 91ed06b0..84beb089 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -396,8 +396,8 @@ func TestAReserveDoneIsAnsweredWithTheVersionAndItsQueuedPublish(t *testing.T) { // TestARegistrysRecordEndsOrMovesAPublish: design 47 H H-A-274 โ€” the artifacts a registry's record holds with the built digest are // marked published from it and package-published is said, as a heard published would; H-A-275 โ€” a done build whose number a -// registry holds at another digest moves to the next number no registry holds, raises version-held-elsewhere about the module, ends -// the old number's item and bound and asks the new number's publish. +// registry holds at another digest (H-A-279) has its number withdrawn, raises version-held-elsewhere naming both digests, and is +// built again: at the next number no registry holds, with a fresh digest, published there; the old number's item ends. func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) { ctx := context.Background() fs := &memFindings{vals: map[string][]byte{}} @@ -431,28 +431,40 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) { if cur, _, _ = rv.r.Of(ctx, "node-tools", res.Version); len(cur.Missing()) != 0 { t.Errorf("the record marked nothing: missing %v", cur.Missing()) } - // Another reservation of the module, done, whose number the registry holds at another digest. + // Another reservation of the module, done, whose number the registry holds at another digest: withdrawn, built again at the + // next free number with a fresh digest, and published there (H-A-279). if _, err := rv.r.Released(ctx, "node-tools", res.Version); err != nil { t.Fatal(err) } - asked, ended, registry = nil, nil, []string{"0.1.0", "0.1.1", "0.1.2"} - if err := rv.r.Store.(*memReserve).Delete(ctx, versions.ReservedKey("node-tools", "0.1")); err != nil { - t.Fatal(err) - } + asked, ended, said, registry = nil, nil, nil, []string{"0.1.0", "0.1.1", "0.1.2"} raw, _ := json.Marshal(versions.ReservedVersion{V: 1, Module: "node-tools", Series: "0.1", Version: "0.1.1", Fingerprint: "fp2", Artifacts: map[string]string{"node-tools:program": "sha256:7c9b10"}, BuiltAt: time.Now()}) if _, err := rv.r.Store.Create(ctx, versions.ReservedKey("node-tools", "0.1"), raw); err != nil { t.Fatal(err) } + rv.rebuild = func(ctx context.Context, module, fp, _ string) error { + again, err := rv.Reserve(ctx, module, "0.1", []string{"node-tools:program"}, versions.Source{Fingerprint: fp, Rebuild: true}) + if err != nil { + return err + } + if err := rv.Built(ctx, module, again.Version, map[string]string{"node-tools:program": "sha256:fresh"}, nil); err != nil { + return err + } + rv.published(ctx, versions.Outcome{Name: "node-tools/program", Version: again.Version, Digest: "sha256:fresh"}) + return nil + } cur, _, _ = rv.r.Of(ctx, "node-tools", "0.1.1") elsewhere := held elsewhere.Version, elsewhere.Digest = "0.1.1", "sha256:83bec1" if err := rv.movedElsewhere(ctx, cur, versions.Judged{Elsewhere: []versions.HeldArtifact{elsewhere}}); err != nil { t.Fatal(err) } - moved, ok, _ := rv.r.Of(ctx, "node-tools", "") - if !ok || moved.Version != "0.1.3" || moved.Withdrawn["0.1.1"] != "fp2" { - t.Fatalf("moved to %+v, want 0.1.3 with 0.1.1 withdrawn", moved) + again, ok, _ := rv.r.Of(ctx, "node-tools", "") + if !ok || again.Version != "0.1.3" || again.Withdrawn["0.1.1"] != "fp2" || again.Artifacts["node-tools:program"] != "sha256:fresh" { + t.Fatalf("built again as %+v, want 0.1.3 with a fresh digest and 0.1.1 withdrawn", again) + } + if len(said) != 1 || said[0].Version != "0.1.3" || said[0].Digest != "sha256:fresh" { + t.Errorf("package-published said %+v, want the fresh build at 0.1.3", said) } if len(asked) != 1 || asked[0] != "node-tools 0.1.3" || len(ended) != 1 || ended[0] != "node-tools 0.1.1" { t.Errorf("asked %v, ended %v: want the new number's publish asked and the old one's item ended", asked, ended) @@ -461,7 +473,7 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) { for _, v := range fs.vals { raised += string(v) } - if !strings.Contains(raised, versions.VersionHeldElsewhere) || !strings.Contains(raised, "sha256:83bec1") || !strings.Contains(raised, "0.1.3") { - t.Errorf("no version-held-elsewhere naming the registry's digest and the new number: %s", raised) + if !strings.Contains(raised, versions.VersionHeldElsewhere) || !strings.Contains(raised, "sha256:83bec1") || !strings.Contains(raised, "sha256:7c9b10") { + t.Errorf("no version-held-elsewhere naming both digests: %s", raised) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 0f337ab9..7fcbaf71 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -231,13 +231,14 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts if err != nil { return ReservedVersion{}, err } - if found && cur.Fingerprint == src.Fingerprint && cur.Fingerprint != "" && !slices.Contains(released, cur.Version) { + withdrawn := found && hasKey(cur.Withdrawn, cur.Version) // a number a registry holds elsewhere (H-A-279) + if found && !withdrawn && cur.Fingerprint == src.Fingerprint && cur.Fingerprint != "" && !slices.Contains(released, cur.Version) { if _, p, err := SplitVersion(cur.Version); err == nil && p > high { return cur, nil // kept until released (H-A-225); one at or below the catalog's highest can never be released } } next := high - if found && !(src.Rebuild && cur.Fingerprint == src.Fingerprint) { + if found && (withdrawn || !(src.Rebuild && cur.Fingerprint == src.Fingerprint)) { // a withdrawn number is never taken again (H-A-226) if _, p, err := SplitVersion(cur.Version); err == nil && p > next { next = p } @@ -303,61 +304,35 @@ func (r *Reserver) get(ctx context.Context, key string) (ReservedVersion, uint64 return v, rev, true, nil } -// Move moves a done build's reservation of version off a number a registry holds at another digest (design 47 H H-A-275): the -// reservation takes the next number after the higher of the catalog's highest released and its own that no registry holds -// (held, the versions a registry seat holds of its artifacts), keeping its digests, with nothing published under it yet; the old -// number is withdrawn and stays a gap (H-A-226). It answers the reservation before and after, and false when it no longer stands -// at version. -func (r *Reserver) Move(ctx context.Context, module, version string, held []string) (ReservedVersion, ReservedVersion, bool, error) { +// Withdraw withdraws module's reservation of version for the build of its fingerprint, whose number a registry holds at another +// digest (design 47 H H-A-275, H-A-279): the number is kept in Withdrawn and stays a gap (H-A-226), so the build again of that +// fingerprint reserves the next number no registry holds, with fresh digests. It answers the reservation as withdrawn, and false +// when none stands at version. +func (r *Reserver) Withdraw(ctx context.Context, module, version string) (ReservedVersion, bool, error) { series, _, err := SplitVersion(version) if err != nil { - return ReservedVersion{}, ReservedVersion{}, false, err - } - released, err := r.ReleasedOf(ctx, module) - if err != nil { - return ReservedVersion{}, ReservedVersion{}, false, fmt.Errorf("the released versions of %s cannot be read: %w", module, err) - } - high := -1 - for _, v := range released { - if s, p, err := SplitVersion(v); err == nil && s == series && p > high { - high = p - } - } - taken := map[int]bool{} - for _, v := range held { - if s, p, err := SplitVersion(v); err == nil && s == series { - taken[p] = true - } + return ReservedVersion{}, false, err } key := ReservedKey(module, series) for { cur, rev, found, err := r.get(ctx, key) - if err != nil || !found || cur.Version != version || cur.BuiltAt.IsZero() { - return cur, ReservedVersion{}, false, err + if err != nil || !found || cur.Version != version { + return cur, false, err } - _, p, _ := SplitVersion(cur.Version) - patch, _ := nextFree(series, max(high, p), taken) - out := cur - out.Version = fmt.Sprintf("%s.%d", series, patch) - if cur.Tag != "" { - out.Tag = GoTagOf(out.Version) + if hasKey(cur.Withdrawn, version) { + return cur, true, nil } - out.Artifacts = maps.Clone(cur.Artifacts) - out.Published, out.Kept, out.Unstored = nil, nil, nil - out.At, out.BuiltAt = r.now(), r.now() - out.Withdrawn = maps.Clone(cur.Withdrawn) - if out.Withdrawn == nil { - out.Withdrawn = map[string]string{} + if cur.Withdrawn == nil { + cur.Withdrawn = map[string]string{} } - out.Withdrawn[cur.Version] = cur.Fingerprint // the held number stays a gap (H-A-226) - raw, _ := json.Marshal(out) + cur.Withdrawn[version] = cur.Fingerprint + raw, _ := json.Marshal(cur) if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) { continue } else if err != nil { - return cur, ReservedVersion{}, false, fmt.Errorf("%s cannot be moved: %w", key, err) + return cur, false, fmt.Errorf("%s cannot be withdrawn: %w", key, err) } - r.said(r.reindex(ctx, key, cur, out)) - return cur, out, true, nil + return cur, true, nil } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 2c23a0d7..dbe5b5ba 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -300,8 +300,8 @@ func TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas(t * // TestANumberARegistryHoldsIsNeverReserved: design 47 H H-A-275 (the rule of H-A-245 for a registry's record) โ€” a reserve skips // every number a registry seat holds of the module's artifacts and says the skipped ones; a read that fails lets it proceed; a -// done build whose number a registry holds at another digest moves to the next free number, its digests kept, nothing published -// under it, the old number withdrawn as a gap (H-A-226) and the index following it. +// done build whose number a registry holds at another digest has it withdrawn as a gap (H-A-226), and its build again reserves the +// next free number with fresh digests (H-A-279), the index following it. func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) { ctx := context.Background() store := &reserveMem{mem: newMem()} @@ -320,25 +320,25 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) { if err != nil || v.Version != "0.1.39" || len(skipped) != 2 { t.Fatalf("reserved %q (%v), skipped %v: want 0.1.39 past the two numbers the registry holds", v.Version, err, skipped) } - // The build ends; the registry holds 0.1.39 at another digest meanwhile: the build moves to the next free number. + // The build ends; the registry holds 0.1.39 at another digest meanwhile: the number is withdrawn, and the build again of the same + // fingerprint reserves the next number no registry holds, with no digest yet (H-A-279). if _, err := r.Built(ctx, "node-tools", "0.1.39", "node-tools:program", "sha256:7c9b10", nil); err != nil { t.Fatal(err) } - old, moved, ok, err := r.Move(ctx, "node-tools", "0.1.39", []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40"}) - if err != nil || !ok || old.Version != "0.1.39" || moved.Version != "0.1.41" || moved.Tag != GoTagOf("0.1.41") { - t.Fatalf("moved %+v -> %+v (%v %v), want 0.1.39 -> 0.1.41", old.Version, moved.Version, ok, err) + if w, ok, err := r.Withdraw(ctx, "node-tools", "0.1.39"); err != nil || !ok || w.Withdrawn["0.1.39"] != "fp1" { + t.Fatalf("withdrew %+v (%v %v), want 0.1.39 withdrawn for fp1", w, ok, err) } - if moved.Artifacts["node-tools:program"] != "sha256:7c9b10" || len(moved.Published) != 0 || moved.BuiltAt.IsZero() || moved.Withdrawn["0.1.39"] != "fp1" { - t.Errorf("the moved reservation is %+v: want its digest kept, nothing published, built, 0.1.39 withdrawn", moved) - } - if at, _ := r.AtVersion(ctx, "0.1.41"); len(at) != 1 { - t.Errorf("the index by version does not name the moved reservation: %v", at) + held = []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40", "0.2.0"} + // The build again of a done key reserves as a rebuild (A-490b): a withdrawn number is never its floor's to reuse. + again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true, Rebuild: true}) + if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != GoTagOf("0.1.41") { + t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest and 0.1.39 kept withdrawn", again, err) } if at, _ := r.AtVersion(ctx, "0.1.39"); len(at) != 0 { - t.Errorf("the index by version still names the old number: %v", at) + t.Errorf("the index by version still names the withdrawn number: %v", at) } - if _, _, ok, _ := r.Move(ctx, "node-tools", "0.1.39", nil); ok { - t.Error("a reservation no longer at the version was moved again") + if _, ok, _ := r.Withdraw(ctx, "node-tools", "0.1.39"); ok { + t.Error("a number no reservation stands at any more was withdrawn") } // A registry that cannot be read lets the reserve proceed. readErr, skipped = errors.New("registry down"), nil