From 564fd5e0bbafe751680d8406b69da84185d993e2 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:26:22 +0100 Subject: [PATCH 1/8] H-A-486: an upstream artifact needs no publish to release Design 51 copies an upstream image by its pinned digest, and distribution's mirror says published with no version, so nats, hello-web, mailu and showcase could never be released. The reservation keeps its upstream refs (Upstream, filled at the reserve from the plan's upstream tasks) and Missing never awaits them; the indexer releases a version once every other artifact is published, naming each upstream artifact with the pin its manifest gives, and refuses the release loudly when an upstream artifact names no pin. Builds: design 47 H2 H-A-486, H-A-486 (addendum) --- .../internal/build/numbers.go | 3 ++ .../internal/build/reserve_test.go | 7 +++ .../internal/versions/reserve.go | 11 +++- .../internal/versions/reserve_test.go | 17 ++++++ .../internal/index/versions.go | 52 ++++++++++++++++++- .../internal/index/versions_test.go | 30 +++++++++++ 6 files changed, 117 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index c34e611bc..b14cca58b 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -199,6 +199,9 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu } src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild} for _, t := range plan.Tasks { + if t.Kind == catalogue.ArtifactUpstream { + src.Upstream = append(src.Upstream, module+":"+t.Artifact) // copied by its pin, never awaited (H-A-486) + } if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage { src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796) } diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index 37d11baa0..bcc214c9a 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -118,6 +118,13 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) { if res.built["m:tool"] != digest || len(res.built) != 2 { t.Errorf("recorded %v, want the digest of m:tool under the reserved version", res.built) } + // H-A-486: an upstream artifact is reserved for the record and named as upstream, so its publish is never awaited. + up := &fakeReservations{filled: true} + plan := Plan{Series: "0.1", Tasks: []Task{{Artifact: "server", Kind: catalogue.ArtifactUpstream}, {Artifact: "tools", Kind: catalogue.ArtifactBundle}}} + if _, ok := (&Builder{Reserved: up}).reserve(context.Background(), "nats", meshcatalog.ModulesByModuleValue{Fp2: "f"}, plan, Origin{}, false); !ok || + !slices.Equal(up.reserved[0], []string{"nats:server", "nats:tools"}) || !slices.Equal(up.sources[0].Upstream, []string{"nats:server"}) { + t.Errorf("an upstream artifact reserved %v from %+v, want both refs reserved and nats:server named upstream", up.reserved, up.sources) + } r.consider(t, "modules/m") if len(res.reserved) != 1 { t.Errorf("a done key reserved again: %v", res.reserved) diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index b76a1534c..724c5f282 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -55,6 +55,10 @@ type ReservedVersion struct { Version string `json:"version"` // MAJOR.MINOR.PATCH // Artifacts are the module's artifacts at the version: the digest once built, empty before. Artifacts map[string]string `json:"artifacts"` + // Upstream are the artifacts of kind upstream, by reference (design 47 H2 H-A-486): kept in Artifacts with their digest for the + // record, never awaited, as design 51 copies an upstream image by its pinned digest and no registry seat says it published at + // a version. A reservation written before carries none. + Upstream []string `json:"upstream,omitempty"` // BuiltAt is when the build ended with every digest known; the publish's bound counts from it. BuiltAt time.Time `json:"built-at,omitzero"` // Published are the artifacts a registry seat said published at this version with the built digest. @@ -126,6 +130,8 @@ type Source struct { Revision int64 // CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did. CatalogRevision int64 + // Upstream are the artifacts of kind upstream among those reserved, by reference (H-A-486), kept on the reservation. + Upstream []string // Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced // and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b). Rebuild bool @@ -377,6 +383,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts for _, a := range artifacts { out.Artifacts[a] = "" } + out.Upstream = slices.Clone(src.Upstream) if found { out.Withdrawn = maps.Clone(cur.Withdrawn) if cur.Fingerprint != src.Fingerprint { @@ -1134,11 +1141,11 @@ const ( ReservedVerb = meshbuilds.VerbReserved ) -// Missing are the artifacts of v no registry seat said published, sorted. +// Missing are the artifacts of v no registry seat said published, sorted; an upstream one is never awaited (H-A-486). func (v ReservedVersion) Missing() []string { var out []string for a := range v.Artifacts { - if !v.Published[a] { + if !v.Published[a] && !slices.Contains(v.Upstream, a) { out = append(out, a) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 3fadf9cd4..a0d96f966 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -171,6 +171,23 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { if got := cur.Missing(); !reflect.DeepEqual(got, []string{"m:b"}) { t.Errorf("missing = %v, want m:b", got) } + // H-A-486: an upstream artifact is reserved and recorded with its digest, but never awaited: once the others are published + // nothing is missing. + up, err := r.Reserve(ctx, "u", "0.1", []string{"u:server", "u:tools"}, Source{Upstream: []string{"u:server"}}) + if err != nil || !reflect.DeepEqual(up.Upstream, []string{"u:server"}) { + t.Fatalf("reserved %+v (%v), want u:server kept as upstream", up, err) + } + for _, a := range []string{"u:server", "u:tools"} { + if _, err := r.Built(ctx, "u", up.Version, a, "sha256:"+a, nil); err != nil { + t.Fatal(err) + } + } + if marked, _, err := r.Published(ctx, "u-tools", up.Version, "sha256:u:tools"); err != nil || len(marked) != 1 { + t.Fatalf("the published of u:tools marked %v (%v)", marked, err) + } + if cur, _, _ := r.At(ctx, ReservedKey("u", "0.1")); len(cur.Missing()) != 0 || cur.Artifacts["u:server"] != "sha256:u:server" { + t.Errorf("missing = %v with artifacts %v, want none missing and the upstream digest kept", cur.Missing(), cur.Artifacts) + } if key, ok := ReservedBoundOf(ReservedBoundKey("m", "1.4")); !ok || key != ReservedKey("m", "1.4") || ReservedBoundFor(key) != ReservedBoundKey("m", "1.4") { t.Errorf("the bound key does not name the reservation back: %q %v", key, ok) } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 71336e1f3..ad9f2f8d4 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -140,7 +140,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag } var want []string for _, a := range entry.Artifacts { - want = append(want, a.Ref) + if a.Kind != catalogue.ArtifactUpstream { // an upstream copy is released by its pinned digest, never published (H-A-486) + want = append(want, a.Ref) + } } key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version) var have published @@ -211,7 +213,15 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0 at := ix.now().UTC() v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late} + pins, err := upstreamPins(entry) + if err != nil { + return err + } for _, a := range entry.Artifacts { + if a.Kind == catalogue.ArtifactUpstream { + v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: a.Ref, Digest: pins[a.Ref]}) + continue + } v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref]) } slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) }) @@ -255,6 +265,46 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli return ix.forgetPublished(ctx, pendingKey) } +// upstreamPins are the pinned digests of the entry's upstream artifacts, by reference, read from its manifest's from +// (`@sha256:`): design 51 copies an upstream image by that digest, so a release names it (H-A-486). An upstream +// artifact whose manifest names no pin refuses the release, naming it. +func upstreamPins(entry meshcatalog.CatalogEntry) (map[string]string, error) { + out := map[string]string{} + if !slices.ContainsFunc(entry.Artifacts, func(a meshcatalog.Artifact) bool { return a.Kind == catalogue.ArtifactUpstream }) { + return out, nil + } + raw, err := json.Marshal(entry.Manifest) + if err != nil { + return nil, err + } + var m struct { + Build struct { + Artifacts []struct { + Name string `json:"name"` + Kind string `json:"kind"` + From string `json:"from"` + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("the manifest of %s's entry does not read, so its upstream artifacts' pins are not known: %w", entry.Module, err) + } + for _, a := range m.Build.Artifacts { + if a.Kind != catalogue.ArtifactUpstream { + continue + } + if _, d, ok := strings.Cut(a.From, "@"); ok && strings.HasPrefix(d, "sha256:") { + out[ArtifactRefOf(entry.Module, a.Name)] = d + } + } + for _, a := range entry.Artifacts { + if _, ok := out[a.Ref]; a.Kind == catalogue.ArtifactUpstream && !ok { + return nil, fmt.Errorf("%s is an upstream artifact whose manifest names no pinned digest (from @sha256:), so %s releases no version", a.Ref, entry.Module) + } + } + return out, nil +} + // trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR. func (ix *Indexer) trimVersions(ctx context.Context, module string) error { kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index b4c56076a..8e7e2e663 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -83,6 +83,36 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) { if compareVersions(oldest, "0.1.5") < 0 { t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest) } + + // H-A-486: an upstream artifact is copied by its pinned digest and never published, so the version is released once every + // other artifact is, the upstream one named with the pin its manifest gives; one whose manifest names no pin releases none. + pin := "sha256:" + strings.Repeat("ab", 32) + nats := func(from string) map[string]meshcatalog.CatalogEntry { + return map[string]meshcatalog.CatalogEntry{"nats": {Module: "nats", Manifest: map[string]any{"build": map[string]any{"artifacts": []any{ + map[string]any{"name": "server", "kind": "upstream", "from": from}, + map[string]any{"name": "tools", "kind": "bundle", "from": "."}}}}, + Artifacts: []meshcatalog.Artifact{{Ref: "nats:server", Kind: "upstream"}, {Ref: "nats:tools", Kind: "bundle", Language: "go"}}}} + } + for i, c := range []struct { + from, version string + released bool + }{{"nats:2.12-alpine@" + pin, "0.1.1", true}, {"nats:2.12-alpine", "0.1.2", false}} { + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: int64(4 + i), Kind: "trunk"}, nats(c.from)) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + err = ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "nats", Artifact: "nats:tools", Digest: "sha256:tools", Version: c.version, Source: 9, Branch: "main"}) + got, _ := ModuleVersions(ctx, ix.Versions, "nats") + switch { + case c.released && (err != nil || len(got) != 1 || len(got[0].Artifacts) != 2 || got[0].Artifacts[0] != (meshcatalog.VersionArtifact{Artifact: "nats:server", Digest: pin})): + t.Fatalf("pinned %q: released %+v (%v); want the server named with its pin", c.from, got, err) + case !c.released && (err == nil || !strings.Contains(err.Error(), "no pinned digest") || len(got) != 1): + t.Fatalf("unpinned %q: released %+v (%v); want a refusal naming the pin", c.from, got, err) + } + } } // Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or From db899d8ad4d9a7a57b8ad83108e19ba7ddb3528d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:30:59 +0100 Subject: [PATCH 2/8] H-A-487: a module's builds are read without every key's history builder_build read every key's whole history to learn when its entry was written, one more round trip per key of the module; on nats's hundreds of keys the read crawled. Each entry's own write time now comes with the key's read, the history is read only for the one key a fingerprint names when its entry says no time, and a listing cut at the 20 newest says so in its note. Builds: design 47 H2 H-A-487 --- .../mesh-delivery-builder-tools/build_test.go | 14 +++++++++++ .../cmd/mesh-delivery-builder-tools/main.go | 11 +++++---- .../internal/build/buildread.go | 16 +++++++++++-- .../internal/build/buildread_test.go | 23 +++++++++++++++++++ 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go index 7feed31e5..c641ec822 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go @@ -5,9 +5,12 @@ import ( "errors" "strings" "testing" + "time" "mesh-delivery-builder/internal/build" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) @@ -54,3 +57,14 @@ func TestABuildSeatsStepsAreReadInItsHoldersState(t *testing.T) { t.Errorf("a seat with no holder reads %v; want an error naming it", err) } } + +// H-A-487: a build key's time is its entry's own write time, read with the key and no history. +func TestABuildKeysTimeIsItsEntrysOwnWriteTime(t *testing.T) { + at := time.Date(2026, 11, 22, 20, 45, 0, 0, time.UTC) + raw, rev, got, err := entryOf(func(key string) (*stdio.StateEntry, error) { + return &stdio.StateEntry{Key: key, Value: json.RawMessage(`{}`), Revision: 4, At: at}, nil + })("k") + if err != nil || string(raw) != "{}" || rev != 4 || !got.Equal(at) { + t.Errorf("entry = %s rev %d at %v (%v); want the entry's own write time %v", raw, rev, got, err, at) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 75e1c08d1..9e164d827 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -430,7 +430,7 @@ func main() { return a, json.Unmarshal(raw, &a) } tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr, reserve) - tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")), + tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds").Get), Asked: read(stdio.State("asked")), Newest: newestVia(stdio.Ask), Steps: r})) tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) as := stdio.State(versions.ActedState) @@ -481,13 +481,14 @@ func main() { } } -// entryOf reads a key with its revision and when it was written. -func entryOf(s stdio.KeptState) func(key string) (json.RawMessage, uint64, time.Time, error) { +// entryOf reads a key with its revision and when it was written, the entry's own write time: no key's history is read +// (design 47 H2 H-A-487). +func entryOf(get func(key string) (*stdio.StateEntry, error)) func(key string) (json.RawMessage, uint64, time.Time, error) { return func(key string) (json.RawMessage, uint64, time.Time, error) { - e, err := s.Get(key) + e, err := get(key) if err != nil || e == nil { return nil, 0, time.Time{}, err } - return e.Value, e.Revision, written(s, key, e.Revision), nil + return e.Value, e.Revision, e.At, nil } } diff --git a/modules/mesh-delivery-builder/internal/build/buildread.go b/modules/mesh-delivery-builder/internal/build/buildread.go index 11b06ff28..56b08045a 100644 --- a/modules/mesh-delivery-builder/internal/build/buildread.go +++ b/modules/mesh-delivery-builder/internal/build/buildread.go @@ -19,6 +19,8 @@ import ( // `module` alone the module's keys newest first, at most MostBuilds, each with its fingerprint whole, its state, revision and // since when, its step and node, its error whole, event-builds and the open ask (§568), and `newest` naming each watched branch // whose newest catalog graph entry of the module names that fingerprint, read from the catalog half by the tools bundle. +// **Each key's time is its entry's own write time** (design 47 H2 H-A-487): the history of a key is read only for the one key a +// fingerprint names, when its entry does not say when it was written, and a listing cut at MostBuilds says so in its note. // BuildTool is the builder's read tool of a module's build keys. const BuildTool = "builder_build" @@ -56,14 +58,16 @@ type BuildAnswer struct { Builds []BuildRow `json:"builds"` // Newest is the fingerprint of the module's newest catalog graph entry on each watched branch. Newest map[string]string `json:"newest-entries,omitempty"` - // Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint. + // Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint, and that the listing + // was cut at MostBuilds (H-A-487). Note string `json:"note,omitempty"` } // BuildReader reads what builder_build answers. type BuildReader struct { Builds StateRead - // Entry is a key's value with its revision and when it was written; nil leaves both out. + // Entry is a key's value with its revision and when it was written, the entry's own write time (H-A-487); nil leaves both + // out. Builds.Written, the key's history, is read only for the key a fingerprint names when Entry says no time. Entry func(key string) (json.RawMessage, uint64, time.Time, error) // Asked is the builder's state asked, which holds the asks the operator was asked. Asked StateRead @@ -96,6 +100,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) { if raw == nil { continue } + if at.IsZero() && fingerprint != "" && r.Builds.Written != nil { + at = r.Builds.Written(key) // one key's history, never every key's (H-A-487) + } v, err := builds.DecodeBuilderBuild(raw) if err != nil { return out, fmt.Errorf("%s holds what is not a build: %w", key, err) @@ -118,7 +125,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) { } return rows[i].Key > rows[j].Key }) + var cut string if len(rows) > MostBuilds { + cut = fmt.Sprintf("%d of %s's %d build keys are answered, the newest by write time; name a fingerprint for another", MostBuilds, module, len(rows)) rows = rows[:MostBuilds] } asks := r.openAsks() @@ -149,6 +158,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) { out.Note = "no build key of " + module + " is the newest entry's: " + strings.Join(says, "; ") } } + if cut != "" { + out.Note = strings.TrimPrefix(out.Note+"; "+cut, "; ") + } out.Builds = append(out.Builds, rows...) return out, nil } diff --git a/modules/mesh-delivery-builder/internal/build/buildread_test.go b/modules/mesh-delivery-builder/internal/build/buildread_test.go index 92be0c203..37f609133 100644 --- a/modules/mesh-delivery-builder/internal/build/buildread_test.go +++ b/modules/mesh-delivery-builder/internal/build/buildread_test.go @@ -2,6 +2,7 @@ package build import ( "encoding/json" + "fmt" "sort" "strings" "testing" @@ -66,6 +67,28 @@ func TestBuilderBuildAnswersAModulesKeysWhole(t *testing.T) { if n, err := r.Build("m", ""); err != nil || !strings.Contains(n.Note, "no build key of m is the newest entry's") || !strings.Contains(n.Note, "main names src2:"+strings.Repeat("c", 64)) { t.Errorf("a module none of whose keys the newest entry names: %+v, %v", n, err) } + + // H-A-487: a module of 100 keys is read by each entry's own write time, no key's history read, and the listing is cut at the + // 20 newest, saying so; a fingerprint's key with no time in its entry has its one history read. + many, at := map[string][]byte{}, map[string]time.Time{} + base := time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC) + for i := range 100 { + k, _ := builds.BuilderBuildKey("big", fmt.Sprintf("src2:%064x", i)) + many[k], at[k] = done, base.Add(time.Duration(i)*time.Minute) + } + histories := 0 + st := stateOf(many) + st.Written = func(string) time.Time { histories++; return base } + big := &BuildReader{Builds: st, Entry: func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, at[k], nil }} + a, err = big.Build("big", "") + if err != nil || len(a.Builds) != MostBuilds || !a.Builds[0].Since.Equal(base.Add(99*time.Minute)) || histories != 0 || + !strings.Contains(a.Note, "20 of big's 100 build keys are answered") { + t.Fatalf("100 keys: %d rows, newest %v, %d histories read, note %q; want the 20 newest, no history, the cut said", len(a.Builds), a.Builds[0].Since, histories, a.Note) + } + big.Entry = func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, time.Time{}, nil } + if one, err := big.Build("big", fmt.Sprintf("src2:%064x", 7)); err != nil || len(one.Builds) != 1 || !one.Builds[0].Since.Equal(base) || histories != 1 { + t.Errorf("one fingerprint with no time in its entry: %+v (%v), %d histories; want its one history read", one, err, histories) + } } // Design 47 G §589: builder_stuck answers each key's fingerprint whole, as the build-input fingerprint the catalog graph entry carries it. From fcac1fc5d08428c9ef06c86f7ddcc215203da030 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:37:22 +0100 Subject: [PATCH 3/8] H-A-487: prove a listing reads no history when its entries carry no time Review round 1 found the fingerprint guard of the build read unproven: every entry of the 100-key listing carried a time. The listing is now also read with entries that say none, and no history may be read. Builds: design 47 H2 H-A-487 (review round 1 finding) --- modules/mesh-delivery-builder/internal/build/buildread_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/mesh-delivery-builder/internal/build/buildread_test.go b/modules/mesh-delivery-builder/internal/build/buildread_test.go index 37f609133..5602a5874 100644 --- a/modules/mesh-delivery-builder/internal/build/buildread_test.go +++ b/modules/mesh-delivery-builder/internal/build/buildread_test.go @@ -86,6 +86,9 @@ func TestBuilderBuildAnswersAModulesKeysWhole(t *testing.T) { t.Fatalf("100 keys: %d rows, newest %v, %d histories read, note %q; want the 20 newest, no history, the cut said", len(a.Builds), a.Builds[0].Since, histories, a.Note) } big.Entry = func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, time.Time{}, nil } + if a, err := big.Build("big", ""); err != nil || len(a.Builds) != MostBuilds || histories != 0 { + t.Fatalf("100 keys with no time in their entries: %d rows (%v), %d histories read; want no history read for a listing", len(a.Builds), err, histories) + } if one, err := big.Build("big", fmt.Sprintf("src2:%064x", 7)); err != nil || len(one.Builds) != 1 || !one.Builds[0].Since.Equal(base) || histories != 1 { t.Errorf("one fingerprint with no time in its entry: %+v (%v), %d histories; want its one history read", one, err, histories) } From 73cb9ba690141ec5824b98e37b5f667cccd5eb01 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:45:03 +0100 Subject: [PATCH 4/8] H-A-489: build only a branch's newest entry; withdrawn is final Two fingerprints of one module took the series from each other (nats flipped between 0.1.140 and 0.1.152): a later look built an older fingerprint again, and its reservation took the newer one's over. The builder now refuses, before it reserves or builds, a fingerprint of the first watched branch that is no longer the newest entry heard there, saying once per key that it is superseded; a step ask of it fails so, never waits. It looks again just before it reserves. A key withdrawn is taken again by no look, only by a person's build again. Builds: design 47 H2 H-A-489 --- .../internal/build/claim.go | 3 + .../internal/build/run.go | 62 +++++++++++++++++-- .../internal/build/run_test.go | 59 +++++++++++++++++- .../internal/build/step.go | 6 +- .../internal/build/step_test.go | 18 ++++++ 5 files changed, 141 insertions(+), 7 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index ed4bc088f..35f0218db 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -170,6 +170,7 @@ func Unreferenced(b builds.BuilderBuild) bool { // clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4), // which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for // another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder); +// - a failed key withdrawn (withdrawnPrefix) is final for its fingerprint, taken again only by a person's build again (H-A-489); // - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is // published (ClaimOptions.Published, ADR 0500 clause 6), and not before; // - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its @@ -229,6 +230,8 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string return nil, nil case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): return nil, nil + case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, withdrawnPrefix) && !opt.ByHand: + return nil, nil // withdrawn is final for its fingerprint: no look builds it again (H-A-489) case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver && !opt.ByEvent: return nil, nil case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()): diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 0e4a3be59..936b27fc6 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -177,6 +177,8 @@ type Builder struct { // publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished): // a package with no pin waits for one (H-A-192). publishedPins map[string]map[string]string + // supersededSaid are the build keys refused as superseded (H-A-489), so each is said once. + supersededSaid sync.Map } type planned struct { @@ -310,11 +312,49 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog. if superseded != nil { // A newer fingerprint of the module: the build of the older one in flight is taken out, and its asks are withdrawn // from the build seats (ADR 0664 §4). - superseded(buildrun.Withdrawn{Why: "the build of " + key + " at " + old.Fp2 + " was superseded by the fingerprint " + m.Fp2}) + superseded(buildrun.Withdrawn{Why: supersededWhy(key, old.Fp2, m.Fp2)}) } b.consider(ctx, key, m) } +// supersededWhy is the words of a build withdrawn because a newer fingerprint of its module superseded it. +func supersededWhy(module, fp, newer string) string { + return "the build of " + module + " at " + fp + " was superseded by the fingerprint " + newer +} + +// withdrawnPrefix opens the error of a key whose build was withdrawn (H-A-225, ADR 0664 §4). It is final for its fingerprint +// (H-A-489): no look builds it again; a person's build again does. +const withdrawnPrefix = "withdrawn: " + +// supersededBy is the fingerprint that supersedes fp, a fingerprint of module whose build key is key, when fp is of its +// registration's first watched branch and is no longer the newest entry heard on that branch (H-A-489); "" otherwise. A key no +// entry named is of the first branch, as originOf has it. +func (b *Builder) supersededBy(module, key, fp string) string { + b.mu.Lock() + defer b.mu.Unlock() + o, ok := b.origin[key] + if !ok { + o = Origin{First: true} + } + if !o.First { + return "" + } + newest, heard := b.heard[o.Branch+"\x00"+module] + if !heard || newest.Fp2 == "" || newest.Fp2 == fp { + return "" + } + return newest.Fp2 +} + +// saySuperseded says, once per key, that the build of fp is refused as superseded by newer (H-A-489), and gives the words. +func (b *Builder) saySuperseded(module, key, fp, newer string) string { + why := "is not built: superseded by " + newer + ", the newest entry heard on its first branch" + if _, said := b.supersededSaid.LoadOrStore(key, true); !said { + b.logf("%s at %s: %s", module, fp, why) + } + return why +} + // ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository, // directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is // built against, its packages and needs, and the named paths of its artifacts when every artifact names them. @@ -637,6 +677,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB whyEnded = fmt.Sprintf(format, args...) b.logf("%s at %s: %s", module, m.Fp2, whyEnded) } + // A fingerprint no longer the newest entry heard on its first branch is neither reserved for nor built (H-A-489): two + // fingerprints of one module never take the series from each other. + if newer := b.supersededBy(module, key, m.Fp2); newer != "" { + whyEnded = b.saySuperseded(module, key, m.Fp2, newer) + return + } plan, planErr, read := b.planOf(ctx, m.Manifest) var noModule *NoBuildModule if errors.As(planErr, &noModule) { @@ -727,7 +773,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } } var reserved versions.ReservedVersion - hasReserved := false + hasReserved, supersededNow := false, "" if rebuild && cl.Build.State == builds.BuildBuilding { // Reserved as A-490b amended by H-A-225 has it: this fingerprint's standing reservation is kept while it is unreleased // and above the highest the catalog's versions answers, else the next above that highest is taken (A-455); the build's @@ -737,10 +783,16 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } reserving = true } else if reserving && cl.Build.State != builds.BuildDone { - reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false) + // Looked at again just before it reserves (H-A-489): a newer entry heard since the claim is not taken over. + if supersededNow = b.supersededBy(module, key, m.Fp2); supersededNow == "" { + reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false) + } } var result builds.BuilderBuild - if reserving && !hasReserved && cl.Build.State != builds.BuildDone { + if supersededNow != "" { + // Ended as the newer entry's take-in withdraws a build in flight: final for this fingerprint. + result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + supersededWhy(module, m.Fp2, supersededNow)})} + } else if reserving && !hasReserved && cl.Build.State != builds.BuildDone { // A trunk build that reserves is never built without its version: it would end done and publish nothing, and its key would // stay done, never built again (A-490). The key fails with why and is taken again at the next look or by builder_build_again. result = failed("no version is reserved for this build: the builder's journal says why, and it is built again once one can be") @@ -765,7 +817,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB var w buildrun.Withdrawn if ctx.Err() == nil && errors.As(context.Cause(run), &w) { // Taken out, not stopped: the key says so, and the asks are withdrawn (the asker dropped them). - result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{"withdrawn: " + w.Why})} + result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + w.Why})} } } if ctx.Err() != nil { diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 4ccd5a305..60c84d914 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -333,8 +333,9 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) { r := newRig(t, goBundle) asked, release := make(chan struct{}), make(chan struct{}) answer := r.answer + var once sync.Once r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { - close(asked) + once.Do(func() { close(asked) }) <-release return answer(a) } @@ -356,6 +357,62 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) { if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "withdrawn: its reservation 0.1.3 was taken over") { t.Fatalf("the key = %+v, want failed withdrawn naming the take-over", v) } + // H-A-489: withdrawn is final for the fingerprint: a later look builds it not again; a person's build again does. + n, rev := len(r.asks.Asks()), r.st.revs[r.key] + r.b.Reconsider(context.Background()) + r.b.Wait() + if v := valueOf(t, r.st, r.key); len(r.asks.Asks()) != n || r.st.revs[r.key] != rev || !strings.HasPrefix(v.Error, "withdrawn: ") { + t.Errorf("a later look built the withdrawn key again: %d asks (was %d), the key %+v", len(r.asks.Asks()), n, v) + } + if cl, err := r.b.Claimer.Claim(context.Background(), "m", fp1, mdigest('a'), ClaimOptions{ByHand: true}); err != nil || cl == nil { + t.Errorf("a person's build again of the withdrawn key was not claimed: %v", err) + } +} + +// H-A-489: a fingerprint of the first branch that is no longer its newest entry heard is neither reserved for nor built, and +// said once; the again loop that looks at it later builds nothing. +func TestAnOlderFingerprintAskedAfterANewerEntryIsRefused(t *testing.T) { + r := newRig(t, goBundle) + var mu sync.Mutex + var said []string + r.b.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() } + r.bm.fails["ace/build_go_build"] = "go build of tool: boom" + older := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: fp1, Manifest: mdigest('a'), + Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)} + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.Wait() + delete(r.bm.fails, "ace/build_go_build") + newer := older + newer.Fp2 = fp2 + data, err := meshcatalogclient.EncodeModule(newer) + if err != nil { + t.Fatal(err) + } + r.b.Module(context.Background(), "m", data) + r.b.Wait() + rev := r.st.revs[r.key] + for range 2 { + r.b.consider(context.Background(), "m", older) // as consider's again loop or a step's look takes it + r.b.Wait() + } + if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || r.st.revs[r.key] != rev { + t.Errorf("the older key = %+v at revision %d (was %d), want it left as it failed", v, r.st.revs[r.key], rev) + } + k2, _ := builds.BuilderBuildKey("m", fp2) + if v := valueOf(t, r.st, k2); v.State != builds.BuildDone { + t.Errorf("the newer key = %+v, want done", v) + } + mu.Lock() + defer mu.Unlock() + n := 0 + for _, l := range said { + if strings.Contains(l, fp1+": is not built: superseded by "+fp2) { + n++ + } + } + if n != 1 { + t.Errorf("superseded said %d times, want once: %q", n, said) + } } // 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index cd27c0ad8..5be6332f0 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -142,6 +142,10 @@ func (b *Builder) buildEntry(ctx context.Context, pipeline string, e meshbuilds. } return answered(meshbuilds.StepEndedOutcomeFailed, failedWords, "existing build") } + if newer := b.supersededBy(entry.Module, key, entry.Fp); newer != "" { + // Refused, not waited on (H-A-489): no look builds a superseded fingerprint, so its end would never come. + return answered(meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s %s", entry.Module, entry.Fp, b.saySuperseded(entry.Module, key, entry.Fp, newer)), "refused") + } m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path, Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages} b.consider(ctx, entry.Module, m) @@ -232,7 +236,7 @@ func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed str switch { case v.State == builds.BuildDone && !Unpublished(v, b.Seats): return true, "", true - case v.State == builds.BuildFailed && builds.FailedUntilChanged(v.Error): + case v.State == builds.BuildFailed && (builds.FailedUntilChanged(v.Error) || strings.HasPrefix(v.Error, withdrawnPrefix)): return false, key + ": " + v.Error, true } return false, "", false diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 1cab29a70..63ef38218 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -460,6 +460,24 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) { if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 { t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources) } + // H-A-489: once the picture heard the newer entry of m on the first branch, the older ask is refused superseded and + // nothing is built or reserved for it; the newer entry's build keeps the series. + rn, resn, entryn := revisionRig(t, &held, make(chan struct{}, 1)) + rn.b.RevisionWait = time.Nanosecond + rn.b.FirstBranch = r.b.FirstBranch + rn.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + return 45, map[string]string{"m": fp1}, nil + } + entryn.Fp = fp2 + rn.b.Entry(context.Background(), Origin{Branch: "main", Source: 46, First: true}, entryn) + rn.b.Wait() + if endn := rn.b.BuildStep(t.Context(), "a1", run); endn.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(endn.Said, "superseded by "+fp2) { + t.Errorf("the older ask after the newer entry: end = %+v; want failed, superseded by %s", endn, fp2) + } + rn.b.Wait() + if _, built := rn.st.vals[rn.key]; built || len(resn.sources) != 1 || resn.sources[0].Fingerprint != fp2 { + t.Errorf("built %v, reserved %+v; want nothing built for the older, one reservation, the newer's", built, resn.sources) + } r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) r2.b.RevisionWait = time.Nanosecond r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { From ffa58e51c4bcb7869b4cc77aa390f2db2b32a9d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:51:59 +0100 Subject: [PATCH 5/8] H-A-489: test the withdrawn step answer and the look before reserving Review round 1 found two paths untested: a step ask of a withdrawn key still newest answers failed at once, and a newer entry heard between the older build's claim and its reservation is not taken over. Each row fails without its line. The comments of H-A-205 say that H-A-489 overrides it once the newer entry is heard. Builds: design 47 H2 H-A-489 (review round 1) --- .../internal/build/revisionwait.go | 4 +++- .../internal/build/run_test.go | 23 +++++++++++++++++++ .../internal/build/step_test.go | 22 +++++++++++++++--- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 651d6f303..f3c98287b 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -107,7 +107,9 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, } } // Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture - // took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded. + // took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded. Once the picture + // heard a newer entry of the module on the first branch, H-A-489 overrides H-A-205: the older fingerprint is refused + // superseded (buildEntry), never built or reserved. b.mu.Lock() for _, e := range run.Entries { key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint) diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 60c84d914..383bca35b 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -369,6 +369,29 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) { } } +// H-A-489 (review round 1): a newer entry of the first branch heard between the older build's claim and its reservation is +// not taken over: the older key ends failed withdrawn with no reservation, and the newer reserves. +func TestANewerEntryHeardBeforeTheReservationIsNotTakenOver(t *testing.T) { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + res := &fakeReservations{filled: true} + r.b.Reserved = res + newer, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", + Fp2: fp2, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)}) + if err != nil { + t.Fatal(err) + } + r.st.before = func() { r.b.Module(context.Background(), "m", newer) } // at the older key's claim + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, "withdrawn: ") || !strings.Contains(v.Error, "superseded by the fingerprint "+fp2) { + t.Errorf("the older key = %+v, want failed withdrawn, superseded by %s", v, fp2) + } + if len(res.sources) != 1 || res.sources[0].Fingerprint != fp2 { + t.Errorf("reserved %+v, want one reservation, the newer's", res.sources) + } +} + // H-A-489: a fingerprint of the first branch that is no longer its newest entry heard is neither reserved for nor built, and // said once; the again loop that looks at it later builds nothing. func TestAnOlderFingerprintAskedAfterANewerEntryIsRefused(t *testing.T) { diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 63ef38218..69abfe82d 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -74,6 +74,21 @@ func TestABuildStepAnswersItsEnd(t *testing.T) { if end := ask(entries...); end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Step != meshbuilds.StepEndedStepBuild || len(end.Entries) != 1 || end.Entries[0].Commit != "c1" || end.Pipeline != "main.1.m" { t.Errorf("a done build: %+v", end) } + // H-A-489 (review round 1): a withdrawn key whose fingerprint is still the newest entry is answered failed at once, naming + // withdrawn: no look builds it again, so its end is never waited for. + rw := newRig(t, catalogue.Artifact{Name: "files", Kind: "archive"}) + rw.b.StepEntry = r.b.StepEntry + withdrawn, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Manifest: mdigest('a'), Error: withdrawnPrefix + "its reservation 0.1.3 was taken over"}) + if _, err := rw.st.Create(t.Context(), rw.key, withdrawn); err != nil { + t.Fatal(err) + } + rw.b.Module(t.Context(), "m", moduleValue(t, "modules/m")) // the newest entry heard is fp1 + rw.b.Wait() + bounded, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + if end := rw.b.BuildStep(bounded, "ask1", meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries}); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "withdrawn: ") { + t.Errorf("a withdrawn key asked: %+v, want failed at once naming withdrawn", end) + } } // A step ask waits for the end of its build without a timer: the log entry of that end wakes it (design 47 G A-97). @@ -434,9 +449,10 @@ func aStepAskWaitsForItsRevision(t *testing.T) { } -// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved -// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs; -// a kept revision naming another fingerprint fails the step, naming it. +// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, while the older pipeline +// asks the build of revision 45. Its kept catalog revision is read by number and the build runs while the picture never heard +// a newer entry of m (a restart); once it heard one on the first branch, H-A-489 overrides H-A-205 and the older ask is refused +// superseded. A kept revision naming another fingerprint fails the step, naming it. func anOlderKeptRevisionIsReadByNumber(t *testing.T) { var held atomic.Int64 held.Store(46) From 21daeb88937a472dfe26ed5e95e535d9928ef451 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 20:17:14 +0100 Subject: [PATCH 6/8] H-A-492: record a version's digests under the builder's own bound A takeover or an end cancelling a build's context cancelled the record of its reservation's digests eight times in a day, each leaving a passed build unreleased. The record now runs under the builder's own bound (one minute, not cancelled with the build), and a done build whose digests are not recorded is recorded from its done key at the builder's next look and at its start (before H-A-484's start check holds it against the released versions), said each time. Builds: design 47 H2 H-A-492 --- .../cmd/mesh-delivery-builder/main.go | 1 + .../cmd/mesh-delivery-builder/reserve.go | 31 +++++++++ .../cmd/mesh-delivery-builder/reserve_test.go | 17 +++++ .../internal/build/numbers.go | 47 +++++++++++++ .../internal/build/reserve_test.go | 66 ++++++++++++++++++- .../internal/build/run.go | 15 ++++- 6 files changed, 174 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 3f0e8e7b4..290698c49 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -259,6 +259,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro b.Reserved = reserved // A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225). reserved.withdraw = b.Withdraw + reserved.recordAgain = b.RecordAgain // a done build's digests its end did not record are recorded at the start (H-A-492) // The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it. reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps) // A state bound later numbers the builds looked at without it. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 5ffcead8b..49867fb0b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -57,6 +57,9 @@ type reservations struct { logf func(string, ...any) // withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none. withdraw func(module, fingerprint, why string) + // recordAgain records the digests of a done build of module at fp its end did not record, saying so (design 47 H2 H-A-492); + // nil records none. + recordAgain func(ctx context.Context, module, fp, when string) (string, error) // call asks a seat's verb: the git seat's tag (design 47 G §370). call toolCaller // pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands @@ -363,12 +366,20 @@ func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionRelea // holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one // said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended. // A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released. +// Before that, a standing reservation with no digests recorded whose build key is done has them recorded from the done key +// (design 47 H2 H-A-492), so it is held against the released versions too. func (rv *reservations) releasedAtStart(ctx context.Context) { all, err := rv.r.All(ctx) if err != nil { rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err) return } + if rv.recordedAtStart(ctx, all) { + if all, err = rv.r.All(ctx); err != nil { + rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err) + return + } + } read := map[string][]string{} for _, v := range builtOf(all) { if versions.NotStanding(v, true) != "" { @@ -392,6 +403,26 @@ func (rv *reservations) releasedAtStart(ctx context.Context) { } } +// recordedAtStart records the digests of every standing reservation that has none recorded and whose build key is done (design 47 +// H2 H-A-492): a record its build's end lost. It answers whether one was recorded; one that cannot be is said. +func (rv *reservations) recordedAtStart(ctx context.Context, all []versions.ReservedVersion) bool { + if rv.recordAgain == nil { + return false + } + recorded := false + for _, v := range all { + if !v.BuiltAt.IsZero() || v.Fingerprint == "" || versions.NotStanding(v, true) != "" { + continue + } + version, err := rv.recordAgain(ctx, v.Module, v.Fingerprint, "at the builder's start") + if err != nil { + rv.logf("%v", err) + } + recorded = recorded || version != "" + } + return recorded +} + // published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told // before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told // nothing. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 076f709b7..ecb85c53b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -219,6 +219,23 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { // stands. refuse = "" tags = nil + // Design 47 H2 H-A-492: at the start, a standing reservation with no digests recorded whose build key is done has them + // recorded (the builder's RecordAgain), and is then held against the released versions as a built one is. + if _, err := rv.Reserve(ctx, "rec", "0.1", []string{"rec:bin"}, versions.Source{Repository: "o/q", Commit: "abc", Fingerprint: "fpR"}); err != nil { + t.Fatal(err) + } + var recordedAt []string + rv.recordAgain = func(ctx context.Context, module, fp, when string) (string, error) { + recordedAt = append(recordedAt, module+" "+fp+" "+when) + _, err := rv.r.Built(ctx, module, "0.1.0", "rec:bin", "sha256:bb", nil) // its done key's digest + return "0.1.0", err + } + catalog = []meshcatalog.VersionEntry{{Module: "rec", Version: "0.1.0"}} + rv.releasedAtStart(ctx) + if _, standing, _ := rv.r.Of(ctx, "rec", "0.1.0"); standing || !slices.Equal(recordedAt, []string{"rec fpR at the builder's start"}) { + t.Fatalf("at the start, rec 0.1.0 unrecorded and released: standing %v, recorded %q; want it recorded once at the start and released", standing, recordedAt) + } + catalog, rv.recordAgain = nil, nil var ended []string rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) } rv.releasedAtStart(ctx) // built nothing yet: nothing taken diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index b14cca58b..421c3c954 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -122,6 +122,53 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string return out } +// recordBound bounds the record of a reservation's digests (design 47 H2 H-A-492): the builder's own bound, never the build's +// context. +const recordBound = time.Minute + +// recordContext is the context a reservation's digests are recorded under: ctx's values with no cancel of its own, bounded by +// recordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record. +func recordContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(ctx), recordBound) +} + +// RecordAgain records the digests of fp's done build under its standing reservation when its end did not record them (design 47 +// H2 H-A-492): at the builder's look of the key and at its start (when says which), said each time. It answers the version +// recorded, "" when nothing was to be recorded: the key is not done, no reservation stands for fp, or it is recorded already. +func (b *Builder) RecordAgain(ctx context.Context, module, fp, when string) (string, error) { + b.init() + if b.Reserved == nil { + return "", nil + } + digests := b.doneDigests(ctx, module, fp) + if digests == nil { + return "", nil + } + res, ok, err := b.Reserved.ReservedFor(ctx, module, fp) + if err != nil { + return "", fmt.Errorf("%s at %s: whether its done build's digests are recorded cannot be read %s: %w", module, fp, when, err) + } + if !ok || !res.BuiltAt.IsZero() || len(res.Artifacts) == 0 || versions.NotStanding(res, true) != "" { + return "", nil + } + refs := map[string]string{} + for ref := range res.Artifacts { + _, artifact, _ := strings.Cut(ref, ":") + d, have := digests[artifact] + if !have { + return "", fmt.Errorf("%s at %s: the digests of %s are not recorded, and the done key holds no digest of %s to record %s", module, fp, res.Version, ref, when) + } + refs[ref] = d + } + rec, stop := recordContext(ctx) + defer stop() + if err := b.Reserved.Built(rec, module, res.Version, refs, nil); err != nil { + return "", fmt.Errorf("%s at %s: the digests of %s are not recorded %s either: %w", module, fp, res.Version, when, err) + } + b.logf("%s at %s: the digests of %s were not recorded at its build's end: recorded %s from the done key", module, fp, res.Version, when) + return res.Version, nil +} + // PackageNamed looks at a module again when the indexer's packages state names a name for one of its packages // (versions.NamedModule): a done trunk build that left the package unnumbered, its name not known at its claim, is // numbered then with the done build's digests, and version.numbered says them for its build module's late publish. diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index bcc214c9a..839554731 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -32,6 +32,8 @@ type fakeReservations struct { next string // the version Reserve answers when set, else the series' .5 fail error // the reservation cannot be made (the catalog's versions cannot be read) tookBy map[string]versions.ReservedVersion // by fingerprint, the reservation that took its series over (H-A-244) + held map[string]versions.ReservedVersion // by fingerprint, the reservation ReservedFor answers as it stands (H-A-492) + lost error // the record of the digests fails, as a store refusing it would } func (f *fakeReservations) TakenOver(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) { @@ -46,6 +48,9 @@ func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, e func (f *fakeReservations) Filled() bool { return f.filled } func (f *fakeReservations) ReservedFor(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) { + if v, ok := f.held[fp]; ok { + return v, true, nil + } for _, s := range f.sources { if s.Fingerprint == fp { return versions.ReservedVersion{Version: "held"}, true, nil @@ -65,7 +70,13 @@ func (f *fakeReservations) Reserve(_ context.Context, module, series string, art return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil } -func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error { +func (f *fakeReservations) Built(ctx context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error { + if err := ctx.Err(); err != nil { + return err // the state's write runs under ctx: a cancelled one records nothing + } + if f.lost != nil { + return f.lost + } f.deps = deps f.built = map[string]string{version: ""} for ref, d := range digests { @@ -153,6 +164,59 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) { } } +// TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound: design 47 H2 H-A-492 — the record of a reservation's digests after a +// build ends runs under the builder's own bound, so a build context cancelled at its end still records; a done build whose +// digests are not recorded is recorded at the builder's next look, said. +func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) { + manifest := catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + t.Run("a build's context cancelled at its end still records", func(t *testing.T) { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = manifest + res := &fakeReservations{filled: true} + r.b.Reserved = res + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + r.b.Ended = func(context.Context, string) { cancel() } // a takeover or the builder's end, once the key is written + r.b.Module(ctx, "m", moduleValue(t, "modules/m")) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { + t.Fatalf("build = %+v, want done", v) + } + if res.built["m:tool"] != digest { + t.Errorf("recorded %v, want the digest of m:tool under 0.1.5 though the build's context ended", res.built) + } + }) + t.Run("a done key whose digests are not recorded is recorded at the next look", func(t *testing.T) { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = manifest + res := &fakeReservations{filled: true, lost: errors.New("context canceled")} + r.b.Reserved = res + var journal []string + r.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) } + if v := r.consider(t, "modules/m"); v.State != builds.BuildDone || res.built != nil { + t.Fatalf("build %s recorded %v, want done and nothing recorded", v.State, res.built) + } + res.lost = nil + res.held = map[string]versions.ReservedVersion{fp1: {Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": ""}}} + r.consider(t, "modules/m") + if res.built["m:tool"] != digest || len(res.built) != 2 { + t.Errorf("recorded %v at the next look, want the done key's digest of m:tool under 0.1.5", res.built) + } + if !slices.ContainsFunc(journal, func(l string) bool { + return strings.Contains(l, "0.1.5 were not recorded") && strings.Contains(l, "recorded at a look") + }) { + t.Errorf("journal %q, want a line saying 0.1.5's digests were recorded at a look", journal) + } + // Recorded once: a reservation whose build ended recorded is left. + res.built = nil + res.held[fp1] = versions.ReservedVersion{Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": digest}, BuiltAt: time.Unix(1, 0)} + r.consider(t, "modules/m") + if res.built != nil { + t.Errorf("a recorded reservation was recorded again: %v", res.built) + } + }) +} + // TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through // reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why. func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) { diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 936b27fc6..e7acff14d 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -726,6 +726,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } // Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371). reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) + if trunk && b.originOf(key).First && planErr == nil && !nothing && !reserving { + // A done build whose digests its end did not record is recorded at the next look (design 47 H2 H-A-492). + if _, err := b.RecordAgain(ctx, module, m.Fp2, "at a look"); err != nil { + b.logf("%v", err) + } + } if reserving && b.waitFill(module, m) { end("is not built yet: wait-held, the catalog has not said versions-filled") return @@ -859,8 +865,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB for _, a := range result.Artifacts { digests[module+":"+a.Name] = a.Digest } - if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil { - b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err) + // Under the builder's own bound, never the build's context: a takeover or an end that cancels it no longer leaves a passed + // build unrecorded, so unreleased (design 47 H2 H-A-492). + rec, stop := recordContext(ctx) + err := b.Reserved.Built(rec, module, reserved.Version, digests, b.takeDependencies(key)) + stop() + if err != nil { + b.logf("%s: the digests of %s are not recorded: %v; recorded again at the builder's next look or start", cl.Key, reserved.Version, err) } } if result.State == builds.BuildFailed { From cfd2afe39c39c4b7b4456a5d80b3a4080164fa18 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 20:25:03 +0100 Subject: [PATCH 7/8] H-A-492: the record's bound is a setting; a build key read error is said Review round 1 (principle 11, configurable, never hard-coded): the bound of the record of a version's digests is the builder's setting record-bound-seconds (default 60), read from its file as revision-wait and package-wait are, set on the Builder and used for the record. A build key the builder cannot read, or that holds no build, is said in its journal instead of being taken silently as not done (principle 3, failure is loud). Builds: design 47 H2 H-A-492 --- .../cmd/mesh-delivery-builder/main.go | 6 +++ .../internal/build/numbers.go | 30 ++++++++++----- .../internal/build/reserve_test.go | 38 +++++++++++++++++++ .../internal/build/run.go | 5 ++- .../mesh-delivery-builder/manifest_test.go | 24 +++++++----- modules/mesh-delivery-builder/module.json | 17 +++++++++ 6 files changed, 100 insertions(+), 20 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 290698c49..a2b0d1000 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -233,6 +233,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } b.PackageWait = time.Duration(packageWait) * time.Minute + // A version's digests are recorded under the builder's own bound, never the build's (H-A-492); it is record-bound-seconds. + recordBound, err := numberFile(getenv("MESH_BUILDER_RECORD_BOUND_FILE"), int(build.DefaultRecordBound/time.Second)) + if err != nil { + return err + } + b.RecordBound = time.Duration(recordBound) * time.Second // Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194). b.PublishedVersions = publishedVersionsVia(calls.Call) waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf) diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 421c3c954..d7ffa9a8e 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -106,11 +106,19 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string return nil } raw, _, found, err := b.Claimer.Store.Get(ctx, key) - if err != nil || !found { + if err != nil { + b.logf("%s: the build key cannot be read, so it is taken as not done: %v", key, err) + return nil + } + if !found { return nil } cur, err := builds.DecodeBuilderBuild(raw) - if err != nil || cur.State != builds.BuildDone { + if err != nil { + b.logf("%s: the build key holds what is no build, so it is taken as not done: %v", key, err) + return nil + } + if cur.State != builds.BuildDone { return nil } out := map[string]string{} @@ -122,14 +130,18 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string return out } -// recordBound bounds the record of a reservation's digests (design 47 H2 H-A-492): the builder's own bound, never the build's -// context. -const recordBound = time.Minute +// DefaultRecordBound is the bound of the record of a reservation's digests, its setting record-bound-seconds' default (design 47 +// H2 H-A-492). +const DefaultRecordBound = time.Minute // recordContext is the context a reservation's digests are recorded under: ctx's values with no cancel of its own, bounded by -// recordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record. -func recordContext(ctx context.Context) (context.Context, context.CancelFunc) { - return context.WithTimeout(context.WithoutCancel(ctx), recordBound) +// RecordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record. +func (b *Builder) recordContext(ctx context.Context) (context.Context, context.CancelFunc) { + bound := b.RecordBound + if bound <= 0 { + bound = DefaultRecordBound + } + return context.WithTimeout(context.WithoutCancel(ctx), bound) } // RecordAgain records the digests of fp's done build under its standing reservation when its end did not record them (design 47 @@ -160,7 +172,7 @@ func (b *Builder) RecordAgain(ctx context.Context, module, fp, when string) (str } refs[ref] = d } - rec, stop := recordContext(ctx) + rec, stop := b.recordContext(ctx) defer stop() if err := b.Reserved.Built(rec, module, res.Version, refs, nil); err != nil { return "", fmt.Errorf("%s at %s: the digests of %s are not recorded %s either: %w", module, fp, res.Version, when, err) diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index 839554731..87341d179 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -186,6 +186,37 @@ func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) { t.Errorf("recorded %v, want the digest of m:tool under 0.1.5 though the build's context ended", res.built) } }) + t.Run("the record's bound is the setting record-bound-seconds, never the build's", func(t *testing.T) { + build, cancel := context.WithCancel(context.Background()) + cancel() + for _, c := range []struct { + bound, want time.Duration + }{{0, DefaultRecordBound}, {5 * time.Second, 5 * time.Second}} { + rec, stop := (&Builder{RecordBound: c.bound}).recordContext(build) + deadline, has := rec.Deadline() + stop() + if left := time.Until(deadline); !has || left > c.want || left < c.want-time.Second { + t.Errorf("RecordBound %v: the record's context has deadline %v (%v left, %v), want %v left of its own", c.bound, deadline, left, has, c.want) + } + } + rec, stop := (&Builder{}).recordContext(build) + defer stop() + if rec.Err() != nil { + t.Errorf("the record's context ended with the build's: %v", rec.Err()) + } + }) + t.Run("a build key that cannot be read is said, not taken silently", func(t *testing.T) { + var journal []string + b := &Builder{Claimer: Claimer{Store: unreadable{newMem()}}, Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }} + if d := b.doneDigests(context.Background(), "m", fp1); d != nil { + t.Errorf("an unreadable key answered digests %v", d) + } + if !slices.ContainsFunc(journal, func(l string) bool { + return strings.Contains(l, "cannot be read") && strings.Contains(l, "bus timeout") + }) { + t.Errorf("journal %q, want a line saying the build key cannot be read, with why", journal) + } + }) t.Run("a done key whose digests are not recorded is recorded at the next look", func(t *testing.T) { r := newRig(t, goBundle) r.mf.byDigest[mdigest('a')] = manifest @@ -217,6 +248,13 @@ func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) { }) } +// unreadable is a builds bucket whose reads fail, as one on a bus that does not answer. +type unreadable struct{ *memStore } + +func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) { + return nil, 0, false, errors.New("bus timeout") +} + // TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through // reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why. func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) { diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index e7acff14d..5242f95ce 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -127,6 +127,9 @@ type Builder struct { // package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished) // gives a pin (H-A-192). PackageWait time.Duration + // RecordBound bounds the record of a reservation's digests after a build ends, its setting record-bound-seconds (design 47 + // H2 H-A-492): the builder's own bound, never the build's context; DefaultRecordBound when 0. + RecordBound time.Duration // PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first // (H-A-194): read once before a build waits for that package. Nil reads none. PublishedVersions func(ctx context.Context, path string) ([]string, error) @@ -867,7 +870,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } // Under the builder's own bound, never the build's context: a takeover or an end that cancels it no longer leaves a passed // build unrecorded, so unreleased (design 47 H2 H-A-492). - rec, stop := recordContext(ctx) + rec, stop := b.recordContext(ctx) err := b.Reserved.Built(rec, module, reserved.Version, digests, b.takeDependencies(key)) stop() if err != nil { diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 0f3fbe919..b09cdf013 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -199,15 +199,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 { t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams) } - if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 || + if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 13 || // design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" || !m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || // H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) || + // H-A-492: the record of a version's digests runs under the builder's own bound, a setting of 60 s by default + m.Config["record-bound-seconds"].Default != float64(60) || m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 { - t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config) + t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), record-bound-seconds of 60 (H-A-492), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config) } // Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends. if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) { @@ -266,7 +268,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("health = %v, want kind unit alone", p["health"]) } env, _ := p["env"].(map[string]any) - if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { + if len(env) != 9 || env["MESH_BUILDER_RECORD_BOUND_FILE"] != "${dir:state}/record-bound-seconds" || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"]) } for k, v := range env { @@ -274,14 +276,16 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("env %s = %v: a setting reaches the process as a file, never its environment (A-71)", k, v) } } - found := false - for _, r := range m.Resources { - if r["type"] == "file" && r["path"] == "${dir:state}/toolchain-wait-minutes" && r["content"] == "${config:toolchain-wait-minutes}" { - found = true + for _, setting := range []string{"toolchain-wait-minutes", "record-bound-seconds"} { // H-A-492: the record's bound too + found := false + for _, r := range m.Resources { + if r["type"] == "file" && r["path"] == "${dir:state}/"+setting && r["content"] == "${config:"+setting+"}" { + found = true + } + } + if !found { + t.Errorf("no file resource with the content ${config:%s}: A-71", setting) } - } - if !found { - t.Errorf("no file resource with the content ${config:toolchain-wait-minutes}: A-71") } for _, r := range m.Resources { if r["id"] == "settings" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index f922c7b79..7c9f4e6e9 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -183,6 +183,14 @@ "trusted": false, "default": 10 }, + "record-bound-seconds": { + "type": "number", + "description": "seconds the builder gives the record of a version's digests after its build ends, under its own bound and never the build's (design 47 H2 H-A-492).", + "minimum": 1, + "maximum": 3600, + "trusted": false, + "default": 60 + }, "ask-taken-bound": { "type": "string", "description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).", @@ -310,6 +318,14 @@ "mode": "0600", "content": "${config:package-wait-minutes}" }, + { + "id": "record-bound", + "type": "file", + "path": "${dir:state}/record-bound-seconds", + "owner": "mesh-delivery-builder", + "mode": "0600", + "content": "${config:record-bound-seconds}" + }, { "id": "publish-pause", "type": "file", @@ -361,6 +377,7 @@ "MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes", "MESH_BUILDER_REVISION_WAIT_FILE": "${dir:state}/revision-wait-minutes", "MESH_BUILDER_PACKAGE_WAIT_FILE": "${dir:state}/package-wait-minutes", + "MESH_BUILDER_RECORD_BOUND_FILE": "${dir:state}/record-bound-seconds", "MESH_BUILDER_PUBLISH_PAUSE_FILE": "${dir:state}/publish-pause", "MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE": "${dir:state}/publish-pause-max", "MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries", From 52fa062e9204184cd248a4e64a386f3b2b7d938b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 20:47:58 +0100 Subject: [PATCH 8/8] H-A-494: the builder writes no git tag at a release and numbers from no tag A Go package comes from the Go package registry alone, so a released version is no longer written as a git tag: the release path and the start's take-in of a released version call no seat git, and a reserve numbers a series from the catalog's versions and the registries' only, never reading tags. A reservation's kept tag field stays for records written before. Builds: design 47 H2 H-A-494 (withdraws H-A-484's tag steps and H-A-493; design 51's fallback tag retired) --- .../cmd/mesh-delivery-builder/main.go | 4 +- .../publishes_integration_test.go | 32 ++-- .../cmd/mesh-delivery-builder/reserve.go | 81 ++-------- .../cmd/mesh-delivery-builder/reserve_test.go | 143 ++++++++---------- .../internal/build/numbers.go | 3 - .../internal/build/reserve_test.go | 6 +- .../internal/versions/reserve.go | 138 ++--------------- .../internal/versions/reserve_test.go | 54 +------ .../internal/versions/tag.go | 31 ---- .../mesh-delivery-builder/manifest_test.go | 2 +- modules/mesh-delivery-builder/module.json | 1 - 11 files changed, 107 insertions(+), 388 deletions(-) delete mode 100644 modules/mesh-delivery-builder/internal/versions/tag.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index a2b0d1000..e0b0b7d1d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -260,7 +260,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // A package's number reserved is said as the seat mesh-builds' version-reserved, so the indexer re-pins at the reservation (§805). reserved.names = versions.NamesFrom(versions.EntriesOf(branches)) - reserved.r.Names = reserved.names // a Go package's module path names its tag's directory (§796) reserved.sayReserved = sayReservedOn(js, cred) b.Reserved = reserved // A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225). @@ -305,8 +304,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1)) - reserved.pause = publishSet.Pause // a version-released whose tag the forge did not write waits the pause (H-A-484 addendum) + pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1)) releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept) if err != nil { return err diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index 9b80f346f..fe51c912e 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -289,7 +289,8 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) { var tagsWritten []string call := func(tool string, body any) (json.RawMessage, error) { switch tool { - case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796) + case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit, never read (H-A-494) + tagsWritten = append(tagsWritten, "read "+body.(gitseat.TagsInput).Prefix) return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}}) case "seat:" + gitseat.Seat + ".tag": tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit) @@ -360,15 +361,14 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) { t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects) } - // Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another - // commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags /v at - // the build's commit, and the module's version-released clears the finding. - rv.r.Names = func(context.Context, string, string) (string, bool, error) { - return "git.example/o/r/modules/m4", true, nil - } + // Design 47 H2 H-A-494: no git tag is read before a reserve nor written at a release: m4's 0.2.0, which the seat git would + // answer tagged at another commit, is reserved, no finding raised, and its version-released writes no tag. // Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it // (sayReservedOn), and waited on by the seat and the event's name. - rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"}) + rv.names = func(context.Context, string, string) (string, bool, error) { + return "git.example/o/r/modules/m4", true, nil + } + rv.sayReserved = sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"}) said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) if err != nil { t.Fatal(err) @@ -383,18 +383,18 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) { if err := said.Flush(); err != nil { t.Fatal(err) } - v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib", + v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1}) elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere) - if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised { - t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised) + if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.0" || v4.Tag != "" || raised { + t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.0 naming no tag, no tag read and no finding", v4, err, elsewhere, raised) } select { case m := <-reservedCh: ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data) - if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" || + if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.0" || ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule { - t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source")) + t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.0, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source")) } case <-time.After(10 * time.Second): t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved) @@ -420,11 +420,11 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) { } return m } - if !rv.heard(ctx, next("m4", "0.2.1")) { + if !rv.heard(ctx, next("m4", "0.2.0")) { t.Fatal("the reservations did not take version-released") } - if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) { - t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten) + if _, standing, _ := rv.r.Of(ctx, "m4", "0.2.0"); standing || len(tagsWritten) != 0 { + t.Errorf("after version-released m4 0.2.0 standing %v, tags read or written %v: want it released and no tag read or written", standing, tagsWritten) } // Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 49867fb0b..d86d25161 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -14,7 +14,6 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/process" @@ -60,11 +59,6 @@ type reservations struct { // recordAgain records the digests of a done build of module at fp its end did not record, saying so (design 47 H2 H-A-492); // nil records none. recordAgain func(ctx context.Context, module, fp, when string) (string, error) - // call asks a seat's verb: the git seat's tag (design 47 G §370). - call toolCaller - // pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands - // it back at once. - pause time.Duration // rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none. rebuild func(ctx context.Context, module, fp, why string) error // say says package-published on the seat mesh-builds; nil says none. @@ -123,7 +117,7 @@ func sayReservedOn(js jetstream.JetStream, cred process.Account) func(context.Co // newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's // verb versions through call (design 47 G §348), the only source numbering reads them from. func newReservations(store versions.ReserveStore, call toolCaller, bounds func() jetstream.KeyValue, found *findings, wake func(context.Context), logf func(string, ...any)) *reservations { - rv := &reservations{call: call, bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf} + rv := &reservations{bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf} rv.r = &versions.Reserver{Store: store, Filled: rv.filled.Load, ReleasedOf: func(_ context.Context, module string) ([]string, error) { raw, err := call("seat:"+meshcatalog.Seat+".versions", meshcatalog.VersionsInput{Module: module}) if err != nil { @@ -140,8 +134,6 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func() return out, nil }} rv.r.Took, rv.r.Logf, rv.r.Reserved = rv.took, logf, rv.reservedSaid - // A module's tags are read before it reserves, and a number tagged at another commit skipped (design 47 H2 §796). - rv.r.Tags, rv.r.TaggedElsewhere = tagsVia(call), rv.taggedElsewhere return rv } @@ -316,15 +308,9 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err) break } - if refused, err := rv.release(ctx, ev); err != nil { + if err := rv.release(ctx, ev); err != nil { rv.logf("%v; heard again", err) - // A tag the seat git's holder did not write is handed back with the publish pause, so the deliveries the consumer - // allows span the holder's outage (design 47 H2 H-A-484 addendum). - nak := m.Nak - if refused && rv.pause > 0 { - nak = func() error { return m.NakWithDelay(rv.pause) } - } - if nakErr := nak(); nakErr != nil { + if nakErr := m.Nak(); nakErr != nil { rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) } givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf) @@ -339,16 +325,12 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { return true } -// release takes a released version in as the catalog's version-released says it (design 47 G §369, §370): its tag written, its -// reservation released, its bound and publish item ended and its findings cleared. refused says the tag was not written (the -// seat git did not write it, or the reservation could not be read to tag it); the reservation then stands. -func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) (refused bool, err error) { - if err := rv.tagged(ctx, ev); err != nil { - return true, err - } +// release takes a released version in as the catalog's version-released says it (design 47 G §369): its reservation released, +// its bound and publish item ended and its findings cleared. No git tag is written (design 47 H2 H-A-494). +func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error { gone, err := rv.r.Released(ctx, ev.Module, ev.Version) if err != nil { - return false, err + return err } for _, k := range gone { rv.dropBound(ctx, k) @@ -358,13 +340,14 @@ func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionRelea } rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module) rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module) - rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) - return false, nil + rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) // one raised before H-A-494 + return nil } // releasedAtStart takes in, at the builder's start, every built, unreleased reservation whose version the catalog's verb versions // holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one -// said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended. +// said while the builder was away, is taken all the same, its reservation released and its publish item ended; no tag is written +// (design 47 H2 H-A-494). // A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released. // Before that, a standing reservation with no digests recorded whose build key is done has them recorded from the done key // (design 47 H2 H-A-492), so it is held against the released versions too. @@ -395,7 +378,7 @@ func (rv *reservations) releasedAtStart(ctx context.Context) { if !slices.Contains(released, v.Version) { continue } - if _, err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil { + if err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil { rv.logf("%s %s is released in the catalog and not taken in at the start: %v", v.Module, v.Version, err) continue } @@ -482,33 +465,6 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str return fmt.Sprintf("the reservation of %s %s is dropped (%s, %s)", module, series, by, why), nil } -// tagged writes the git tag of a released version (design 47 G §370): on the catalog's version-released, through the seat -// git's verb tag, at the commit the reservation of that version was made from. A version with no reservation (a superseded -// build's version is never released, and the key of a version released by another builder is gone) or one that names no -// tag (a module with no Go package) is tagged nowhere. A tag the forge could not write is an error, the message heard again. -func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error { - res, found, err := rv.r.Of(ctx, ev.Module, ev.Version) - if err != nil { - return fmt.Errorf("the reservation of %s %s cannot be read to tag it: %w", ev.Module, ev.Version, err) - } - if !found || res.Tag == "" || res.Commit == "" { - return nil - } - raw, err := rv.call("seat:"+git.Seat+".tag", git.TagInput{Repository: res.Repository, Commit: res.Commit, Tag: res.Tag}) - if err != nil { - return fmt.Errorf("the tag %s of %s is not written: %w", res.Tag, ev.Module, err) - } - var a git.TagAnswer - if err := json.Unmarshal(raw, &a); err != nil { - return fmt.Errorf("the git seat answered what is no tag answer for %s: %w", res.Tag, err) - } - if a.Error != "" || !a.Performed { - return fmt.Errorf("the tag %s of %s is not written: %s (%s)", res.Tag, ev.Module, a.Error, a.ErrorKind) - } - rv.logf("%s is tagged %s at %s", ev.Module, res.Tag, res.Commit) - return nil -} - // noMark says why a published marked nothing, from the reservations the published read (design 47 H H-A-280: no second read): // what each holds of digests, and how many stand at that version by the index by version (its one key, no reservation read). func (rv *reservations) noMark(ctx context.Context, o versions.Outcome, read []versions.ReadReservation) string { @@ -580,19 +536,6 @@ func (rv *reservations) heldElsewhere(ctx context.Context, module, why string) { needs: "builder_reserved shows the reservation and its withdrawn numbers; the registry seat's versions shows what it holds."}) } -// taggedElsewhere raises version-tagged-elsewhere about module (design 47 H2 §796): numbers whose git tag stands at another -// commit were skipped at reserving; a tag is never moved. Cleared at the module's next version-released. -func (rv *reservations) taggedElsewhere(ctx context.Context, module string, skipped []string) { - why := "the tags of " + strings.Join(skipped, ", ") + " stand at another commit, so the reserve skipped them" - rv.logf("%s: %s", module, why) - rv.found.raise(ctx, finding{kind: versions.VersionTaggedElsewhere, scope: meshconditionsclient.ScopeModule, id: module, - summary: module + ": " + why, - headline: "A number whose tag stands at another commit is not used", - explanation: "The repository holds a tag of " + module + " at a commit other than the build's: " + why + ". A tag is never moved.", - resolved: "The catalog says version-released for " + module + ".", - needs: "builder_reserved shows the reservation; the seat git's tags shows each tag with its commit."}) -} - // movedElsewhere takes a done build whose number a registry holds at another digest off that number (design 47 H H-A-275, // H-A-279): every artifact carries its version from its build, so no digest survives a move. The number is withdrawn as a gap // (H-A-226), the finding raised naming both digests, and the build of the same fingerprint asked again (the build-again path of a diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index ecb85c53b..fc20389bc 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -156,69 +156,65 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { } } -// TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom: design 47 G §370 — the git tag go/vMAJOR.MINOR.PATCH is written through -// the seat git's tag on the catalog's version-released, at the commit the version's reservation was made from; a module with no -// Go package, and a version never released (a superseded build's), are tagged nowhere; a tag the forge did not write leaves -// version-released to be heard again with the reservation standing. -func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { +// TestAReleasedVersionWritesNoTag: design 47 H2 H-A-494 — the builder writes no git tag at a release, and numbers a series from the +// catalog's versions and the registries' alone: the catalog's version-released releases the reservation with no call of the seat +// git, a reservation written before (one naming a tag) included; the start's take-in of a version the catalog holds released +// (H-A-484 c) the same; and a reserve reads no git tag, a tag standing at another commit no reason to skip a number. +func TestAReleasedVersionWritesNoTag(t *testing.T) { ctx := context.Background() - var tags []git.TagInput - refuse := "" - var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796) + var gitCalls []string // every call of the seat git: none is made var catalog []meshcatalog.VersionEntry // the released versions the catalog's verb versions answers call := func(addr string, body any) (json.RawMessage, error) { - switch addr { - case "seat:git.tags": - return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true}) - case "seat:git.tag": - default: - return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog}) + if strings.HasPrefix(addr, "seat:git.") { + gitCalls = append(gitCalls, addr) + if addr == "seat:git.tags" { // a tag standing at another commit, as the forge may still hold one + return json.Marshal(map[string]any{"form": 2, "tags": []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}}}) + } + return json.Marshal(git.TagAnswer{Performed: true}) } - tags = append(tags, body.(git.TagInput)) - return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse}) + return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog}) } fw := &memFindings{vals: map[string][]byte{}} found := &findings{module: builds.BuilderModule, w: fw, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }} - rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf) + store := &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}} + rv := newReservations(store, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf) rv.filled.Store(true) - // Design 47 H2 §796: the tag names the Go module's directory, read from its module path: mesh-sdk's `go` here. - rv.r.Names = func(_ context.Context, module, artifact string) (string, bool, error) { - return "git.example/o/r/go", artifact == "lib" || artifact == "pkg", nil + rv.names = func(_ context.Context, _, artifact string) (string, bool, error) { + return "git.example/o/r/go", artifact == "lib", nil } - goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true, GoArtifact: "lib"} - if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { - t.Fatal(err) - } - if _, err := rv.Reserve(ctx, "prog", "2.0", []string{"prog:bin"}, versions.Source{Repository: "o/p", Commit: "beef"}); err != nil { - t.Fatal(err) + goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", Fingerprint: "fpA"} + // oldRecord rewrites a reservation as one written before H-A-494, naming the tag its release was to be written as. + oldRecord := func(module, series, tag string) { + key := versions.ReservedKey(module, series) + var v versions.ReservedVersion + if err := json.Unmarshal(store.vals[key], &v); err != nil { + t.Fatal(err) + } + v.Tag = tag + store.vals[key], _ = json.Marshal(v) } release := func(module, version string) *wakeMsg { m := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"` + module + `","version":"` + version + `","source":1,"artifacts":[]}`)} rv.heard(ctx, m) return m } - if m := release("prog", "2.0.0"); !m.acked || len(tags) != 0 { - t.Fatalf("a module with no Go package: acked %v, tags %v; want none written", m.acked, tags) + // The release path: a Go package's version-released releases it, acknowledged, with no tag written. + if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil || v.Tag != "" { + t.Fatalf("reserved %+v (%v), want a reservation naming no tag", v, err) } - refuse = "forge down" - if m := release("pkg", "0.1.0"); !m.naked || m.acked { - t.Fatalf("a tag the forge did not write: naked %v acked %v; want the release heard again", m.naked, m.acked) + oldRecord("pkg", "0.1", "go/v0.1.0") + if m := release("pkg", "0.1.0"); !m.acked || m.naked { + t.Fatalf("the release: acked %v naked %v; want it acknowledged", m.acked, m.naked) } - // Design 47 H2 H-A-484 addendum: with a publish pause set, a refused tag hands the version-released back after the pause, so - // the deliveries its consumer allows span the holder's outage. - rv.pause = 30 * time.Second - if m := release("pkg", "0.1.0"); m.delayed != 30*time.Second || m.naked || m.acked { - t.Fatalf("a tag the forge did not write, with the pause: delayed %v naked %v acked %v; want handed back after 30s", m.delayed, m.naked, m.acked) + if len(gitCalls) != 0 { + t.Errorf("the release called the seat git: %v; want no tag written", gitCalls) + gitCalls = nil } - rv.pause = 0 - if all, _ := rv.r.All(ctx); len(all) != 1 { - t.Fatalf("the reservation went with an unwritten tag: %v", all) + if all, _ := rv.r.All(ctx); len(all) != 0 { + t.Fatalf("the released version still stands: %v", all) } - // Design 47 H2 H-A-484 addendum: at the builder's start, a built reservation the catalog holds released is tagged and released - // as its version-released would (a version-released the bus gave up on), and one the catalog does not hold, or not built, - // stands. - refuse = "" - tags = nil + // The start check (H-A-484 c): a built reservation the catalog holds released is released at the start, no tag written; one + // not built, or not released, stands. // Design 47 H2 H-A-492: at the start, a standing reservation with no digests recorded whose build key is done has them // recorded (the builder's RecordAgain), and is then held against the released versions as a built one is. if _, err := rv.Reserve(ctx, "rec", "0.1", []string{"rec:bin"}, versions.Source{Repository: "o/q", Commit: "abc", Fingerprint: "fpR"}); err != nil { @@ -238,58 +234,39 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { catalog, rv.recordAgain = nil, nil var ended []string rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) } + if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", Fingerprint: "fpB"}); err != nil { + t.Fatal(err) // 0.1.0 again: the stub catalog holds none released + } + oldRecord("pkg", "0.1", "go/v0.1.0") rv.releasedAtStart(ctx) // built nothing yet: nothing taken - if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 { - t.Fatalf("at the start with no build done: reservations %v, tags %+v; want the reservation standing, no tag", all, tags) + if all, _ := rv.r.All(ctx); len(all) != 1 { + t.Fatalf("at the start with no build done: reservations %v; want the reservation standing", all) } if _, err := rv.r.Built(ctx, "pkg", "0.1.0", "pkg:lib", "sha256:aa", nil); err != nil { t.Fatal(err) } rv.releasedAtStart(ctx) // the catalog holds none released: it stands - if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 { - t.Fatalf("at the start with nothing released: reservations %v, tags %+v; want the reservation standing", all, tags) + if all, _ := rv.r.All(ctx); len(all) != 1 { + t.Fatalf("at the start with nothing released: reservations %v; want the reservation standing", all) } catalog = []meshcatalog.VersionEntry{{Module: "pkg", Version: "0.1.0"}} rv.releasedAtStart(ctx) - if all, _ := rv.r.All(ctx); len(all) != 0 || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) || !slices.Equal(ended, []string{"pkg 0.1.0"}) { - t.Fatalf("at the start with 0.1.0 released: reservations %v, tags %+v, items ended %v; want it tagged, released and its item ended", all, tags, ended) + if all, _ := rv.r.All(ctx); len(all) != 0 || !slices.Equal(ended, []string{"pkg 0.1.0"}) { + t.Fatalf("at the start with 0.1.0 released: reservations %v, items ended %v; want it released and its item ended", all, ended) + } + if len(gitCalls) != 0 { + t.Errorf("the start check called the seat git: %v; want no tag written", gitCalls) + gitCalls = nil } catalog, rv.released = nil, nil - if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // the version is reserved again for the rows below - t.Fatal(err) - } - tags = nil - if m := release("pkg", "0.1.0"); !m.acked || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) { - t.Fatalf("the release: acked %v, tags %+v; want go/v0.1.0 at c0ffee of o/r", m.acked, tags) - } - tags = nil - if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // build A: 0.1.0 (the stub catalog holds none released) - t.Fatal(err) - } - if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true, GoArtifact: "lib"}); err != nil || v.Version != "0.1.1" { // build B supersedes A - t.Fatalf("second build reserved %q (%v)", v.Version, err) - } - release("pkg", "0.1.0") // A's version is never released in fact; were it said, it is no reservation of the key - if len(tags) != 0 { - t.Fatalf("a superseded build's version was tagged: %+v", tags) - } - if release("pkg", "0.1.1"); len(tags) != 1 || tags[0].Commit != "b0b" { - t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags) - } - // Design 47 H2 §796: a tag standing at another commit makes the next number and the finding version-tagged-elsewhere, which - // clears at the module's next version-released. - standing = []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}, {Tag: "go/v0.2.1", Commit: "elsewhere"}} - if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.2" || v.Tag != "go/v0.2.2" { - t.Fatalf("reserved %+v (%v) with 0.2.0 and 0.2.1 tagged elsewhere, want 0.2.2 tagged go/v0.2.2", v, err) + // Numbering reads no tag: the seat git would answer go/v0.2.0 at another commit, and 0.2.0 is reserved all the same. + if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.0" || v.Tag != "" || len(gitCalls) != 0 { + t.Errorf("reserved %+v (%v), git calls %v; want 0.2.0 naming no tag, no git tag read", v, err, gitCalls) } key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "pkg", versions.VersionTaggedElsewhere) - if got, err := decodeFinding(fw.vals[key]); err != nil || !strings.Contains(got.Summary, "0.2.0") || !strings.Contains(got.Summary, "0.2.1") { - t.Fatalf("the finding %s = %+v (%v), want it naming 0.2.0 and 0.2.1", key, got, err) + if fw.vals[key] != nil { + t.Errorf("a reserve raised %s", key) } - if release("pkg", "0.2.2"); fw.vals[key] != nil { - t.Errorf("version-released of pkg left %s standing", key) - } - standing = nil // Design 47 H2 §805, H-A-420 (2): a package's number newly reserved is said as version-reserved, its path the package's name // and its revision the indexer's `.`; a reservation kept as it stood, an artifact that is no package and a // reservation naming no catalog revision say none. @@ -298,7 +275,7 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { return "git.example/o/r/" + artifact, module == "lib2" && artifact == "pkg", nil } rv.sayReserved = func(_ context.Context, ev meshbuilds.VersionReservedEvent) error { said = append(said, ev); return nil } - at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", GoPackage: true, GoArtifact: "pkg", Branch: "main", CatalogRevision: 158} + at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", Branch: "main", CatalogRevision: 158} for range 2 { if _, err := rv.Reserve(ctx, "lib2", "0.3", []string{"lib2:pkg", "lib2:bin"}, at); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index d7ffa9a8e..8e0cd70e6 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -261,9 +261,6 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu if t.Kind == catalogue.ArtifactUpstream { src.Upstream = append(src.Upstream, module+":"+t.Artifact) // copied by its pin, never awaited (H-A-486) } - if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage { - src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796) - } } res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs, src) if err != nil { diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index 87341d179..bb63848d4 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -345,7 +345,7 @@ func TestAReservedBuildKeepsWhatItsStepResolvedAndWasBuiltWith(t *testing.T) { // TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion: design 47 G §370 — a module whose one artifact is // a Go package (mesh-wireguard's go-overlay) is planned on the Go build seat, its ask carries the reserved version and names -// the seat go-package-registry the package is published and tagged through, and the build is done; a build reserving none +// the seat go-package-registry the package is published through, and the build is done; a build reserving none // names no seat. func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *testing.T) { goPackage := catalogue.Artifact{Name: "go-overlay", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}} @@ -374,10 +374,6 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t len(asked[0].Build.Artifacts) != 1 || asked[0].Build.Artifacts[0] != (buildstep.AskArtifact{Artifact: "go-overlay", Seat: c.seat}) { t.Fatalf("asks %+v, want one Go ask with version %q naming go-overlay on seat %q", asked, c.version, c.seat) } - // Design 47 H2 §796: the reservation names the Go package's artifact, whose module path names its tag's directory. - if c.version != "" && (len(res.sources) != 1 || !res.sources[0].GoPackage || res.sources[0].GoArtifact != "go-overlay") { - t.Errorf("reserved from %+v, want one source naming the Go package go-overlay", res.sources) - } }) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 724c5f282..c80cbbe21 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -10,10 +10,8 @@ import ( "sort" "strconv" "strings" - "sync" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) @@ -42,8 +40,8 @@ const ( // VersionHeldElsewhere is the finding of a number a registry seat already holds, skipped at reserving or moved from (design 47 // H H-A-275), scope module. VersionHeldElsewhere = "version-held-elsewhere" - // VersionTaggedElsewhere is the finding of a number whose git tag stands at another commit, skipped at reserving (design 47 - // H2 §796), scope module; cleared at the module's next version-released. + // VersionTaggedElsewhere is the finding of a number whose git tag stood at another commit, which a reserve skipped before no + // tag was read (design 47 H2 H-A-494); one raised before is cleared at the module's next version-released. VersionTaggedElsewhere = "version-tagged-elsewhere" ) @@ -67,8 +65,8 @@ type ReservedVersion struct { // was kept. PublishedAt map[string]time.Time `json:"published-at,omitempty"` At time.Time `json:"at"` - // Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go - // package's, GoTagOf): empty for a module with no Go package (design 47 G §370), or whose Go package's directory is not known. + // Repository and Commit are what the build was made from. Tag is kept for a reservation written before design 47 H2 H-A-494: + // no git tag is written at a release any more, and a new reservation names none. Repository string `json:"repository,omitempty"` Commit string `json:"commit,omitempty"` Tag string `json:"tag,omitempty"` @@ -114,16 +112,11 @@ type KeptPublished struct { Digest string `json:"digest"` } -// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is -// written at, and whether the module has a Go package, whose version is tagged. +// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit, kept on the reservation. type Source struct { Repository, Commit string // Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76). Fingerprint string - GoPackage bool - // GoArtifact is the artifact that is the module's Go package, whose module path names the directory its tag is written in - // (design 47 H2 §796). - GoArtifact string // Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of // that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication. Branch string @@ -177,92 +170,6 @@ type Reserver struct { HeldNumbers func(ctx context.Context, module, fingerprint string, refs []string) ([]string, error) // HeldElsewhere is told the numbers of module a reserve skipped because a registry holds them (H-A-275); nil is told none. HeldElsewhere func(ctx context.Context, module string, skipped []string) - // Names answers a Go package's module path, which names the directory its tag is written in (design 47 H2 §796); nil, or a - // name not known, tags the version nowhere, said. - Names PackageNames - // Tags reads repository's tags starting with prefix, each with the commit it names, through the seat git's tags (design 47 - // H2 §796); away says the read met a transport fault or the error kind away, which lets the reservation go on, said. Nil - // reads none. - Tags func(ctx context.Context, repository, prefix string) (tags []git.Tag, away bool, err error) - // TaggedElsewhere is told the numbers of module a reserve skipped because their tag stands at another commit (§796); nil - // is told none. - TaggedElsewhere func(ctx context.Context, module string, skipped []string) - - tagsMu sync.Mutex - // tagsRead are the tags last read of each module, by series patch to the commit, with the catalog revision they were read - // under: a module's tags are read once per catalog revision it reserves under (§796). - tagsRead map[string]tagsAt -} - -// tagsAt are a module's tags of one series read under one catalog revision: patch to the commit its tag names. -type tagsAt struct { - revision, prefix string - commits map[int]string -} - -// goDirectory is the directory of module's Go package in its repository (§796), from its module path; ok is false, said, when -// no name or no directory of the repository is known. -func (r *Reserver) goDirectory(ctx context.Context, module string, src Source) (string, bool) { - if r.Names == nil { - return "", false - } - path, known, err := r.Names(ctx, module, src.GoArtifact) - switch { - case err != nil: - r.said(fmt.Errorf("%s: its Go package's name cannot be read, so its version is tagged nowhere: %w", module, err)) - return "", false - case !known: - r.said(fmt.Errorf("%s: no name of its Go package %s is known yet, so its version is tagged nowhere", module, src.GoArtifact)) - return "", false - } - dir, ok := GoDirectory(path, src.Repository) - if !ok { - r.said(fmt.Errorf("%s: its Go package %s is not a module path of the repository %s, so its version is tagged nowhere", module, path, src.Repository)) - } - return dir, ok -} - -// taggedAt are the patches of series whose tag `/v.` stands, each with its commit, read once per -// catalog revision the module reserves under (§796). A read that meets a transport fault or away answers none, said, and the -// reservation goes on; any other failure refuses the reservation, naming it. -func (r *Reserver) taggedAt(ctx context.Context, module, series, dir string, src Source) (map[int]string, error) { - if r.Tags == nil { - return nil, nil - } - prefix := TagPrefix(dir) + series + "." - revision := "" - if src.CatalogRevision > 0 { - revision = fmt.Sprintf("%s.%d", src.Branch, src.CatalogRevision) - r.tagsMu.Lock() - got, ok := r.tagsRead[module] - r.tagsMu.Unlock() - if ok && got.revision == revision && got.prefix == prefix { - return got.commits, nil - } - } - tags, away, err := r.Tags(ctx, src.Repository, prefix) - if err != nil && away { - r.said(fmt.Errorf("%s: its tags %s* cannot be read (%w), so the reservation goes on without them", module, prefix, err)) - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("the tags %s* of %s cannot be read: %w", prefix, module, err) - } - commits := map[int]string{} - for _, t := range tags { - if s, p, err := SplitVersion(strings.TrimPrefix(t.Tag, TagPrefix(dir))); err == nil && s == series && strings.HasPrefix(t.Tag, prefix) { - commits[p] = t.Commit - } - } - if revision != "" { - r.tagsMu.Lock() - if r.tagsRead == nil { - r.tagsRead = map[string]tagsAt{} - } - r.tagsRead[module] = tagsAt{revision: revision, prefix: prefix, commits: commits} - r.tagsMu.Unlock() - } - return commits, nil } // heldNumbers are the patches of series a registry holds of module's artifacts (H-A-275); a read that fails is said, and none. @@ -283,19 +190,15 @@ func (r *Reserver) heldNumbers(ctx context.Context, module, series, fingerprint return out } -// nextFree is the first patch after after that no registry holds and whose tag stands at no other commit than commit, with the -// held numbers it skipped (H-A-275) and those tagged elsewhere (design 47 H2 §796). -func nextFree(series string, after int, held map[int]bool, tagged map[int]string, commit string) (int, []string, []string) { - var skipped, elsewhere []string +// nextFree is the first patch after after that no registry holds, with the held numbers it skipped (H-A-275). No git tag is +// read: a series is numbered from the catalog's versions and the registries' alone (design 47 H2 H-A-494). +func nextFree(series string, after int, held map[int]bool) (int, []string) { + var skipped []string for p := after + 1; ; p++ { - switch c, ok := tagged[p]; { - case held[p]: - skipped = append(skipped, fmt.Sprintf("%s.%d", series, p)) - case ok && c != commit: - elsewhere = append(elsewhere, fmt.Sprintf("%s.%d (tagged at %s)", series, p, c)) - default: - return p, skipped, elsewhere + if !held[p] { + return p, skipped } + skipped = append(skipped, fmt.Sprintf("%s.%d", series, p)) } } @@ -348,15 +251,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts } key := ReservedKey(module, series) held := r.heldNumbers(ctx, module, series, src.Fingerprint, artifacts) - dir, tagged := "", false - var tags map[int]string - if src.GoPackage { - if dir, tagged = r.goDirectory(ctx, module, src); tagged { - if tags, err = r.taggedAt(ctx, module, series, dir, src); err != nil { - return ReservedVersion{}, err - } - } - } for { cur, rev, found, err := r.get(ctx, key) if err != nil { @@ -374,12 +268,9 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts next = p } } - patch, skipped, elsewhere := nextFree(series, next, held, tags, src.Commit) + patch, skipped := nextFree(series, next, held) out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, patch), Artifacts: map[string]string{}, At: r.now(), Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision} - if tagged { - out.Tag = GoTagOf(dir, out.Version) - } for _, a := range artifacts { out.Artifacts[a] = "" } @@ -417,9 +308,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts if len(skipped) > 0 && r.HeldElsewhere != nil { r.HeldElsewhere(ctx, module, skipped) } - if len(elsewhere) > 0 && r.TaggedElsewhere != nil { - r.TaggedElsewhere(ctx, module, elsewhere) - } if found && r.Took != nil { r.Took(ctx, cur, out) } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index a0d96f966..e19e53251 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -5,11 +5,9 @@ import ( "encoding/json" "errors" "reflect" - "strings" "testing" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" ) @@ -416,7 +414,7 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) { return held, readErr }, HeldElsewhere: func(_ context.Context, _ string, s []string) { skipped = append(skipped, s...) }} - v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true}) + v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1"}) if err != nil || v.Version != "0.1.39" || len(skipped) != 2 { t.Fatalf("reserved %q (%v), skipped %v: want 0.1.39 past the two numbers the registry holds", v.Version, err, skipped) } @@ -430,9 +428,9 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) { } held = []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40", "0.2.0"} // The build again of a done key reserves as a rebuild (A-490b): a withdrawn number is never its floor's to reuse. - again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true, Rebuild: true}) + again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1", Rebuild: true}) if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != "" { - t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (no Go package name known)", again, err) + t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (design 47 H2 H-A-494)", again, err) } if at, _ := r.AtVersion(ctx, "0.1.39"); len(at) != 0 { t.Errorf("the index by version still names the withdrawn number: %v", at) @@ -445,50 +443,4 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) { if v, err := r.Reserve(ctx, "node-tools", "0.2", []string{"node-tools:program"}, Source{Fingerprint: "fp1"}); err != nil || v.Version != "0.2.1" { t.Errorf("with the registry down reserved %q (%v), want 0.2.1 (what it answered still skipped)", v.Version, err) } - // Design 47 H2 §796: a Go package's tags are read before it reserves, once per catalog revision; a number whose tag - // `/v` stands at another commit is skipped and said, one at the build's own commit is not; the version's - // tag names the Go module's directory; a read met away lets the reservation go on, any other failure refuses it. - r.HeldNumbers = nil - var tagReads int - var tagsErr error - var tagsAway bool - var elsewhere []string - tags := []git.Tag{{Tag: "modules/nt/v0.3.0", Commit: "c-other"}, {Tag: "modules/nt/v0.3.1", Commit: "c-mine"}, {Tag: "modules/nt/v0.3.2", Commit: "c-other2"}} - r.Names = func(_ context.Context, module, artifact string) (string, bool, error) { - return "git.example/o/r/modules/nt", module == "node-tools" && artifact == "lib", nil - } - r.Tags = func(_ context.Context, repository, prefix string) ([]git.Tag, bool, error) { - tagReads++ - if repository != "o/r" || prefix != "modules/nt/v0.3." { - t.Errorf("tags read of %s %q, want o/r modules/nt/v0.3.", repository, prefix) - } - return tags, tagsAway, tagsErr - } - r.TaggedElsewhere = func(_ context.Context, _ string, s []string) { elsewhere = append(elsewhere, s...) } - at := Source{Fingerprint: "fp3", GoPackage: true, GoArtifact: "lib", Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 7} - v3, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at) - if err != nil || v3.Version != "0.3.1" || v3.Tag != "modules/nt/v0.3.1" || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.0") { - t.Fatalf("reserved %+v (%v), tagged elsewhere %v: want 0.3.1 (0.3.0 tagged at another commit, 0.3.1 at the build's own), tag modules/nt/v0.3.1", v3, err, elsewhere) - } - at.Fingerprint, at.Commit, elsewhere = "fp4", "c-new", nil - if v4, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v4.Version != "0.3.3" || tagReads != 1 || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.2") { - t.Fatalf("reserved %+v (%v) after %d reads, tagged elsewhere %v: want 0.3.3 past 0.3.2's tag, the tags read once for revision main.7", v4, err, tagReads, elsewhere) - } - at.CatalogRevision, at.Fingerprint, tagsErr, tagsAway = 8, "fp5", errors.New("the forge is away"), true - if v5, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v5.Version != "0.3.4" || tagReads != 2 { - t.Fatalf("with the tags away reserved %+v (%v) after %d reads, want 0.3.4 and the reservation going on", v5, err, tagReads) - } - at.CatalogRevision, at.Fingerprint, tagsAway = 9, "fp6", false - if _, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err == nil || !strings.Contains(err.Error(), "tags") { - t.Errorf("a tags read refused not away reserved anyway (%v), want the reservation refused naming the tags", err) - } - if d, ok := GoDirectory("git.example/novox/mesh-sdk/go", "https://git.example/novox/mesh-sdk.git"); !ok || d != "go" || GoTagOf(d, "0.1.9") != "go/v0.1.9" { - t.Errorf("mesh-sdk's Go package: directory %q (%v), want go and the tag go/v0.1.9", d, ok) - } - if d, ok := GoDirectory("git.example/o/r", "o/r"); !ok || d != "" || GoTagOf(d, "1.0.0") != "v1.0.0" { - t.Errorf("a root Go module: directory %q (%v), want the root and the tag v1.0.0", d, ok) - } - if _, ok := GoDirectory("git.example/o/rr/x", "o/r"); ok { - t.Error("a module path of another repository was given a directory") - } } diff --git a/modules/mesh-delivery-builder/internal/versions/tag.go b/modules/mesh-delivery-builder/internal/versions/tag.go deleted file mode 100644 index dcb070c26..000000000 --- a/modules/mesh-delivery-builder/internal/versions/tag.go +++ /dev/null @@ -1,31 +0,0 @@ -package versions - -import ( - "strings" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" -) - -// GoTagOf is the git tag a Go package's released version is written as (design 47 H2 §796): `/v`, the -// directory of its Go module in the repository, which the Go toolchain's direct fallback reads; `v` at the root. The -// `go/v` tags go from new versions. -func GoTagOf(directory, version string) string { return TagPrefix(directory) + version } - -// GoDirectory is the directory of the Go module path modulePath in repository (`/`, or a clone address naming -// it): the part of the path after the repository's own, as the Go toolchain reads a module kept in a subdirectory; empty at -// the root. ok is false when the path is not one of that repository. -func GoDirectory(modulePath, repository string) (string, bool) { - repo := "/" + strings.ToLower(trunk.RepositoryName(repository)) - lower := strings.ToLower(modulePath) - for from := 0; ; { - i := strings.Index(lower[from:], repo) - if i < 0 || repo == "/" { - return "", false - } - end := from + i + len(repo) - if end == len(lower) || lower[end] == '/' { - return strings.Trim(modulePath[end:], "/"), true - } - from = end - } -} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index b09cdf013..2fcaa01e7 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -152,7 +152,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // Design 51 *The floor*: the floor's three sources, the registry seats' versions, git.tags and indexer_requirements. // The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3). if want := []string{"seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions", - "seat:git.tag", "seat:git.tags", + "seat:git.tags", // the floor's source alone: no tag is written at a release (design 47 H2 H-A-494) "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions", // A-231: the build asks on each build seat's queue delivery (accept.delivery.). diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 7c9f4e6e9..b6514011d 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -91,7 +91,6 @@ "invokes": [ "seat:container-image-registry.remove", "seat:container-image-registry.versions", - "seat:git.tag", "seat:git.tags", "seat:go-package-registry.remove", "seat:go-package-registry.versions",