From f26ff8f8bbcd923d0178b977052e95b4591665bc Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:57:01 +0200 Subject: [PATCH 01/25] =?UTF-8?q?Write=20a=20key=20done=20only=20when=20ev?= =?UTF-8?q?ery=20bundle=20and=20archive=20names=20where=20it=20is=20stored?= =?UTF-8?q?,=20a=20candidate's=20too,=20so=20a=20merge=20builds=20nothing?= =?UTF-8?q?=20its=20check=20built=20(issue=201587,=20B1,=20ADR=200498=20?= =?UTF-8?q?=C2=A71-=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins mesh-sdk feat/r4e-sdk a90f523 (R4e's group member). A trunk key's builds are asked to publish; a build answering no reference fails the key 'not stored:', and a step's own 'not stored:', 'not pinned:', 'version taken:' or 'refused kind:' opens the key's error. --- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 2 + .../internal/build/r4e_test.go | 76 +++++++++++++++++++ .../internal/build/run.go | 55 ++++++++++++-- .../internal/build/run_test.go | 11 ++- 5 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-builder/internal/build/r4e_test.go diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 59b27dd..a96fd50 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022154726-a90f523edc75 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 3515fb8..d5ce48e 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022154726-a90f523edc75 h1:e/aOJLHaNSBPFFHHjUF+0fAfjmtMUy/vDWqetauWp/U= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022154726-a90f523edc75/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go new file mode 100644 index 0000000..6d77b9a --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -0,0 +1,76 @@ +package build + +import ( + "context" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// R4e: references in the key (novox/hq ADR 0498), upstream copies, packages published from the trunk, the package wait +// (ADR 0500 clause 6) and one node per fingerprint (ADR 0478 clause 4). + +func TestACandidatesBundleKeyNamesItsReferenceAndTheTrunkBuildsItNoMore(t *testing.T) { // B1, ADR 0498 clause 3 and its check 3 + r := newRig(t, goBundle, archiveA) + trunkModule(t, r) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Wait() + v := valueOf(t, r.st, r.key) + if v.State != delivery.BuildDone { + t.Fatalf("build = %+v, want done", v) + } + if ok, missing := v.Referenced(); !ok { + t.Fatalf("a candidate's bundle and archive name no reference: %v (%+v)", missing, v.Artifacts) + } + rev := r.st.revs[r.key] + r.bm.mu.Lock() + r.bm.calls = nil + r.bm.mu.Unlock() + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk now names the same fingerprint + r.b.Wait() + if n := len(r.bm.calls); n != 0 || r.st.revs[r.key] != rev { + t.Errorf("the trunk built its candidate's fingerprint again: %d calls, revision %d (was %d)", n, r.st.revs[r.key], rev) + } +} + +func TestABundleWhosePushAnsweredNoReferenceFailsNotStoredAndIsBuiltAgain(t *testing.T) { // B1, ADR 0498 clauses 1 and 2 + r := newRig(t, goBundle) + r.bm.noRef["ace/tool"] = true + v := r.consider(t, "modules/m") + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "tool") { + t.Fatalf("build = %+v, want failed opening %q and naming the artifact", v, delivery.NotStored) + } + delete(r.bm.noRef, "ace/tool") + if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone { + t.Errorf("build = %+v, want a not-stored key built again as a new revision", v) + } +} + +func TestAPushRefusedByTheStoreFailsTheKeyOpeningNotStored(t *testing.T) { // B1, ADR 0498 clause 2 + r := newRig(t, goBundle) + r.bm.fails["ace/build_go_build"] = delivery.NotStored + "tool: the artifact store answered 507 insufficient storage" + v := r.consider(t, "modules/m") + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "507") || !strings.Contains(v.Error, "ace") { + t.Fatalf("build = %+v, want failed opening %q with the store's words and the node", v, delivery.NotStored) + } +} + +func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a): publish only for a trunk key + r := newRig(t, goBundle) + r.consider(t, "modules/m") + for _, c := range r.bm.callsOf("build_go_build") { + if c.body.(map[string]any)["publish"] != true { + t.Errorf("trunk build args = %+v, want publish", c.body) + } + } + c := newRig(t, goBundle) + trunkModule(t, c) + c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + c.b.Wait() + for _, call := range c.bm.callsOf("build_go_build") { + if _, ok := call.body.(map[string]any)["publish"]; ok { + t.Errorf("candidate build args = %+v, want no publish", call.body) + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ea67174..1607a71 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -483,7 +483,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } } if len(errs) > 0 { - return failed("%s", strings.Join(errs, "; ")) + return failed("%s", keyError(errs)) } var artifacts []delivery.BuiltArtifact keptModule := map[string]bool{} @@ -515,7 +515,16 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } built := delivery.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d} for _, l := range lanes { - if r := l.references[t.Artifact]; r != "" && l.module == t.Module { + if l.module != t.Module { + continue + } + r := l.references[t.Artifact] + if r == "" && stored(t.Kind) { + // A bundle, an archive or an upstream copy is done only when its push was answered with where it is (ADR 0498 + // clause 1); a candidate's included (clause 3). + return failed("%s%s on %s: the build answered no reference, so its push to the artifact store was not answered", delivery.NotStored, t.Artifact, l.node) + } + if r != "" { built.Reference = r } } @@ -537,7 +546,40 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p if len(used) == 0 { used = nil } - return delivery.BuilderBuild{State: delivery.BuildDone, Step: step, Artifacts: artifacts, Toolchains: used} + done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used} + if _, err := delivery.EncodeBuilderBuild(done); err != nil { + // A reference the build module answered that does not name the artifact and its digest is no reference. + return failed("%s%v", delivery.NotStored, err) + } + return done +} + +// stored is whether an artifact of this kind is in the artifact store once built, a candidate's too, so its done key +// names its reference (ADR 0498 clauses 1, 3 and 4); an image or a package names one only when built for the trunk. +func stored(kind string) bool { + return kind == catalogue.ArtifactBundle || kind == catalogue.ArtifactArchive || kind == catalogue.ArtifactUpstream +} + +// keyPrefixes are the errors a build module's step may open its reason with that the key's error must open with too, so +// the claim reads them (delivery.FailedUntilChanged), first the ones not built again. +var keyPrefixes = []string{delivery.NotPinned, delivery.VersionTaken, delivery.RefusedKind, delivery.NotStored} + +// keyError is the error of a key from its lanes' errors: when one names a prefix of keyPrefixes inside the step's words +// (` on : the step failed: not stored: ...`), the key's error opens with that prefix, so a reader of the +// key need not search for it. +func keyError(errs []string) string { + for _, prefix := range keyPrefixes { + for i, e := range errs { + if strings.HasPrefix(e, prefix) { + return strings.Join(errs, "; ") + } + if at := strings.Index(e, prefix); at >= 0 { + rest := append([]string{prefix + e[:at] + e[at+len(prefix):]}, append(append([]string(nil), errs[:i]...), errs[i+1:]...)...) + return strings.Join(rest, "; ") + } + } + } + return strings.Join(errs, "; ") } func (b *Builder) online(node string) bool { return b.Presence == nil || b.Presence.Online(node) } @@ -657,8 +699,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del } for _, t := range l.tasks { args := map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact} - if t.Kind == catalogue.ArtifactImage && !candidate { - // Only a trunk key pushes the image to the registry; a candidate's build never does (ADR 0487 clause 12). + if !candidate { + // Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12, + // ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3). args["publish"] = true } raw, err := b.step(ctx, l.module, l.node, "build", args) @@ -672,7 +715,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del return } l.digests[t.Artifact] = built.Digest - if built.Reference != "" && !candidate { + if built.Reference != "" && (stored(t.Kind) || !candidate) { l.references[t.Artifact] = built.Reference } } diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 6da1ebe..940e4fe 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -23,6 +23,8 @@ type buildModules struct { steps *fakeSteps digests map[string]string // `/` -> digest; the default is `digest` fails map[string]string // `/` -> the reason the step fails with + // noRef names `/` whose bundle or archive build answers no reference (its push not answered). + noRef map[string]bool // hold, when set before the builds start, blocks every call until it is closed; entered is sent to as a call arrives. // conflicts, when set, is a file a fetch with a base reports in conflict; noMerge makes it answer neither a merge nor a conflict. conflicts string @@ -32,7 +34,7 @@ type buildModules struct { } func newBuildModules() *buildModules { - return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}} + return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}, noRef: map[string]bool{}} } func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, error) { @@ -69,6 +71,9 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro } if tool == "build_image_build" && args["publish"] == true { step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/img@` + d + `"}`) + } else if tool != "build_image_build" && !b.noRef[node+"/"+fmt.Sprint(args["artifact"])] { + // A bundle or an archive is pushed to the artifact store and its reference answered (ADR 0498 clause 1). + step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://m/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`) } else { step.Result = json.RawMessage(`{"digest":"` + d + `"}`) } @@ -173,8 +178,8 @@ func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *test t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v) } for i, want := range []string{"files", "tool", "web"} { - if a := v.Artifacts[i]; a.Name != want || a.Digest != digest || a.Reference != "" { - t.Errorf("artifact %d = %+v, want %s with the digest and no reference (ADR 0478 clause 6)", i, a, want) + if a := v.Artifacts[i]; a.Name != want || a.Digest != digest || a.Reference != "artifact-store://m/"+want+"/blobs/"+digest { + t.Errorf("artifact %d = %+v, want %s with the digest and its artifact-store reference (ADR 0498 clause 1)", i, a, want) } } if v.Artifacts[0].Kind != "archive" || v.Artifacts[1].Kind != "bundle" { From 9871ed2d09f766a0d7fc96f9a05fdb995275fc17 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:57:35 +0200 Subject: [PATCH 02/25] =?UTF-8?q?Build=20a=20key=20done=20before=20R4e=20a?= =?UTF-8?q?gain=20once,=20so=20every=20bundle=20and=20archive=20the=20cont?= =?UTF-8?q?roller=20delivers=20names=20its=20stored=20reference=20(issue?= =?UTF-8?q?=201587,=20B2,=20ADR=200498=20=C2=A78)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/build/claim.go | 22 ++++++++++++- .../internal/build/claim_test.go | 4 +-- .../internal/build/r4e_test.go | 31 +++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index cb5f74c..d598bc7 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -85,9 +85,26 @@ func Unpublished(b delivery.BuilderBuild) bool { return false } +// Unreferenced is whether a done build holds a bundle, an archive or an upstream copy that names no reference: a key done +// before R4e, whose artifacts stayed on their build nodes (ADR 0498 clause 8). An artifact of no kind (a key written before +// kinds) is one too. Claim takes such a key again, once: its new revision names them. +func Unreferenced(b delivery.BuilderBuild) bool { + if b.State != delivery.BuildDone { + return false + } + for _, a := range b.Artifacts { + if a.Reference == "" && (a.Kind == "" || stored(a.Kind)) { + return true + } + } + return false +} + // Claim takes the build of module at fingerprint, or answers nil when it is not this builder's to build now: // - no key: it is created building, by this builder; a create that finds the key made meanwhile gets nothing; -// - a done key is not taken again (the same source is never built twice), except one built on one node only (OneNodePrefix) +// - a done key is not taken again (the same source is never built twice), except one whose bundle, archive or upstream +// copy names no reference (Unreferenced, a key done before R4e: ADR 0498 clause 8), which is claimed building as a new +// revision and built whole, and one built on one node only (OneNodePrefix) // when a second node qualifies now (ClaimOptions.MoreNodes): that is taken as it stands, with the done value, and its // builder writes the new revision when the second build ends (ADR 0478 clause 4); // - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no @@ -115,6 +132,9 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string } switch { case cur.State == delivery.BuildDone: + if Unreferenced(cur) { + return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) + } if modules := OneNodeModules(cur); len(modules) > 0 && opt.MoreNodes != nil && opt.MoreNodes(modules) { return &Claim{Key: key, Rev: rev, Build: cur}, nil } diff --git a/modules/mesh-delivery-builder/internal/build/claim_test.go b/modules/mesh-delivery-builder/internal/build/claim_test.go index cab71e3..fdf6d93 100644 --- a/modules/mesh-delivery-builder/internal/build/claim_test.go +++ b/modules/mesh-delivery-builder/internal/build/claim_test.go @@ -125,9 +125,9 @@ func TestClaimLosingTheCreateRaceGetsNothing(t *testing.T) { // 1607/AC3 } } -func TestADoneKeyIsNeverTakenAgain(t *testing.T) { // 1607/AC3 +func TestADoneKeyIsNeverTakenAgain(t *testing.T) { // 1607/AC3; a key whose artifacts name references (ADR 0498 clause 8) st := newMem() - seed(t, st, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Digest: digest}}}) + seed(t, st, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/x/blobs/" + digest}}}) writes := st.writes got, err := Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{}) if err != nil || got != nil || st.writes != writes { diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index 6d77b9a..ad5b584 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -74,3 +74,34 @@ func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's } } } + +// putBuild writes a builder key's value as the store would hold it. +func putBuild(t *testing.T, st *memStore, key string, b delivery.BuilderBuild) uint64 { + t.Helper() + data, err := delivery.EncodeBuilderBuild(b) + if err != nil { + t.Fatal(err) + } + st.mu.Lock() + defer st.mu.Unlock() + return st.put(key, data) +} + +func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) { // B2, ADR 0498 clause 8 + r := newRig(t, goBundle, archiveA) + rev := putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: manifestD, Step: "built mesh-build-go on ace and novox", + Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest}, {Name: "tool", Kind: "bundle", Digest: digest}}}) + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev { + t.Fatalf("build = %+v at revision %d (was %d), want built again as a new revision", v, r.st.revs[r.key], rev) + } + if ok, missing := v.Referenced(); !ok { + t.Fatalf("the new revision names no reference for %v", missing) + } + rev = r.st.revs[r.key] + r.b.tried = map[string]bool{} // a builder started anew looks again + r.consider(t, "modules/m") + if r.st.revs[r.key] != rev { + t.Errorf("a key whose artifacts all name references was built again (revision %d, was %d): once only", r.st.revs[r.key], rev) + } +} From c26f90915258bf98a72ec3e69d21ec470956a575 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:58:33 +0200 Subject: [PATCH 03/25] =?UTF-8?q?Send=20a=20pinned=20upstream=20image=20to?= =?UTF-8?q?=20mesh-build-image's=20copy=20and=20fail=20an=20unpinned=20one?= =?UTF-8?q?=20'not=20pinned:'=20until=20its=20fingerprint=20changes;=20ref?= =?UTF-8?q?usals=20use=20mesh-sdk's=20RefusedKind=20(issue=201587,=20B3,?= =?UTF-8?q?=20ADR=200498=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/build/claim.go | 5 +- .../internal/build/claim_test.go | 10 +++- .../internal/build/plan.go | 42 +++++++++++------ .../internal/build/plan_test.go | 46 +++++++++++-------- .../internal/build/run_test.go | 12 ++--- 5 files changed, 74 insertions(+), 41 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index d598bc7..c99779f 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -108,7 +108,8 @@ func Unreferenced(b delivery.BuilderBuild) bool { // when a second node qualifies now (ClaimOptions.MoreNodes): that is taken as it stands, with the done value, and its // builder writes the new revision when the second build ends (ADR 0478 clause 4); // - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no -// builder builds yet (RefusedPrefix, ADR 0478 clause 2) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4), +// builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478 +// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4), // which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for // another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder); // - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its @@ -142,7 +143,7 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string return &Claim{Key: key, Rev: rev, Build: cur}, nil } return nil, nil - case cur.State == delivery.BuildFailed && (strings.HasPrefix(cur.Error, RefusedPrefix) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): + case cur.State == delivery.BuildFailed && (delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): return nil, nil case cur.State == delivery.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()): return nil, nil diff --git a/modules/mesh-delivery-builder/internal/build/claim_test.go b/modules/mesh-delivery-builder/internal/build/claim_test.go index fdf6d93..ed359a3 100644 --- a/modules/mesh-delivery-builder/internal/build/claim_test.go +++ b/modules/mesh-delivery-builder/internal/build/claim_test.go @@ -148,11 +148,19 @@ func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) { } st = newMem() - key = seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: RefusedPrefix + " image (artifact x): build-agent builds it"}) + key = seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: delivery.RefusedKind + "package of language \"python\" (artifact x)"}) writes := st.writes if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got != nil || st.writes != writes { t.Errorf("a refused kind: claim = %v, %v, writes %d (was %d), want none and no write", got, err, st.writes, writes) } + for _, e := range []string{delivery.NotPinned + "up: nats:2 names no digest", delivery.VersionTaken + "@novox/mesh-sdk@0.1.15"} { // B3, B5: ADR 0498 clauses 4, 5 + st = newMem() + seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: e}) + writes = st.writes + if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got != nil || st.writes != writes { + t.Errorf("%s: claim = %v, %v, writes %d (was %d), want none until the fingerprint changes", e, got, err, st.writes, writes) + } + } } func TestAKeyFailedForNoImageBuilderIsTakenAgainOnlyWhenANodeHasTheModule(t *testing.T) { // 1705b diff --git a/modules/mesh-delivery-builder/internal/build/plan.go b/modules/mesh-delivery-builder/internal/build/plan.go index dc1f6b7..33d48e6 100644 --- a/modules/mesh-delivery-builder/internal/build/plan.go +++ b/modules/mesh-delivery-builder/internal/build/plan.go @@ -5,16 +5,14 @@ package build import ( "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "sort" + "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) -// RefusedPrefix opens the error of a build key that is failed for a kind no builder builds yet (ADR 0478 clause 2): such -// a key is not built again as a new revision. -const RefusedPrefix = "refused kind:" - // ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that // run it. It has no toolchain language; the builder finds its nodes from the node-engine's have. const ImageBuildModule = "mesh-build-image" @@ -31,6 +29,9 @@ type Task struct { Kind string Module string Language string + // From is, for an upstream artifact, the reference its manifest pins, which the image build module copies (ADR 0498 + // clause 4). + From string } // Plan is what a manifest asks the builder to build, by artifact name. @@ -54,25 +55,34 @@ func (p Plan) Modules() []string { // (the indexer gives an archive no language, so a fingerprint names no toolchain for it; ADR 0478 clause 1). func (p Plan) NeedsToolchain(module string) bool { for _, t := range p.Tasks { - if t.Module == module && t.Kind != catalogue.ArtifactArchive && t.Kind != catalogue.ArtifactImage { + if t.Module == module && t.Kind != catalogue.ArtifactArchive && t.Kind != catalogue.ArtifactImage && t.Kind != catalogue.ArtifactUpstream { return true } } return false } -// Refused is the error of a plan the builder does not build: an artifact of a kind (or of a language) it has no build -// module for. +// Refused is the error of a plan the builder does not build: an artifact of a kind (or of a language) no build module +// builds. It opens delivery.RefusedKind, so its key is not built again until the fingerprint changes. type Refused struct{ Artifact, What string } func (r *Refused) Error() string { - return fmt.Sprintf("%s %s (artifact %s): the builder builds Go and TypeScript bundles and archives; build-agent builds it", RefusedPrefix, r.What, r.Artifact) + return fmt.Sprintf("%s%s (artifact %s): no build module builds it; the builder builds Go and TypeScript bundles, archives, images, upstream copies and TypeScript packages", delivery.RefusedKind, r.What, r.Artifact) } -// PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2): a Go bundle and an archive are built by the Go build -// module, an image by the image build module (ADR 0487), a TypeScript bundle by the Node one, each in the order of the artifacts' names. A manifest with an -// upstream or package artifact, or a bundle of another language, is refused as a whole, naming the first such artifact -// by name. A manifest with no artifact has an empty plan. +// NotPinned is the error of a plan with an upstream artifact whose reference names no digest (ADR 0498 clause 4): it opens +// delivery.NotPinned, so its key is not built again until the fingerprint changes. +type NotPinned struct{ Artifact, From string } + +func (n *NotPinned) Error() string { + return fmt.Sprintf("%s%s: %q names no digest (@sha256:), so what would be copied is whatever its tag means today", delivery.NotPinned, n.Artifact, n.From) +} + +// PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2, ADR 0498 clauses 4 and 5): a Go bundle and an archive are +// built by the Go build module, an image by the image build module (ADR 0487), which also copies an upstream image pinned +// by its digest, and a TypeScript bundle or package by the Node one, each in the order of the artifacts' names. A manifest +// with an upstream artifact pinned by no digest, or a bundle or a package of another language, or a kind no build module +// builds, is refused as a whole, naming the first such artifact by name. A manifest with no artifact has an empty plan. func PlanOf(m catalogue.Manifest) (Plan, error) { if m.Build == nil { return Plan{}, nil @@ -90,8 +100,14 @@ func PlanOf(m catalogue.Manifest) (Plan, error) { p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"}) case a.Kind == catalogue.ArtifactImage: p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule}) + case a.Kind == catalogue.ArtifactUpstream && !strings.Contains(a.From, "@sha256:"): + return Plan{}, &NotPinned{Artifact: a.Name, From: a.From} + case a.Kind == catalogue.ArtifactUpstream: + p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule, From: a.From}) case a.Kind == catalogue.ArtifactBundle: return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("bundle of language %q", a.Language)} + case a.Kind == catalogue.ArtifactPackage: + return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("package of language %q", a.Language)} default: return Plan{}, &Refused{Artifact: a.Name, What: a.Kind} } diff --git a/modules/mesh-delivery-builder/internal/build/plan_test.go b/modules/mesh-delivery-builder/internal/build/plan_test.go index d6b18b2..4a70662 100644 --- a/modules/mesh-delivery-builder/internal/build/plan_test.go +++ b/modules/mesh-delivery-builder/internal/build/plan_test.go @@ -2,6 +2,8 @@ package build import ( "errors" + + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strings" "testing" @@ -41,29 +43,35 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2 } } -func TestPlanRefusesUpstreamAndPackageByName(t *testing.T) { // 1607/AC2, ADR 0478 clause 2 - for _, c := range []struct { - artifact catalogue.Artifact - kind string - }{ - {catalogue.Artifact{Name: "up", Kind: "upstream"}, "upstream"}, - {catalogue.Artifact{Name: "pkg", Kind: "package", Language: "typescript"}, "package"}, - } { - m := manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, c.artifact) - p, err := PlanOf(m) - if err == nil { - t.Errorf("%s: plan %+v, want a refusal", c.kind, p) - continue - } - var r *Refused - if !asRefused(err, &r) || !strings.HasPrefix(err.Error(), RefusedPrefix) || !strings.Contains(err.Error(), c.kind) || !strings.Contains(err.Error(), "build-agent") { - t.Errorf("%s: error = %v, want it to open %q and name the kind and build-agent", c.kind, err, RefusedPrefix) - } +const pinned = "nats:2.11.17-alpine@sha256:3333333333333333333333333333333333333333333333333333333333333333" + +func TestPlanSendsAPinnedUpstreamToTheImageBuildModulesCopyAndRefusesAnUnpinnedOne(t *testing.T) { // B3, ADR 0498 clause 4 + p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "up", Kind: "upstream", From: pinned})) + if err != nil || len(p.Tasks) != 2 { + t.Fatalf("plan %+v, error %v", p, err) + } + if up := p.Tasks[1]; up.Artifact != "up" || up.Kind != "upstream" || up.Module != ImageBuildModule || up.From != pinned || up.Language != "" { + t.Errorf("task = %+v, want the upstream sent to %s with its pinned reference and no toolchain", up, ImageBuildModule) + } + if p.NeedsToolchain(ImageBuildModule) { + t.Error("a copy needs no toolchain") + } + _, err = PlanOf(manifestOf(catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"})) + if err == nil || !strings.HasPrefix(err.Error(), delivery.NotPinned+"up") || !delivery.FailedUntilChanged(err.Error()) { + t.Errorf("an upstream with no digest: error = %v, want it to open %q naming the artifact", err, delivery.NotPinned) + } +} + +func TestPlanRefusesAPackageOfAnotherLanguageByName(t *testing.T) { // ADR 0498 clause 5 + _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "pkg", Kind: "package", Language: "python"})) + var r *Refused + if err == nil || !asRefused(err, &r) || !strings.HasPrefix(err.Error(), delivery.RefusedKind) || !strings.Contains(err.Error(), "python") || !strings.Contains(err.Error(), "pkg") { + t.Errorf("error = %v, want it to open %q naming the artifact and its language", err, delivery.RefusedKind) } } func TestPlanRefusesABundleOfAnotherLanguageAndAModuleWithNoArtifact(t *testing.T) { - if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), RefusedPrefix) { + if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), delivery.RefusedKind) { t.Errorf("a python bundle: error = %v, want a refusal", err) } p, err := PlanOf(manifestOf()) diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 940e4fe..5a4395b 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -362,19 +362,19 @@ func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b } } -func TestAnUpstreamArtifactIsStillRefusedKeyNeverBuiltAgain(t *testing.T) { // 1607/AC2, ADR 0478 clause 2 - r := newRig(t, archiveA, catalogue.Artifact{Name: "up", Kind: "upstream"}) +func TestAnUnpinnedUpstreamFailsNotPinnedAndIsNeverBuiltAgain(t *testing.T) { // B3, ADR 0498 clause 4 + r := newRig(t, archiveA, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"}) v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, RefusedPrefix) || !strings.Contains(v.Error, "upstream") || !strings.Contains(v.Error, "build-agent") { - t.Fatalf("build = %+v, want failed 'refused kind: upstream' naming build-agent", v) + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotPinned) || !strings.Contains(v.Error, "up") { + t.Fatalf("build = %+v, want failed 'not pinned:' naming the artifact", v) } if len(r.bm.calls) != 0 { - t.Errorf("%d calls, want none for a refused kind", len(r.bm.calls)) + t.Errorf("%d calls, want none for an unpinned upstream", len(r.bm.calls)) } writes := r.st.writes r.consider(t, "modules/m") if r.st.writes != writes { - t.Errorf("a refused key was written again (%d writes, was %d)", r.st.writes, writes) + t.Errorf("an unpinned upstream's key was written again (%d writes, was %d)", r.st.writes, writes) } } From 081f32d6c929932d25315af7ce653d0baf7f957e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:59:19 +0200 Subject: [PATCH 04/25] =?UTF-8?q?Copy=20a=20pinned=20upstream=20image=20wi?= =?UTF-8?q?th=20mesh-build-image's=20copy=20step,=20with=20no=20fetch,=20s?= =?UTF-8?q?o=20its=20key=20names=20the=20registry's=20reference=20(issue?= =?UTF-8?q?=201587,=20B4,=20ADR=200498=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/build/r4e_test.go | 50 ++++++++++++++++ .../internal/build/run.go | 59 ++++++++++++------- .../internal/build/run_test.go | 6 +- .../mesh-delivery-builder/manifest_test.go | 4 +- modules/mesh-delivery-builder/module.json | 1 + 5 files changed, 95 insertions(+), 25 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index ad5b584..03fa95d 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -105,3 +106,52 @@ func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) { t.Errorf("a key whose artifacts all name references was built again (revision %d, was %d): once only", r.st.revs[r.key], rev) } } + +func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheReference(t *testing.T) { // B4, ADR 0498 clause 4 + up := catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest} + r := newRig(t, up) + r.b.Health = imageHave(t, "ace", "novox") + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone || len(v.Artifacts) != 1 { + t.Fatalf("build = %+v, want done with the copy", v) + } + if a := v.Artifacts[0]; a.Kind != "upstream" || a.Digest != digest || a.Reference != "registry.example/m/up@"+digest { + t.Errorf("artifact = %+v, want the upstream's digest and the registry's reference", a) + } + copies := r.bm.callsOf("build_image_copy") + if len(copies) != 1 { + t.Fatalf("%d copies, want one on one node", len(copies)) + } + if args := copies[0].body.(map[string]any); args["module"] != "m" || args["artifact"] != "up" || args["from"] != up.From || len(args) != 3 { + t.Errorf("copy args = %+v, want the module's name, the artifact and the pinned reference", args) + } + if n := len(r.bm.callsOf("build_image_fetch")) + len(r.bm.callsOf("build_image_build")); n != 0 { + t.Errorf("%d fetch or build calls for a copy, want none", n) + } +} + +func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(t *testing.T) { // B4 + r := newRig(t, imageArt, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}) + r.b.Health = imageHave(t, "ace") + v := r.consider(t, "modules/m") + if ok, missing := v.Referenced(); !ok { + t.Fatalf("build = %+v, missing references %v", v, missing) + } + if f, b, c := len(r.bm.callsOf("build_image_fetch")), len(r.bm.callsOf("build_image_build")), len(r.bm.callsOf("build_image_copy")); f != 1 || b != 1 || c != 1 { + t.Errorf("fetch %d, build %d, copy %d; want one each", f, b, c) + } +} + +func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2 + r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}) + r.b.Health = imageHave(t, "ace") + r.bm.fails["ace/build_image_copy"] = delivery.NotStored + "up: the registry answered 500" + v := r.consider(t, "modules/m") + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || delivery.FailedUntilChanged(v.Error) { + t.Fatalf("build = %+v, want failed 'not stored:' and built again", v) + } + delete(r.bm.fails, "ace/build_image_copy") + if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone { + t.Errorf("build = %+v, want done on the new revision", v) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 1607a71..64d8889 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -658,6 +658,8 @@ type buildResult struct { } // runLane takes a place on the node, fetches the commit there and builds each of the lane's artifacts, one step at a time. +// An upstream artifact is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches +// nothing. func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string) { release, err := b.Places.Acquire(ctx, l.node) if err != nil { @@ -674,44 +676,49 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del fetchArgs["ref"], fetchArgs["base"] = v.Ref, v.Base } } - raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs) - if err != nil { - l.err = err - return - } - var fetched fetchResult - if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" { - l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw) - return - } - // A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at - // the trunk, and building that would store the trunk's build under the candidate's fingerprint. _, candidate := fetchArgs["base"] - if candidate { - if len(fetched.Conflicts) > 0 { - l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", ")) + var fetched fetchResult + if needsFetch(l.tasks) { + raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs) + if err != nil { + l.err = err return } - if fetched.Merge == "" { - l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw) + if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" { + l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw) return } + // A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at + // the trunk, and building that would store the trunk's build under the candidate's fingerprint. + if candidate { + if len(fetched.Conflicts) > 0 { + l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", ")) + return + } + if fetched.Merge == "" { + l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw) + return + } + } } for _, t := range l.tasks { - args := map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact} - if !candidate { + tool, args := "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact} + if t.Kind == catalogue.ArtifactUpstream { + // The copy runs no code of a change: the module's name (the reference's path), the artifact and its pin. + tool, args = "copy", map[string]any{"module": module, "artifact": t.Artifact, "from": t.From} + } else if !candidate { // Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12, // ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3). args["publish"] = true } - raw, err := b.step(ctx, l.module, l.node, "build", args) + raw, err := b.step(ctx, l.module, l.node, tool, args) if err != nil { l.err = err return } var built buildResult if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" { - l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, l.node, raw) + l.err = fmt.Errorf("the %s of %s of %s on %s answered no digest: %s", tool, t.Artifact, module, l.node, raw) return } l.digests[t.Artifact] = built.Digest @@ -720,3 +727,13 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del } } } + +// needsFetch is whether any of the tasks builds from the module's source: every one but an upstream copy. +func needsFetch(tasks []Task) bool { + for _, t := range tasks { + if t.Kind != catalogue.ArtifactUpstream { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 5a4395b..739bdb3 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -69,8 +69,10 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok { d = v } - if tool == "build_image_build" && args["publish"] == true { - step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/img@` + d + `"}`) + if tool == "build_image_copy" { // mesh-build-image's copy answers the digest and where it copied to (ADR 0498 clause 4) + step.Result = json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`) + } else if tool == "build_image_build" && args["publish"] == true { + step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/` + fmt.Sprint(args["artifact"]) + `@` + d + `"}`) } else if tool != "build_image_build" && !b.noRef[node+"/"+fmt.Sprint(args["artifact"])] { // A bundle or an archive is pushed to the artifact store and its reference answered (ADR 0498 clause 1). step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://m/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`) diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 28d9afa..3fd061d 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -90,8 +90,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) { - t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node)", m.Invokes) + if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node) and the image module's copy (ADR 0498 clause 4)", m.Invokes) } // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 6d36190..460ae22 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -25,6 +25,7 @@ "mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", + "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch" From 2399a4f4cc8e4761bc97f170ab0d8860bba9cdea Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:59:43 +0200 Subject: [PATCH 05/25] =?UTF-8?q?Build=20a=20TypeScript=20package=20with?= =?UTF-8?q?=20the=20Node=20build=20module,=20published=20only=20for=20a=20?= =?UTF-8?q?trunk=20fingerprint;=20a=20version=20taken=20fails=20the=20key?= =?UTF-8?q?=20until=20the=20fingerprint=20changes=20(issue=201587,=20B5,?= =?UTF-8?q?=20ADR=200498=20=C2=A75,=20ADR=200500=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/build/plan.go | 2 + .../internal/build/r4e_test.go | 45 +++++++++++++++++++ .../internal/build/run_test.go | 9 +++- 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-builder/internal/build/plan.go b/modules/mesh-delivery-builder/internal/build/plan.go index 33d48e6..8c83b36 100644 --- a/modules/mesh-delivery-builder/internal/build/plan.go +++ b/modules/mesh-delivery-builder/internal/build/plan.go @@ -104,6 +104,8 @@ func PlanOf(m catalogue.Manifest) (Plan, error) { return Plan{}, &NotPinned{Artifact: a.Name, From: a.From} case a.Kind == catalogue.ArtifactUpstream: p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule, From: a.From}) + case a.Kind == catalogue.ArtifactPackage && a.Language == "typescript": + p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"}) case a.Kind == catalogue.ArtifactBundle: return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("bundle of language %q", a.Language)} case a.Kind == catalogue.ArtifactPackage: diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index 03fa95d..a4b7379 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -155,3 +155,48 @@ func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4 t.Errorf("build = %+v, want done on the new revision", v) } } + +var tsPackage = catalogue.Artifact{Name: "lib", Kind: "package", Language: "typescript"} + +func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4 + c := newRig(t, tsPackage) + c.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + trunkModule(t, c) + c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + c.b.Wait() + v := valueOf(t, c.st, c.key) + if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "package" || v.Artifacts[0].Reference != "" { + t.Fatalf("candidate build = %+v, want done with the packed digest and no reference", v) + } + for _, call := range c.bm.callsOf("build_node_build") { + if _, ok := call.body.(map[string]any)["publish"]; ok { + t.Errorf("a candidate's package build was asked to publish: %+v", call.body) + } + } + + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + v = r.consider(t, "modules/m") + if v.State != delivery.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" { + t.Fatalf("trunk build = %+v, want done naming the published version", v) + } + if n := len(r.bm.callsOf("build_node_build")); n == 0 { + t.Error("the package was not built by the Node build module") + } +} + +func TestAVersionTakenFailsTheKeyUntilTheFingerprintChanges(t *testing.T) { // B5, ADR 0498 clause 5 + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + r.bm.fails["ace/build_node_build"] = delivery.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file" + r.bm.fails["novox/build_node_build"] = r.bm.fails["ace/build_node_build"] + v := r.consider(t, "modules/m") + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.VersionTaken) || !strings.Contains(v.Error, "0.1.15") { + t.Fatalf("build = %+v, want failed opening %q naming the version", v, delivery.VersionTaken) + } + writes := r.st.writes + r.consider(t, "modules/m") + if r.st.writes != writes { + t.Errorf("a version-taken key was built again (%d writes, was %d)", r.st.writes, writes) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 739bdb3..f5d6068 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -25,6 +25,8 @@ type buildModules struct { fails map[string]string // `/` -> the reason the step fails with // noRef names `/` whose bundle or archive build answers no reference (its push not answered). noRef map[string]bool + // packages are the package artifacts by name, each with the `@` its trunk build publishes (ADR 0498 clause 5). + packages map[string]string // hold, when set before the builds start, blocks every call until it is closed; entered is sent to as a call arrives. // conflicts, when set, is a file a fetch with a base reports in conflict; noMerge makes it answer neither a merge nor a conflict. conflicts string @@ -34,7 +36,7 @@ type buildModules struct { } func newBuildModules() *buildModules { - return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}, noRef: map[string]bool{}} + return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}, noRef: map[string]bool{}, packages: map[string]string{}} } func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, error) { @@ -71,6 +73,11 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro } if tool == "build_image_copy" { // mesh-build-image's copy answers the digest and where it copied to (ADR 0498 clause 4) step.Result = json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`) + } else if pkg, ok := b.packages[fmt.Sprint(args["artifact"])]; ok { // a package names its version only when published + step.Result = json.RawMessage(`{"digest":"` + d + `"}`) + if args["publish"] == true { + step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"` + pkg + `"}`) + } } else if tool == "build_image_build" && args["publish"] == true { step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/` + fmt.Sprint(args["artifact"]) + `@` + d + `"}`) } else if tool != "build_image_build" && !b.noRef[node+"/"+fmt.Sprint(args["artifact"])] { From 851811a318ee90adc6bec93c8bb51f31f5e4b26a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:00:19 +0200 Subject: [PATCH 06/25] =?UTF-8?q?Build=20a=20candidate's=20done=20package?= =?UTF-8?q?=20key=20again=20once=20the=20trunk=20names=20it,=20so=20the=20?= =?UTF-8?q?package=20is=20published=20from=20the=20trunk=20(issue=201587,?= =?UTF-8?q?=20B6,=20ADR=200498=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/build/claim.go | 49 ++++++++++++++----- .../internal/build/r4e_test.go | 44 +++++++++++++++++ .../internal/build/run.go | 12 +++-- 3 files changed, 89 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index c99779f..576a450 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -4,10 +4,12 @@ import ( "context" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given. @@ -66,23 +68,48 @@ type ClaimOptions struct { // ImageBuilder says whether a node runs a healthy image build module now: a key failed for NoImageBuilderPrefix is taken // again only then. Nil means none does. ImageBuilder func() bool - // Trunk is whether the trunk's record names this fingerprint. A done key whose image has no reference was built as a - // candidate's, which never pushes (ADR 0487 clause 12); on the trunk it is taken again to publish the image, when an image - // builder runs (ImageBuilder). + // Trunk is whether the trunk's record names this fingerprint. A done key whose image or package has no reference was + // built as a candidate's, which never pushes or publishes (ADR 0487 clause 12, ADR 0498 clauses 5 and 6); on the trunk it + // is taken again to publish them, an image only when an image builder runs (ImageBuilder). Trunk bool } -// Unpublished is whether a done build holds an image built without a push: it has no reference. -func Unpublished(b delivery.BuilderBuild) bool { +// Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has +// no reference (a candidate's, ADR 0487 clause 12, ADR 0498 clauses 5 and 6). +func Unpublished(b delivery.BuilderBuild) bool { return len(UnpublishedModules(b)) > 0 } + +// UnpublishedModules are the build modules that publish what a done build holds unpublished: the image build module for +// an image, the Node build module for a package; sorted, none when it holds nothing unpublished. +func UnpublishedModules(b delivery.BuilderBuild) []string { if b.State != delivery.BuildDone { - return false + return nil } + var image, pkg bool for _, a := range b.Artifacts { - if a.Kind == "image" && a.Reference == "" { - return true + image = image || (a.Kind == catalogue.ArtifactImage && a.Reference == "") + pkg = pkg || (a.Kind == catalogue.ArtifactPackage && a.Reference == "") + } + var out []string + if image { + out = append(out, ImageBuildModule) + } + if pkg { + out = append(out, buildrun.NodeBuildModule) + } + sort.Strings(out) + return out +} + +// publishable are the build modules of UnpublishedModules that can publish now: the image build module only while a node +// runs it (imageBuilder). +func publishable(b delivery.BuilderBuild, imageBuilder func() bool) []string { + var out []string + for _, bm := range UnpublishedModules(b) { + if bm != ImageBuildModule || (imageBuilder != nil && imageBuilder()) { + out = append(out, bm) } } - return false + return out } // Unreferenced is whether a done build holds a bundle, an archive or an upstream copy that names no reference: a key done @@ -139,7 +166,7 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string if modules := OneNodeModules(cur); len(modules) > 0 && opt.MoreNodes != nil && opt.MoreNodes(modules) { return &Claim{Key: key, Rev: rev, Build: cur}, nil } - if opt.Trunk && Unpublished(cur) && opt.ImageBuilder != nil && opt.ImageBuilder() { + if opt.Trunk && len(publishable(cur, opt.ImageBuilder)) > 0 { return &Claim{Key: key, Rev: rev, Build: cur}, nil } return nil, nil diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index a4b7379..3205bb7 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -200,3 +200,47 @@ func TestAVersionTakenFailsTheKeyUntilTheFingerprintChanges(t *testing.T) { // B t.Errorf("a version-taken key was built again (%d writes, was %d)", r.st.writes, writes) } } + +func TestACandidatesPackageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // B6, ADR 0498 clause 6 + r := newRig(t, tsPackage, tsBundle) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + trunkModule(t, r) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + t.Fatalf("candidate build = %+v", v) + } + rev := r.st.revs[r.key] + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge names the candidate's fingerprint + r.b.Wait() + v := valueOf(t, r.st, r.key) + if r.st.revs[r.key] <= rev { + t.Fatalf("the trunk did not build the unpublished package again (revision %d, was %d)", r.st.revs[r.key], rev) + } + if ok, missing := v.Referenced(); !ok { + t.Fatalf("build = %+v, missing references %v", v, missing) + } + rev = r.st.revs[r.key] + r.b.tried = map[string]bool{} + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.Wait() + if r.st.revs[r.key] != rev { + t.Error("a published package was built again: once only") + } +} + +func TestAPublishThatFailsVersionTakenOnTheTrunkKeepsThePrefixFirst(t *testing.T) { // B6, ADR 0498 clauses 5 and 6 + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + trunkModule(t, r) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Wait() + for _, n := range []string{"ace", "novox"} { + r.bm.fails[n+"/build_node_build"] = delivery.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file" + } + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.VersionTaken) { + t.Errorf("build = %+v, want failed opening %q", v, delivery.VersionTaken) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 64d8889..6754c17 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -316,9 +316,11 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { for _, bm := range OneNodeModules(cl.Build) { only[bm] = true } - if trunk && Unpublished(cl.Build) { - only[ImageBuildModule] = true - publishing = true + if trunk { + for _, bm := range publishable(cl.Build, b.imageBuilderRuns) { + only[bm] = true + publishing = true + } } } result := b.build(ctx, module, m, plan, planErr, prior, only) @@ -331,9 +333,9 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { } result.By, result.Manifest = b.Claimer.By, m.Manifest if publishing && result.State == delivery.BuildFailed { - // The trunk's image was built only as a candidate's and its push failed: the key says so, so the builds view, the + // The trunk's image or package was built only as a candidate's and its push or publish failed: the key says so, so the builds view, the // checker and the failed-key retake see it, and a look does not build the image again for nothing. - result.Error = "publishing the image of the merged commit failed: " + result.Error + result.Error = keyError([]string{"publishing what the merged commit built failed: " + result.Error}) } else if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, buildrun.DigestsDifferPrefix) { // A done key built again on a second node keeps its done value when the new build fails for another reason than // the digests: the first build stands, and a later change of the nodes looks again. From 2e567b9b30d433139db43e87b5c4333956c94933 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:01:33 +0200 Subject: [PATCH 07/25] =?UTF-8?q?Let=20a=20key=20whose=20lock=20names=20an?= =?UTF-8?q?=20unpublished=20package=20version=20wait=20queued,=20naming=20?= =?UTF-8?q?it,=20and=20build=20it=20once=20the=20builder's=20own=20done=20?= =?UTF-8?q?key=20publishes=20it=20(issue=201587,=20B7,=20ADR=200500=20?= =?UTF-8?q?=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prefix 'waits for: ' is kept in the builder until mesh-sdk's delivery.WaitsFor lands on feat/r4e-sdk. --- .../internal/build/claim.go | 11 +++ .../internal/build/r4e_test.go | 55 ++++++++++++ .../internal/build/run.go | 86 ++++++++++++++++--- 3 files changed, 142 insertions(+), 10 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index 576a450..6a6659f 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -72,6 +72,10 @@ type ClaimOptions struct { // built as a candidate's, which never pushes or publishes (ADR 0487 clause 12, ADR 0498 clauses 5 and 6); on the trunk it // is taken again to publish them, an image only when an image builder runs (ImageBuilder). Trunk bool + // Published says whether a package version `@` is published: one of the builder's done keys names it as + // a package's reference. A key queued waiting for a version (BuilderBuild.Waits) is taken only once it is (ADR 0500 + // clause 6). Nil means none is. + Published func(ref string) bool } // Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has @@ -139,6 +143,8 @@ func Unreferenced(b delivery.BuilderBuild) bool { // clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4), // which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for // another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder); +// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is +// published (ClaimOptions.Published, ADR 0500 clause 6), and not before; // - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its // own again when it starts), and so is one left by another builder when ClaimOptions.TakeOver (clause 6). func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string, opt ClaimOptions) (*Claim, error) { @@ -176,6 +182,11 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string return nil, nil case cur.State == delivery.BuildFailed: return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) + case cur.State == delivery.BuildQueued && cur.Waits != "": + if opt.Published == nil || !opt.Published(cur.Waits) { + return nil, nil // the package it waits for is not published yet + } + return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) case cur.By == c.By || opt.TakeOver: return &Claim{Key: key, Rev: rev, Build: cur}, nil } diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index 3205bb7..b0246c5 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -244,3 +244,58 @@ func TestAPublishThatFailsVersionTakenOnTheTrunkKeepsThePrefixFirst(t *testing.T t.Errorf("build = %+v, want failed opening %q", v, delivery.VersionTaken) } } + +// sdkModule adds to the rig a module "sdk" whose manifest (digest b) holds the TypeScript package lib, published as ref. +func sdkModule(t *testing.T, r *rig, ref string) []byte { + t.Helper() + r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "sdk", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{tsPackage}}} + r.bm.packages["lib"] = ref + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "8a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: "src2:" + hex64('5'), + Manifest: mdigest('b'), Path: ".", Toolchains: agreedToolchains}) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePublished(t *testing.T) { // B7, ADR 0500 clause 6 + const ref = "@novox/mesh-sdk@0.1.15" + r := newRig(t, tsBundle) + for _, n := range []string{"ace", "novox"} { + r.bm.fails[n+"/build_node_fetch"] = waitsFor + ref + ": the lock names a version the registry does not hold" + } + v := r.consider(t, "modules/m") + if v.State != delivery.BuildQueued || v.Waits != ref || v.Error != "" { + t.Fatalf("build = %+v, want queued, waiting for %s, not failed", v, ref) + } + delete(r.bm.fails, "ace/build_node_fetch") + delete(r.bm.fails, "novox/build_node_fetch") + writes := r.st.writes + r.b.tried = map[string]bool{} + r.consider(t, "modules/m") + if r.st.writes != writes { + t.Fatalf("a key waiting for an unpublished package was claimed (%d writes, was %d)", r.st.writes, writes) + } + r.b.Module(context.Background(), "sdk", sdkModule(t, r, ref)) // the package's trunk key is built and published + r.b.Wait() + if v = valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Waits != "" { + t.Errorf("build = %+v, want built once the package it waits for is published", v) + } +} + +func TestAKeyWaitingForAPackageAlreadyPublishedIsBuiltWhenTheBuilderStarts(t *testing.T) { // B7: the scan at start + const ref = "@novox/mesh-sdk@0.1.15" + r := newRig(t, tsBundle) + putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildQueued, By: "novox.builder", Manifest: manifestD, Waits: ref, Step: "waits"}) + sdkKey, _ := delivery.BuilderBuildKey("sdk", "src2:"+hex64('5')) + putBuild(t, r.st, sdkKey, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: mdigest('b'), Step: "built", + Artifacts: []delivery.BuiltArtifact{{Name: "lib", Kind: "package", Digest: digest, Reference: ref}}}) + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + if err := r.b.Sweep(context.Background()); err != nil { + t.Fatal(err) + } + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + t.Errorf("build = %+v, want built: the package it waits for was published before the builder started", v) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 6754c17..c3acd58 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -63,6 +63,9 @@ type Builder struct { // whose second-node build failed is held back until it moves (issue 1664). epoch uint64 secondAt map[string]uint64 // by build key: the epoch a second-node build of a one-node key failed at + // published are the package versions `@` a done key of this bucket names as a package's reference: a key + // waiting for one is built once it is here (ADR 0500 clause 6). + published map[string]bool } type planned struct { @@ -79,7 +82,7 @@ func (b *Builder) init() { b.sem = make(chan struct{}, limit) b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} - b.secondAt = map[string]uint64{} + b.secondAt, b.published = map[string]uint64{}, map[string]bool{} if b.Places == nil { b.Places = &buildrun.Places{} } @@ -126,10 +129,11 @@ func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) } -// Sweep closes, at start, every key left building for a fingerprint its module no longer has (issue 1660, ADR 0478 -// clause 6): the key can never be built, since nothing asks for it, and would stay building for ever. It fails the key -// naming who left it. A key of the fingerprint the module has now is not touched: the build takes it over. Call it -// once the modules bucket has been read, so every module is known. +// Sweep reads, at start, every key of the bucket: it keeps the package versions done keys name as published (a key +// waiting for one is then looked at again, ADR 0500 clause 6), and closes every key left building for a fingerprint its +// module no longer has (issue 1660, ADR 0478 clause 6): the key can never be built, since nothing asks for it, and would +// stay building for ever. It fails the key naming who left it. A key of the fingerprint the module has now is not +// touched: the build takes it over. Call it once the modules bucket has been read, so every module is known. func (b *Builder) Sweep(ctx context.Context) error { b.init() keys, err := b.Claimer.Store.Keys(ctx) @@ -146,16 +150,18 @@ func (b *Builder) Sweep(ctx context.Context) error { } } b.mu.Unlock() + newly := false for _, key := range keys { - if current[key] { - continue - } value, rev, found, err := b.Claimer.Store.Get(ctx, key) if err != nil || !found { continue } cur, err := delivery.DecodeBuilderBuild(value) - if err != nil || cur.State != delivery.BuildBuilding { + if err != nil { + continue + } + newly = b.notePublished(cur) || newly + if current[key] || cur.State != delivery.BuildBuilding { continue } closed := delivery.BuilderBuild{State: delivery.BuildFailed, By: b.Claimer.By, Manifest: cur.Manifest, Step: cur.Step, @@ -169,9 +175,36 @@ func (b *Builder) Sweep(ctx context.Context) error { } b.logf("%s: closed, left building by %s for a fingerprint the module no longer has", key, cur.By) } + if newly { + // A key read before the scan may wait for a package the scan found published (ADR 0500 clause 6). + b.Reconsider(ctx) + } return nil } +// notePublished keeps the package versions a done build names as references, and says whether one was new. +func (b *Builder) notePublished(v delivery.BuilderBuild) bool { + if v.State != delivery.BuildDone { + return false + } + b.mu.Lock() + defer b.mu.Unlock() + newly := false + for _, a := range v.Artifacts { + if a.Kind == catalogue.ArtifactPackage && a.Reference != "" && !b.published[a.Reference] { + b.published[a.Reference] = true + newly = true + } + } + return newly +} + +func (b *Builder) isPublished(ref string) bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.published[ref] +} + // Forget takes a module out (its key was deleted). func (b *Builder) Forget(key string) { b.init() @@ -298,7 +331,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { // A second build that failed is not tried again until the nodes change (issue 1664). return (!failedBefore || epoch > failedAt) && b.moreNodes(modules) } - cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, MoreNodes: moreNodes, ImageBuilder: b.imageBuilderRuns, Trunk: trunk}) + cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, MoreNodes: moreNodes, ImageBuilder: b.imageBuilderRuns, Trunk: trunk, Published: b.isPublished}) if err != nil { b.logf("%s: %v", module, err) return @@ -352,6 +385,13 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if result.State == delivery.BuildFailed { b.logf("%s: failed: %s", cl.Key, result.Error) } + if result.State == delivery.BuildQueued { + b.logf("%s: waits for the package %s to be published", cl.Key, result.Waits) + } + if b.notePublished(result) { + // A key may wait for the package this build published (ADR 0500 clause 6). + b.Reconsider(ctx) + } } // stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan @@ -485,6 +525,11 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } } if len(errs) > 0 { + if ref := waitsOf(errs); ref != "" { + // The module's lock names a version of a package the registry does not hold yet: the key waits for it, queued, and + // is not failed (ADR 0500 clause 6). + return delivery.BuilderBuild{State: delivery.BuildQueued, Waits: ref, Step: "waits for the package " + ref + " to be published"} + } return failed("%s", keyError(errs)) } var artifacts []delivery.BuiltArtifact @@ -562,6 +607,27 @@ func stored(kind string) bool { return kind == catalogue.ArtifactBundle || kind == catalogue.ArtifactArchive || kind == catalogue.ArtifactUpstream } +// waitsFor opens the reason of a fetch step whose module's lock names a package version the registry does not hold, then +// `@` (the Node build module's, ADR 0500 clause 6). +const waitsFor = "waits for: " + +// waitsOf is the package version a lane's error says the module waits for, or "" when none does or what it names is not +// `@`. +func waitsOf(errs []string) string { + for _, e := range errs { + at := strings.Index(e, waitsFor) + if at < 0 { + continue + } + ref, _, _ := strings.Cut(e[at+len(waitsFor):], " ") + ref = strings.TrimRight(ref, ":;,") + if _, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildQueued, Waits: ref}); err == nil { + return ref + } + } + return "" +} + // keyPrefixes are the errors a build module's step may open its reason with that the key's error must open with too, so // the claim reads them (delivery.FailedUntilChanged), first the ones not built again. var keyPrefixes = []string{delivery.NotPinned, delivery.VersionTaken, delivery.RefusedKind, delivery.NotStored} From 957afa6926a87c0c457b1251ce24dfb154525a9a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:02:08 +0200 Subject: [PATCH 08/25] =?UTF-8?q?Ask=20the=20fetch=20with=20mesh-sdk's=20F?= =?UTF-8?q?etchRequest,=20naming=20a=20module's=20directory=20and=20the=20?= =?UTF-8?q?paths=20its=20kept=20inputs=20hold,=20so=20a=20module=20naming?= =?UTF-8?q?=20its=20paths=20is=20built=20from=20those=20alone=20(issue=201?= =?UTF-8?q?758,=20ADR=200499=20=C2=A74;=20the=20lead's=20fetch=20interface?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/build/r4e_test.go | 33 +++++++++++++++++++ .../internal/build/run.go | 19 ++++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index b0246c5..9a03c7d 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -299,3 +300,35 @@ func TestAKeyWaitingForAPackageAlreadyPublishedIsBuiltWhenTheBuilderStarts(t *te t.Errorf("build = %+v, want built: the package it waits for was published before the builder started", v) } } + +func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *testing.T) { // the lead's interface (c), ADR 0499 clause 4 + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + paths := map[string]string{"src": hex40('1'), "package.json": hex40('2')} + tree, err := delivery.NamedTree(manifestD, paths) + if err != nil { + t.Fatal(err) + } + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: fp1, Manifest: manifestD, Path: ".", + Toolchains: agreedToolchains, Inputs2: &delivery.KeptInputs2{Tree: tree, Paths: paths}}) + if err != nil { + t.Fatal(err) + } + r.b.Module(context.Background(), "m", data) + r.b.Wait() + f := r.bm.callsOf("build_node_fetch") + if len(f) == 0 { + t.Fatal("no fetch") + } + req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any)) + if err != nil || req.Dir != "." || strings.Join(req.Paths, ",") != "package.json,src" || req.Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" { + t.Errorf("fetch request = %+v, %v; want dir . and the kept paths, sorted", req, err) + } + r2 := newRig(t, goBundle) + r2.consider(t, "modules/m") + if a := r2.bm.callsOf("build_go_fetch")[0].body.(map[string]any); a["dir"] != nil || a["paths"] != nil { + t.Errorf("fetch args = %+v, want no dir or paths for a module naming none", a) + } +} + +func hex40(c byte) string { return strings.Repeat(string(c), 40) } diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index c3acd58..627cc0d 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -5,13 +5,14 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "sort" "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet). @@ -735,16 +736,26 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del return } defer release() - fetchArgs := map[string]any{"repository": repository, "commit": m.Commit} + req := buildstep.FetchRequest{Repository: repository, Commit: m.Commit} if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil { b.mu.Lock() v, candidate := b.via[key] b.mu.Unlock() if candidate { - fetchArgs["ref"], fetchArgs["base"] = v.Ref, v.Base + req.Ref, req.Base = v.Ref, v.Base } } - _, candidate := fetchArgs["base"] + if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 { + // A module whose every artifact names its paths is built from those alone (ADR 0499 clause 4): the paths come from + // the indexer's kept inputs, never from a manifest (the lead's decision L8 of R4e). + req.Dir = m.Path + for p := range m.Inputs2.Paths { + req.Paths = append(req.Paths, p) + } + sort.Strings(req.Paths) + } + fetchArgs := req.Args() + candidate := req.Base != "" var fetched fetchResult if needsFetch(l.tasks) { raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs) From 7651e47cc669a7965aef72267b457be8275bc107 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:03:02 +0200 Subject: [PATCH 09/25] =?UTF-8?q?Write=20a=20module=20with=20no=20artifact?= =?UTF-8?q?=20a=20done=20key=20with=20no=20artifact,=20so=20the=20controll?= =?UTF-8?q?er=20has=20a=20done=20key=20to=20register=20it=20from=20(issue?= =?UTF-8?q?=201613,=20ADR=200478=20=C2=A71,=20ADR=200501=20=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves the mesh-sdk pin to feat/r4e-sdk 9cb4d59, which accepts such a key and names delivery.WaitsFor; the builder's package wait now uses that shared prefix (B7). --- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 2 ++ .../internal/build/run.go | 17 ++++++++----- .../internal/build/run_test.go | 25 ++++++++++++++++--- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index a96fd50..10090b3 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022154726-a90f523edc75 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index d5ce48e..a9c5711 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022154726-a90f523edc75 h1:e/aOJLHaNSBPFFHHjUF+0fAfjmtMUy/vDWqetauWp/U= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022154726-a90f523edc75/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 627cc0d..e725a4f 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -309,10 +309,8 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module // building by this builder: it takes it again when it starts (ADR 0478 clause 5). func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { plan, planErr, read := b.planOf(ctx, m.Manifest) - if read && planErr == nil && len(plan.Tasks) == 0 { - return // nothing in the manifest for a builder to build - } - if planErr == nil { + nothing := read && planErr == nil && len(plan.Tasks) == 0 + if planErr == nil && !nothing { if why := b.stale(m, plan); why != "" { // The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build // now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written. @@ -357,7 +355,14 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { } } } - result := b.build(ctx, module, m, plan, planErr, prior, only) + var result delivery.BuilderBuild + if nothing { + // A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as + // any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1). + result = delivery.BuilderBuild{State: delivery.BuildDone, Step: "nothing to build: the manifest names no artifact"} + } else { + result = b.build(ctx, module, m, plan, planErr, prior, only) + } if ctx.Err() != nil { return } @@ -610,7 +615,7 @@ func stored(kind string) bool { // waitsFor opens the reason of a fetch step whose module's lock names a package version the registry does not hold, then // `@` (the Node build module's, ADR 0500 clause 6). -const waitsFor = "waits for: " +const waitsFor = delivery.WaitsFor // waitsOf is the package version a lane's error says the module waits for, or "" when none does or what it names is not // `@`. diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index f5d6068..aef2d5a 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -404,14 +404,33 @@ func TestAnUnreadableManifestFailsTheBuild(t *testing.T) { } } -func TestAModuleWithNoFingerprintOrNoArtifactIsNotBuilt(t *testing.T) { // 1607/AC2 +func TestAModuleWithNoFingerprintIsNotBuilt(t *testing.T) { // 1607/AC2 r := newRig(t) - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) data, _ := delivery.EncodeModule(delivery.Module{Repository: "novox/r", Commit: "c", FP: "src1:" + hex64('f'), None2: "toolchain not yet reported"}) r.b.Module(context.Background(), "n", data) r.b.Wait() if len(r.st.vals) != 0 { - t.Errorf("keys %v were written: a module with no new-form fingerprint or no artifact has nothing to build", r.st.vals) + t.Errorf("keys %v were written: a module with no new-form fingerprint has nothing to build", r.st.vals) + } +} + +func TestAModuleWithNoArtifactGetsADoneKeyWithNoArtifactAndNoBuildCall(t *testing.T) { // issue 1613, ADR 0478 clause 1 + for name, manifest := range map[string]catalogue.Manifest{"no build": {Module: "m"}, "no artifact": {Module: "m", Build: &catalogue.Build{}}} { + r := newRig(t) + r.mf.byDigest[mdigest('a')] = manifest + v := r.consider(t, "") // a module with nothing to build needs no directory + if v.State != delivery.BuildDone || len(v.Artifacts) != 0 || v.Manifest != manifestD || v.By != "novox.builder" { + t.Errorf("%s: build = %+v, want done with no artifact", name, v) + } + if len(r.bm.calls) != 0 { + t.Errorf("%s: %d build module calls, want none", name, len(r.bm.calls)) + } + writes := r.st.writes + r.b.tried = map[string]bool{} + r.consider(t, "") + if r.st.writes != writes { + t.Errorf("%s: the done key was written again", name) + } } } From 8b2e8895c223a89b274372c07408c2912ce2e75b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:03:39 +0200 Subject: [PATCH 10/25] Prove on a real bus that every done key names its references, a refused push is built again, and a pinned upstream is copied (issue 1587, B8, ADR 0498 checks) --- .../mesh-delivery-builder/integration_test.go | 53 +++++++++++- .../catalogue-49caf17f-hello-web.module.json | 82 +++++++++++++++++++ 2 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-49caf17f-hello-web.module.json diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 6dd6b42..3dcd257 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -54,6 +54,8 @@ type fakeBuildModules struct { ctx context.Context n atomic.Int64 failBuild atomic.Bool + // refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2). + refusePush atomic.Bool mu sync.Mutex calls map[string]int // by `.` fetches []map[string]any @@ -121,6 +123,14 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"`+merge+`}`) case f.failBuild.Load() && node == "n1": step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"]) + case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4) + d := digestOf(fmt.Sprint(args["artifact"])) + step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"artifact":"`+fmt.Sprint(args["artifact"])+`","digest":"`+d+`","reference":"registry.example/`+fmt.Sprint(args["module"])+`/`+fmt.Sprint(args["artifact"])+`@`+d+`","pushed":true}`) + case tool != "build_image_build" && f.refusePush.Load(): + step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"]) + case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1) + d := digestOf(fmt.Sprint(args["artifact"])) + step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"artifact-store://`+filepath.Base(fmt.Sprint(args["module"]))+`/`+fmt.Sprint(args["artifact"])+`/blobs/`+d+`"}`) case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference d := digestOf(fmt.Sprint(args["artifact"])) step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"registry.example/imaged/server@`+d+`"}`) @@ -374,8 +384,8 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 t.Fatalf("build = %+v", v) } for _, a := range v.Artifacts { - if a.Digest != digestOf(a.Name) || a.Kind != "bundle" || a.Reference != "" { - t.Errorf("artifact = %+v, want the digest every node built and no reference", a) + if a.Digest != digestOf(a.Name) || a.Kind != "bundle" || a.Reference != "artifact-store://netcheck/"+a.Name+"/blobs/"+a.Digest { + t.Errorf("artifact = %+v, want the digest every node built and the reference its push answered (ADR 0498 clause 1)", a) } } if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" { @@ -674,3 +684,42 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170 t.Errorf("builds = %v, want three, two of them with publish (the trunk's two fingerprints)", b.fake.builds) } } + +// TestEveryDoneKeyNamesItsReferencesAndARefusedPushIsBuiltAgain: with the artifact store refusing pushes, the key of an +// archive module ends failed opening "not stored:"; once it takes them, a new modules value builds the key again, done, its +// every artifact naming a reference (issue 1587, B8, ADR 0498 clauses 1, 2 and 8). +func TestEveryDoneKeyNamesItsReferencesAndARefusedPushIsBuiltAgain(t *testing.T) { + b := startBed(t) + b.fake.refusePush.Store(true) + b.archive(t, "ca-trust", "modules/ca-trust", commit) + b.start(t, "n1", nil) + v := b.await(t, "ca-trust", isState(delivery.BuildFailed)) + if !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "507") { + t.Fatalf("build = %+v, want failed opening %q with the store's words", v, delivery.NotStored) + } + b.fake.refusePush.Store(false) + b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef") + v = b.await(t, "ca-trust", isState(delivery.BuildDone)) + if ok, missing := v.Referenced(); !ok { + t.Errorf("build = %+v, artifacts naming no reference: %v", v, missing) + } +} + +// TestAPinnedUpstreamIsCopiedByTheImageBuildModule: hello-web's manifest (captured from mesh-catalog 49caf17f) names an +// upstream image pinned by its digest and an archive; the upstream is copied on n1, where mesh-build-image runs, with no +// fetch of the image module, and the key ends done with both references (issue 1587, B8, ADR 0498 clause 4). +func TestAPinnedUpstreamIsCopiedByTheImageBuildModule(t *testing.T) { + b := startBed(t) + b.moduleCommit(t, "hello-web", "catalogue-49caf17f-hello-web.module.json", "modules/hello-web", nil, commit) + b.start(t, "n1", nil) + v := b.await(t, "hello-web", isState(delivery.BuildDone)) + if ok, missing := v.Referenced(); !ok { + t.Fatalf("build = %+v, artifacts naming no reference: %v", v, missing) + } + if n := b.fake.count("mesh-build-image.build_image_copy"); n != 1 { + t.Errorf("build_image_copy called %d times, want once", n) + } + if n := b.fake.count("mesh-build-image.build_image_fetch"); n != 0 { + t.Errorf("build_image_fetch called %d times, want none for a copy", n) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-49caf17f-hello-web.module.json b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-49caf17f-hello-web.module.json new file mode 100644 index 0000000..005578c --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-49caf17f-hello-web.module.json @@ -0,0 +1,82 @@ +{ + "module": "hello-web", + "slug": "hello", + "version": "1", + "capabilities": [ + "container-runtime" + ], + "requires": [ + "route" + ], + "contributes": { + "route": { + "label": "hello", + "endpoint": "web" + } + }, + "binds": { + "route": "${dir:state}/route.json" + }, + "listens": [ + { + "name": "web", + "port": 8080, + "protocol": "tcp", + "from": "mesh", + "why": "the demo web page; only the route-proxy reaches it, and the public name is a route grant" + } + ], + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, + { + "id": "page", + "type": "file", + "path": "${dir:state}/index.html", + "mode": "0644", + "template": "files/index.html" + }, + { + "id": "net", + "type": "network", + "name": "hello-web" + }, + { + "id": "server", + "type": "container", + "name": "hello-web", + "network": "hello-web", + "ports": [ + "8080" + ], + "volumes": [ + "${dir:state}/index.html:/www/index.html:ro" + ], + "args": [ + "sh", + "-c", + "while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done" + ], + "artifact": "server" + } + ], + "build": { + "artifacts": [ + { + "name": "server", + "kind": "upstream", + "from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} From 9f7e082688a14d6163d20bfd598ee940bf6c1a09 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:08:48 +0200 Subject: [PATCH 11/25] =?UTF-8?q?List=20a=20repository's=20modules=20from?= =?UTF-8?q?=20the=20merged=20message's=20manifests=20and=20the=20indexer's?= =?UTF-8?q?=20own=20bucket,=20not=20the=20controller's=20build=20records,?= =?UTF-8?q?=20so=20a=20new=20module=20gets=20a=20value=20and=20a=20key=20w?= =?UTF-8?q?ith=20no=20build=20asked=20(issue=201851,=20I1=20I2=20I4,=20ADR?= =?UTF-8?q?=200501=20=C2=A76,=20ADR=200453=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I1: a merge writes each module its manifests name, at its directory. I2: a pull request's changed directory, a preview's touched module and a source revision's modules come from the modules bucket (a module a pull request adds is found in the preview by its manifest). I4: the value of today's form (fp, none) is no longer written, since it was computed from the build records; mesh-sdk keeps it readable until R4g. One commit: the three share applyMerged and its tests. Pins mesh-sdk feat/r4e-sdk 9cb4d59. --- .../mesh-delivery-indexer/integration_test.go | 24 +- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 2 + .../index/candidate_manifests_test.go | 4 +- .../internal/index/candidates.go | 12 +- .../internal/index/index_test.go | 32 ++- .../internal/index/live_builds_test.go | 33 --- .../internal/index/manifests.go | 17 +- .../internal/index/manifests_test.go | 5 +- .../internal/index/modules.go | 105 ++++---- .../internal/index/modules_test.go | 224 +++++++----------- .../internal/index/newform_test.go | 31 +-- .../internal/index/own.go | 81 +++++++ .../internal/index/previews.go | 27 ++- .../internal/index/source.go | 56 +++-- .../internal/index/source_test.go | 9 +- 16 files changed, 325 insertions(+), 339 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/own.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index a885d9d..2d49b96 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -183,30 +183,18 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b // TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged // message on FORGE makes modules. within 5 s, the fingerprint of the merge's tree and the recorded inputs. -func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9 - // Inputs and record in the controller's layout: SourceInputs under its Go field names (mesh-sdk - // go/delivery/testdata/build-record-build-agent.json, captured from the controller's encoder). - const inputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` - js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-delivery","ref":"main","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/a","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` - if _, err := builds.Put(ctx, delivery.BuildKey("a", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey("a"), []byte("build-1")); err != nil { - t.Fatal(err) - } - }) +func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1: the modules are the merged message's, no build record names them (issue 1851) + js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}) const merge = "89abcdef0123456789abcdef0123456789abcdef" data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}}) + Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}, + Manifests: map[string]string{"modules/a": `{"module":"a","version":"1"}`}}) if err != nil { t.Fatal(err) } if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged", data); err != nil { t.Fatal(err) } - want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) var m delivery.Module for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if e, err := modules.Get(ctx, "a"); err == nil { @@ -215,8 +203,8 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9 } } } - if m.FP != want || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" { // 1606/AC2 - t.Errorf("module = %+v within 5 s, want fp %s and path modules/a", m, want) + if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.FP != "" || m.None != "" { // 1606/AC2 + t.Errorf("module = %+v within 5 s, want path modules/a, its manifest, and no value of today's form", m) } } diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index a97df59..1f77135 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 3515fb8..bb52af2 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go index bdadf94..41c2f1b 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go @@ -73,8 +73,8 @@ func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/ func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3 avahi := catalogueManifest(t, "avahi") - // The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory. - other := catalogueManifest(t, "ca-trust") + // The second pull request carries avahi's manifest changed (a line added at its end) for the same directory. + other := avahi + "\n" ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 2f1bf70..c322283 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -78,11 +78,15 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { } } var digests map[string]string + var held map[string]delivery.Module if f.Kind == delivery.ForgePull { var err error if digests, err = ix.keepManifests(ctx, f); err != nil { return err } + if held, err = ix.heldModules(ctx); err != nil { + return err + } } if trunkLike[f.Branch] { ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch) @@ -101,7 +105,7 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { } cur = &c } - next := ix.next(cur, f, digests) + next := ix.next(cur, f, digests, held) if next != nil { ix.withPreviews(ctx, next) if err := ix.refuseDisagreement(*next); err != nil { @@ -207,8 +211,8 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte, // A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of // its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a // pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by -// directory. -func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate { +// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir). +func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate { var pulls []delivery.CandidatePull var base uint64 if cur != nil { @@ -225,7 +229,7 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s switch { case f.DirsKnown: for _, dir := range f.Dirs { - m := ix.Builds.ModuleOf(f.Repository, dir) + m := moduleOfDir(held, f, dir) if !slices.Contains(entry.Modules, m) { entry.Modules = append(entry.Modules, m) } diff --git a/modules/mesh-delivery-indexer/internal/index/index_test.go b/modules/mesh-delivery-indexer/internal/index/index_test.go index 8fa182b..74bcc57 100644 --- a/modules/mesh-delivery-indexer/internal/index/index_test.go +++ b/modules/mesh-delivery-indexer/internal/index/index_test.go @@ -148,27 +148,35 @@ func TestTwoRepositoriesFormOneGroupByBranch(t *testing.T) { // 1238/AC2 (the gr } } -func TestModuleOfADirectory(t *testing.T) { // 1238/AC3 +func TestModuleOfADirectory(t *testing.T) { // 1238/AC3, as R4e has it: the module of a directory is the indexer's own (I2, ADR 0501 clause 6) ix, st, _ := newIndexer(t) - // modules/odd is named by a build record of the repository (module odd-name); modules/plain is named by none. - if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil { + mods := newMem() + ix.Modules = mods + ctx := context.Background() + // The trunk holds modules/odd as the module odd-name (its manifest names it so); modules/plain is held by none. + merged := mergedMsg("novox/mesh-delivery", map[string]string{"modules/odd": "t"}) + merged.Manifests["modules/odd"] = `{"module":"odd-name","version":"1"}` + if err := ix.Apply(ctx, merged); err != nil { + t.Fatal(err) + } + if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil { t.Fatal(err) } c, _ := read(t, st, "feat/x") if strings.Join(c.Pulls[0].Modules, ",") != "odd-name,plain" { - t.Errorf("modules = %v, want the record's module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules) + t.Errorf("modules = %v, want the held module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules) } - // A record of another repository does not name this repository's directory. - if m, ok := ix.Builds.Module("novox/other", "modules/a"); ok { - t.Errorf("modules/a of novox/other is %q", m) + held := map[string]delivery.Module{"x": {Repository: "novox/other", Path: "modules/a"}} + if name, ok := moduleAt(held, "novox/mesh-delivery", "modules/a"); ok { + t.Errorf("modules/a of novox/mesh-delivery is %q, held only of novox/other", name) } - // The newest record of a directory wins. - ix.Builds.Put(delivery.BuildKey("renamed", "build-9"), buildRecord(t, "novox/mesh-delivery", "renamed", "modules/a", "2026-10-19T10:00:00Z")) - if m := ix.Builds.ModuleOf("novox/mesh-delivery", "modules/a"); m != "renamed" { - t.Errorf("modules/a = %q, want the newest record's module", m) + // A manifest the pull message carries names its module, whatever directory it is in. + f := pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/odd") + f.Manifests = map[string]string{"modules/odd": `{"module":"renamed","version":"1"}`} + if name := moduleOfDir(held, f, "modules/odd"); name != "renamed" { + t.Errorf("modules/odd = %q, want the module its manifest names", name) } } - func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4 ix, st, _ := newIndexer(t) ctx := context.Background() diff --git a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go index 4794ab6..a196a54 100644 --- a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go +++ b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go @@ -1,7 +1,6 @@ package index import ( - "context" "testing" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -51,38 +50,6 @@ func liveIndexer(t *testing.T) (*Indexer, *memStore) { return ix, mods } -func TestMergeWritesEveryModuleTheLiveRecordsNameInTheRepository(t *testing.T) { // 1281/AC3 - ix, mods := liveIndexer(t) - msg := mergedMsg("novox/mesh-catalog", map[string]string{ - "modules/zsh": liveTree, "modules/mesh-delivery": "newtree", "modules/mailu": "newtree", "modules/sonarr": "newtree", - }) - if err := ix.Apply(context.Background(), msg); err != nil { - t.Fatal(err) - } - for _, name := range []string{"zsh", "mesh-delivery", "mailu"} { - m, ok := moduleOf(t, mods, name) - if !ok { - t.Errorf("%s: no module key written", name) - continue - } - if (m.FP == "") == (m.None == "") || m.Repository != "novox/mesh-catalog" || m.Commit != merge { - t.Errorf("%s: module = %+v, want an fp or a none, repository novox/mesh-catalog, commit %s", name, m, merge) - } - } - // the tree of zsh at the merge is the tree its inputs record, so its fingerprint is the one those inputs give - var in delivery.SourceInputs - in.Module, in.Tree = "zsh", liveTree - in.Bases = []string{"sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"} - in.Toolchains = map[string]string{"code": "typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"} - want, _ := delivery.SourceFingerprint(in) - if m, _ := moduleOf(t, mods, "zsh"); m.FP != want { - t.Errorf("zsh fp = %q, want %q", m.FP, want) - } - if _, ok := mods.vals["sonarr"]; ok { - t.Error("the key of a module of mesh-media-catalog was written at a merge of mesh-catalog") - } -} - func TestChangedDirectoryMapsToTheModuleItsLiveRecordNamesByPath(t *testing.T) { // 1281/AC4 ix, _ := liveIndexer(t) for dir, want := range map[string]string{"modules/zsh": "zsh", "modules/mesh-delivery": "mesh-delivery", "modules/mailu/": "mailu"} { diff --git a/modules/mesh-delivery-indexer/internal/index/manifests.go b/modules/mesh-delivery-indexer/internal/index/manifests.go index d1883b1..740e9ce 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests.go @@ -2,27 +2,12 @@ package index import ( "context" - "encoding/json" "errors" "fmt" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests -// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the -// module a build record names the directory the path of. -func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string { - var m struct { - Module string `json:"module"` - } - if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" { - return m.Module - } - name, _ := ix.Builds.Module(repository, dir) - return name -} - // keepManifest writes manifests. for a manifest as the forge answered it, create-once (issue 1449): a key that // exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for // one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the @@ -66,7 +51,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) { digests := make(map[string]string, len(f.Manifests)) for dir, raw := range f.Manifests { - module := ix.moduleNameOf(f.Repository, dir, raw) + module := manifestModule(raw) // the name the manifest gives itself: one digest always has one module digest, err := ix.keepManifest(ctx, module, raw) switch { case err == nil: diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go index 8c26287..08455bb 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -96,7 +96,8 @@ func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a if len(manifests.vals) != 0 || len(logs) == 0 { t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs) } - if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" { - t.Errorf("module = %+v (%v), want no manifest digest", mod, ok) + // A manifest that is not JSON names no module (I1): nothing is written for its directory. + if mod, ok := moduleOf(t, mods, "avahi"); ok { + t.Errorf("module = %+v, want none written for a manifest naming no module", mod) } } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 7e55e43..1503df6 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -2,78 +2,65 @@ package index import ( "context" - "sort" - "strings" + "maps" + "slices" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// The reasons a module has no fingerprint (ADR 0437 §5; issue 1209's decision). -const ( - // NoneNarrow is for a module whose image compiles a Go program from a context: the context is the hash of the build - // source the builder handed it, not a git tree. - NoneNarrow = "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4" - // NoneContext is for a context that is a repository's root tree. - NoneContext = "an artifact reads the root tree of a repository the indexer does not know the tree of before release 4" - // NoneNoBuild is for a module no successful trunk build recorded the inputs of. - NoneNoBuild = "no successful trunk build recorded its inputs" -) - -// fingerprintOf is the module's fingerprint at the merge, or none and why: its directory's tree from the merged -// message and the bases, contexts and toolchains its newest successful trunk build recorded. -func fingerprintOf(k Known, trees map[string]string) (fp, none string) { - if k.Trunk == nil { - return "", NoneNoBuild - } - in := *k.Trunk.SourceInputs - artifacts := make([]string, 0, len(in.Contexts)) - for a := range in.Contexts { - artifacts = append(artifacts, a) - } - sort.Strings(artifacts) - for _, a := range artifacts { - if strings.HasPrefix(in.Contexts[a], "narrow:") { - return "", NoneNarrow - } - } - if len(artifacts) > 0 { - return "", NoneContext - } - in.Tree = trees[k.Path] // the directory "" is the repository's root - return delivery.SourceFingerprint(in) -} - -// applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and -// why), with the source revision that names the merge commit as the repository's head when the indexer has seen it -// (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already -// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it -// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449, -// 1450); the fingerprint itself is computed by Recompute (toolchains.go) once the merge is written, and again when a build -// module reports its toolchain. +// applyMerged writes, for each module whose manifest the merged message carries (each directory of the merge commit that +// holds a module.json, the module its manifest names), its value at the merge commit: its directory, its manifest digest, +// its new-form inputs and why it has no new-form fingerprint yet, with the source revision that names the merge commit as +// the repository's head when the indexer has seen it (ApplySource), and Src 0 when it has not. The fingerprint itself is +// computed by Recompute (toolchains.go) once the merge is written, and again when a build module reports its toolchain. +// +// The modules are listed from the message, never from the controller's build records (novox/hq ADR 0501 clause 6, issue +// 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's +// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the +// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420 +// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other. +// A value the module already holds is not written again. func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { digests, err := ix.keepManifests(ctx, f) if err != nil { return err } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } overlay := map[string]delivery.Module{} - for _, k := range ix.Builds.Modules(f.Repository) { - key, err := delivery.ModulesKey(k.Module) - if err != nil { - ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, k.Module, err) + dirOf := map[string]string{} + for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) { + raw := f.Manifests[dir] + name := manifestModule(raw) + if name == "" { + ix.logf("%s at %s: the manifest in %q names no module, so no module is written for it", f.Repository, f.Merge, dir) continue } - fp, none := fingerprintOf(k, f.Trees) - next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path], Path: pathOf(k.Path)} - // The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which - // names what is missing, from its manifest and lock hashes (inputs.go). - raw, has := f.Manifests[k.Path] - switch { - case !has: - next.None2 = None2NoManifest - case next.Manifest == "": + key, err := delivery.ModulesKey(name) + if err != nil { + ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, name, err) + continue + } + cur, has := held[key] + if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) { + ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository) + continue + } + if other, dup := dirOf[name]; dup { + if !has || cur.Path != pathOf(dir) { + ix.logf("%s at %s: %q and %q both name the module %s; %q is written", f.Repository, f.Merge, other, dir, name, other) + continue + } + ix.logf("%s at %s: %q and %q both name the module %s; %q, which the indexer holds, is written", f.Repository, f.Merge, other, dir, name, dir) + } + dirOf[name] = dir + next := delivery.Module{Repository: f.Repository, Commit: f.Merge, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[dir], Path: pathOf(dir)} + if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" - default: - in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx)) + } else { + in := inputsOf(name, raw, f.Trees[dir], f.Locks[dir], ix.baseFP(ctx)) next.Inputs2, next.None2 = in.Kept, in.None2 } overlay[key] = next diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 36eee4f..ca0cf36 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -3,6 +3,7 @@ package index import ( "context" "encoding/json" + "path" "strings" "testing" @@ -51,8 +52,22 @@ func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2 return [2]any{k, v} } +// mergedMsg is a merged message of the repository with these module directories' trees, each with a manifest naming its +// module, the directory's last element ("r" for the root), as the forge bridge sends it (one manifest per directory that +// holds a module.json). func mergedMsg(repo string, trees map[string]string) delivery.Forge { - return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees} + manifests := map[string]string{} + for dir := range trees { + manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1"}` + } + return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests} +} + +func nameOfDir(dir string) string { + if dir == "" { + return "r" + } + return path.Base(dir) } func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) { @@ -70,126 +85,6 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) { const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` -func TestModuleFingerprintFromTheMergeTreeAndTheRecordedInputs(t *testing.T) { // 1239/AC1 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree", "modules/b": "other"})); err != nil { - t.Fatal(err) - } - want, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) - if none != "" { - t.Fatal(none) - } - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP != want || m.None != "" || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Src != 0 { - t.Errorf("module = %+v (%v), want fp %s", m, ok, want) - } - if _, ok := mods.vals["b"]; ok { - t.Error("a key was written for a directory no build names") - } -} - -func TestMergeOfTheRecordedTreeReproducesTheRecordedFingerprint(t *testing.T) { // 1239/AC2 - // mailu, build-1792391134620131370: mesh-catalog at b4527d9091e972f28cb365623d9280045e7741a3, dir modules/mailu. Its - // inputs and recorded fingerprint are the live ones mesh-sdk go/delivery/fingerprint_test.go TestFingerprintLiveBuilds - // captured (its tree is `git rev-parse :modules/mailu`, its bases from the build's [bases] log line). - const tree = "cfc6fa8ff5fc1e220132e51ade653b9d3f387b3e" - const recorded = "src1:539f2e64000d436c572ef53b40e524012455945f8c5a1c6bb173ee0707e88ef4" - inputs := `{"Module":"mailu","Tree":"` + tree + `","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,` + - `"Toolchains":{"code":"typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"},"Unpinned":""}` - ix, mods := modulesIndexer(t, rec(t, "build-1792391134620131370", "mailu", "novox/mesh-catalog", "modules/mailu", "main", "", "2026-10-19T09:00:00Z", inputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-catalog", map[string]string{"modules/mailu": tree})); err != nil { - t.Fatal(err) - } - if m, ok := moduleOf(t, mods, "mailu"); !ok || m.FP != recorded { - t.Errorf("module = %+v (%v), want fp %s", m, ok, recorded) - } -} - -func TestANarrowContextHasNoFingerprint(t *testing.T) { // 1239/AC3, 1209/AC1 - // A Go program's build source as the builder records it (build-agent's record, mesh-sdk go/delivery/testdata): - // the context "server" is "narrow:" and a hash. - inputs := `{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""}` - ix, mods := modulesIndexer(t, rec(t, "build-1", "build-agent", "novox/mesh-controller", "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", inputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-controller", map[string]string{"modules/build-agent": "newtree"})); err != nil { - t.Fatal(err) - } - m, ok := moduleOf(t, mods, "build-agent") - if !ok || m.FP != "" || !strings.Contains(m.None, "Go program") || !strings.Contains(m.None, "release 4") { - t.Errorf("module = %+v (%v), want none naming the Go program's build source and release 4", m, ok) - } -} - -func TestUnpinnedBuildHasNoFingerprint(t *testing.T) { // 1239/AC4 - inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":null,"Toolchains":null,"Unpinned":"the build resolves packages when built"}` - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs)) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "the build resolves packages when built" { - t.Errorf("module = %+v (%v)", m, ok) - } -} - -func TestARootTreeContextHasNoFingerprint(t *testing.T) { // 1239/AC5 - inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":{"web":"f00dcafef00dcafef00dcafef00dcafef00dcafe"},"Toolchains":null,"Unpinned":""}` - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs)) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP != "" || !strings.Contains(m.None, "root tree") || !strings.Contains(m.None, "release 4") { - t.Errorf("module = %+v (%v)", m, ok) - } -} - -func TestDirectoryGoneFromTheMergeHasNoTree(t *testing.T) { // 1239/AC6 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/other": "t"})) - if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "no tree" { - t.Errorf("module = %+v (%v)", m, ok) - } -} - -func TestAModuleWithNoSuccessfulTrunkBuildHasNone(t *testing.T) { // 1239/AC7 - ix, mods := modulesIndexer(t, - rec(t, "build-1", "failed-only", "novox/mesh-delivery", "modules/f", "main", "failed", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-2", "branch-only", "novox/mesh-delivery", "modules/g", "feat/x", "", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-3", "no-inputs", "novox/mesh-delivery", "modules/h", "main", "", "2026-10-19T09:00:00Z", ""), - ) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/f": "t", "modules/g": "t", "modules/h": "t"})) - for _, name := range []string{"failed-only", "branch-only", "no-inputs"} { - if m, ok := moduleOf(t, mods, name); !ok || m.FP != "" || !strings.Contains(m.None, "no successful trunk build") { - t.Errorf("%s: module = %+v (%v)", name, m, ok) - } - } -} - -func TestTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1239 (the newest key is only a hint) - older := strings.Replace(aInputs, `"Bases":["registry.example/base@sha256:aaa"]`, `"Bases":["registry.example/base@sha256:old"]`, 1) - ix, mods := modulesIndexer(t, - rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", older), - rec(t, "build-2", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T10:00:00Z", aInputs), - rec(t, "build-3", "a", "novox/mesh-delivery", "modules/a", "main", "failed", "2026-10-19T11:00:00Z", older), - rec(t, "build-4", "a", "novox/mesh-delivery", "modules/a", "feat/x", "", "2026-10-19T12:00:00Z", older), - ) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) - if m, _ := moduleOf(t, mods, "a"); m.FP != want { - t.Errorf("fp = %s, want the 10:00 trunk build's %s", m.FP, want) - } -} - -func TestModulesOfOtherRepositoriesAreNotWritten(t *testing.T) { // 1239/AC8 - ix, mods := modulesIndexer(t, - rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", strings.Replace(aInputs, `"Module":"a"`, `"Module":"c"`, 1))) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/c": "t"})) - if _, ok := mods.vals["c"]; ok { - t.Error("the key of a module of another repository was written") - } - if _, ok := mods.vals["a"]; !ok { - t.Error("the key of the merged repository's module was not written") - } -} - func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing) ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}) @@ -211,21 +106,6 @@ func TestMergedMessageRemovesThePullFromItsGroupToo(t *testing.T) { // 1238/AC5 } } -func TestARecordAsTheControllerWritesItGivesAFingerprint(t *testing.T) { // 1281/AC8 - // The layout of mesh-controller internal/chain/records.go Build (commit 1f708e2d8d72): an outcome has no state key, a build - // asked by a plan after a merge no ref, and the repository is the clone URL. - raw := `{"id":"build-1","repository":"http://novox.internal:20000/novox/mesh-delivery.git","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20",` + - `"path":"modules/a","source-fingerprint":"src1:ignored","source-inputs":` + aInputs + `,"at":"2026-10-19T09:00:00Z"}` - ix, mods := modulesIndexer(t, [2]any{delivery.BuildKey("a", "build-1"), []byte(raw)}) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { - t.Fatal(err) - } - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP == "" || m.None != "" { - t.Errorf("module = %+v (%v), want a fingerprint, not none", m, ok) - } -} - func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *testing.T) { // 1606/AC1, AC2 ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), @@ -263,3 +143,75 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means t.Errorf("module = %+v (%v), want path %q", m, ok, delivery.RootPath) } } + +func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6 + ix, mods, _ := manifestsIndexer(t) // no build record of any module + avahi := catalogueManifest(t, "avahi") + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, + map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "avahi") + if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" { + t.Fatalf("module = %+v (%v), want its directory, merge, manifest and inputs from the message alone", m, ok) + } + if m.FP != "" || m.None != "" { + t.Errorf("module = %+v, want no value of today's form (I4: it came from the build records)", m) + } + if m.FP2 == "" && m.None2 == "" { + t.Errorf("module = %+v, want a new-form fingerprint or its reason", m) + } +} + +func TestAMergeWritesOnlyTheModulesItsManifestsName(t *testing.T) { // I1 + ix, mods := modulesIndexer(t, rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/b": "t"}) + f.Trees["modules/c"] = "t" // a directory whose manifest the message does not carry + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + for name, want := range map[string]bool{"a": true, "b": true, "c": false} { + if _, ok := mods.vals[name]; ok != want { + t.Errorf("modules.%s written %v, want %v", name, ok, want) + } + } +} + +func TestAModuleHeldOfAnotherRepositoryIsLeftAndSaid(t *testing.T) { // I1: a move between repositories is the operator's (ADR 0420 clause 7) + ix, mods := modulesIndexer(t) + var logs []string + ix.Logf = func(format string, args ...any) { logs = append(logs, format) } + ctx := context.Background() + if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "t"})); err != nil { + t.Fatal(err) + } + before := string(mods.vals["a"]) + if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t2"})); err != nil { + t.Fatal(err) + } + if string(mods.vals["a"]) != before || len(logs) == 0 { + t.Errorf("modules.a = %s (was %s), logs %v: want it left, and said", mods.vals["a"], before, logs) + } +} + +func TestTwoDirectoriesNamingOneModuleWriteOneAndSayTheOther(t *testing.T) { // I1 + ix, mods := modulesIndexer(t) + var logs []string + ix.Logf = func(format string, args ...any) { logs = append(logs, format) } + f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t1", "other/a": "t2"}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, ok := moduleOf(t, mods, "a"); !ok || m.Path != "modules/a" || len(logs) == 0 { + t.Errorf("module = %+v (%v), logs %v: want the first directory by name, and the other said", m, ok, logs) + } + // Once held, the directory the bucket holds stays the module's. + f = mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t1", "aaa/a": "t3"}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "a"); m.Path != "modules/a" { + t.Errorf("module = %+v, want the held directory kept", m) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/newform_test.go b/modules/mesh-delivery-indexer/internal/index/newform_test.go index a610a9c..31247a6 100644 --- a/modules/mesh-delivery-indexer/internal/index/newform_test.go +++ b/modules/mesh-delivery-indexer/internal/index/newform_test.go @@ -7,17 +7,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go). -func wantAvahiFP(t *testing.T) string { - t.Helper() - fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) - if none != "" { - t.Fatal(none) - } - return fp -} - func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1 avahi := catalogueManifest(t, "avahi") ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", @@ -31,16 +20,14 @@ func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t if !ok { t.Fatal("no module avahi") } - todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum { t.Errorf("module = %+v, want its manifest digest and inputs", m) } if m.None2 != "toolchain not yet reported" || m.FP2 != "" { t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2) } - if m.FP != todays || m.None != "" { - t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays) + if m.FP != "" || m.None != "" { + t.Errorf("fp = %q none = %q, want no value of today's form (I4)", m.FP, m.None) } } @@ -55,17 +42,15 @@ func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) } } -func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3 +func TestAMergedMessageWithoutManifestsWritesNoModule(t *testing.T) { // 1444/AC3 as R4e has it: the modules are the manifests' (I1) ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { + f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}) + f.Manifests = nil + if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) } - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" { - t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok) - } - if len(manifests.vals) != 0 { - t.Errorf("manifests holds %d keys", len(manifests.vals)) + if len(mods.vals) != 0 || len(manifests.vals) != 0 { + t.Errorf("modules %d keys, manifests %d keys, want none: a message naming no manifest names no module", len(mods.vals), len(manifests.vals)) } } diff --git a/modules/mesh-delivery-indexer/internal/index/own.go b/modules/mesh-delivery-indexer/internal/index/own.go new file mode 100644 index 0000000..bcb0f81 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/own.go @@ -0,0 +1,81 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "path" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// From R4e the indexer lists a repository's modules from what it holds itself (novox/hq ADR 0501 clause 6, issue 1851): +// a merged message's manifests name the modules of the merge, and the modules bucket names the modules it wrote before, +// with their repository and directory. The controller's build records name none of them (ADR 0453 clause 5 keeps only +// the builder's own input from them until R4g, builds.go). + +// manifestModule is the module a manifest names itself (its `module`), or "" when it names none or is not JSON. Only the +// name is read here; the manifest is read whole by the catalogue's parser (inputs.go). +func manifestModule(raw string) string { + var m struct { + Module string `json:"module"` + } + if json.Unmarshal([]byte(raw), &m) != nil { + return "" + } + return m.Module +} + +// heldModules are the values the modules bucket holds, by key (the module's name). A value that is not a module is left +// out and said. +func (ix *Indexer) heldModules(ctx context.Context) (map[string]delivery.Module, error) { + out := map[string]delivery.Module{} + if ix.Modules == nil { + return out, nil + } + keys, err := ix.Modules.Keys(ctx) + if err != nil { + return nil, fmt.Errorf("listing the modules: %w", err) + } + for _, key := range keys { + value, _, found, err := ix.Modules.Get(ctx, key) + if err != nil { + return nil, fmt.Errorf("modules.%s: %w", key, err) + } + if !found { + continue + } + m, err := delivery.DecodeModule(value) + if err != nil { + ix.logf("modules.%s holds what is not a module, left: %v", key, err) + continue + } + out[key] = m + } + return out, nil +} + +// moduleAt is the module whose directory in the repository is dir, as the modules bucket holds it. +func moduleAt(held map[string]delivery.Module, repository, dir string) (string, bool) { + want := pathOf(strings.Trim(dir, "/")) + for name, m := range held { + if m.Path == want && delivery.SameRepository(m.Repository, repository) { + return name, true + } + } + return "", false +} + +// moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else +// the one the modules bucket holds at that directory of the repository, else the directory's last element (the +// catalogue's rule for nearly every manifest). +func moduleOfDir(held map[string]delivery.Module, f delivery.Forge, dir string) string { + if name := manifestModule(f.Manifests[dir]); name != "" { + return name + } + if name, ok := moduleAt(held, f.Repository, dir); ok { + return name + } + return path.Base(strings.Trim(dir, "/")) +} diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index ddd0a51..1f804c4 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -26,7 +26,7 @@ const ( NonePreviewNoModule = "the merge preview names no manifest for the module's directory" // NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold. NonePreviewNoManifest = "the manifest of the merge is not kept" - // NoneNoDirectory is for a module whose directory in its repository the builds do not tell. + // NoneNoDirectory is for a module whose directory in its repository neither the modules bucket nor the preview tells. NoneNoDirectory = "the module's directory in its repository is not known" ) @@ -143,9 +143,27 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool if !found && !stale { continue } + // A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory + // whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6). dirs := map[string]string{} - for _, k := range ix.Builds.Modules(p.Repository) { - dirs[k.Module] = k.Path + if held, err := ix.heldModules(ctx); err == nil { + for name, m := range held { + if delivery.SameRepository(m.Repository, p.Repository) && m.Path != "" { + dirs[name] = m.Path + } + } + } else { + ix.logf("the modules cannot be listed, so no touched module's directory is known: %v", err) + } + for _, m := range p.Modules { + if _, known := dirs[m]; known || p.Fps[m].Manifest == "" { + continue + } + for dir, pm := range pv.Modules { + if pm.Manifest == p.Fps[m].Manifest { + dirs[m] = dir + } + } } fps := maps.Clone(p.Fps) for _, m := range p.Modules { @@ -189,6 +207,9 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive if !ok && dir == "" { pm, ok = pv.Modules[delivery.RootPath] } + if !ok && dir == delivery.RootPath { + pm, ok = pv.Modules[""] + } if !ok { return delivery.CandidateModule{None2: NonePreviewNoModule} } diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 2235218..0f2c781 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -34,7 +34,8 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 { // ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each // repository the source names, each module built from it whose fingerprint was computed at the head the source names is // written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped -// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. +// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The +// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6). func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error { ix.mu.Lock() if rev < ix.source.rev { @@ -46,34 +47,39 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc if ix.Modules == nil { return ix.RecomputeCandidates(ctx) } - repositories := slices.Sorted(maps.Keys(s.Heads)) - for _, repository := range repositories { + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + for _, key := range slices.Sorted(maps.Keys(held)) { + repository := "" + for r := range s.Heads { + if delivery.SameRepository(r, held[key].Repository) { + repository = r + } + } + if repository == "" { + continue + } head := s.Heads[repository] - for _, k := range ix.Builds.Modules(repository) { - key, err := delivery.ModulesKey(k.Module) - if err != nil { - ix.logf("%s: the module %q has no usable key, skipped: %v", repository, k.Module, err) - continue + err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil } - err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { - if !found { - return nil, false, nil - } - cur, err := delivery.DecodeModule(value) - if err != nil { - ix.logf("modules.%s holds what is not a module, left: %v", key, err) - return nil, false, nil - } - if cur.Repository != repository || cur.Commit != head || cur.Src == rev { - return nil, false, nil - } - cur.Src = rev - data, err := delivery.EncodeModule(cur) - return data, false, err - }) + cur, err := delivery.DecodeModule(value) if err != nil { - return fmt.Errorf("modules.%s: %w", key, err) + ix.logf("modules.%s holds what is not a module, left: %v", key, err) + return nil, false, nil } + if cur.Repository != repository || cur.Commit != head || cur.Src == rev { + return nil, false, nil + } + cur.Src = rev + data, err := delivery.EncodeModule(cur) + return data, false, err + }) + if err != nil { + return fmt.Errorf("modules.%s: %w", key, err) } } // A trunk that moved makes the previews made onto the old one out of date. diff --git a/modules/mesh-delivery-indexer/internal/index/source_test.go b/modules/mesh-delivery-indexer/internal/index/source_test.go index 634e216..d7b16a5 100644 --- a/modules/mesh-delivery-indexer/internal/index/source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/source_test.go @@ -15,8 +15,7 @@ func sourceOf(repo, head string) delivery.Source { } func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.T) { // 1344/AC1 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", aInputs)) + ix, mods := modulesIndexer(t) ctx := context.Background() if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { t.Fatal(err) @@ -63,7 +62,7 @@ func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T) if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { t.Fatal(err) } - if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.FP == "" { + if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.Path != "modules/a" { t.Errorf("module a = %+v (%v), want the merge's fingerprint with Src 7", m, ok) } } @@ -75,8 +74,8 @@ func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344 if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { t.Fatal(err) } - if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.FP == "" { - t.Fatalf("module a = %+v (%v), want a fingerprint with Src 0", m, ok) + if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.Path != "modules/a" { + t.Fatalf("module a = %+v (%v), want the merge written with Src 0", m, ok) } // A source revision that names another head of the repository: a merge of the head it does not name keeps Src 0. if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", otherMerge)); err != nil { From af4eaefb14cb9d03ad01cd4dc44245ecbcf1da55 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:09:56 +0200 Subject: [PATCH 12/25] Read the controller's build records only for the builder's own input and to fill a key's missing directory once, so a key written before paths gets one with no merge (issue 1851 I3, issue 1614) Both reads go in R4g with build-agent. A path a key names is never overwritten, and no fingerprint changes. --- .../mesh-delivery-indexer/integration_test.go | 10 +- .../internal/index/builds.go | 131 +++++------------- .../internal/index/live_builds_test.go | 16 +-- .../internal/index/paths_backfill_test.go | 67 +++++++++ .../internal/index/toolchains.go | 43 ++++++ 5 files changed, 159 insertions(+), 108 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/paths_backfill_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 2d49b96..e65d5fd 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -183,8 +183,14 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b // TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged // message on FORGE makes modules. within 5 s, the fingerprint of the merge's tree and the recorded inputs. -func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1: the modules are the merged message's, no build record names them (issue 1851) - js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}) +func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851) + // The bucket holds only build-agent's record, the one the indexer reads until R4g (the builder's own input). + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { + rec := `{"id":"build-1","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}` + if _, err := builds.Put(ctx, delivery.BuildKey("build-agent", "build-1"), []byte(rec)); err != nil { + t.Fatal(err) + } + }) const merge = "89abcdef0123456789abcdef0123456789abcdef" data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}, diff --git a/modules/mesh-delivery-indexer/internal/index/builds.go b/modules/mesh-delivery-indexer/internal/index/builds.go index 72eec2a..5dfd681 100644 --- a/modules/mesh-delivery-indexer/internal/index/builds.go +++ b/modules/mesh-delivery-indexer/internal/index/builds.go @@ -1,31 +1,24 @@ -// Package index keeps the candidate groups (novox/hq ADR 0437 §5) the forge bridge's pull messages make. package index import ( "encoding/json" - "path" - "sort" "strings" "sync" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// Builds holds, from the controller's builds bucket (mesh-controller_builds), which module a directory of a -// repository is: the module of the newest build record naming that repository and path (BuildRecord.Path). Safe for use -// from a watch and from the message handler at once. +// Builds holds the controller's build records (mesh-controller_builds) for the two things the indexer still reads from +// them until release R4g, when build-agent goes (the lead's decision of R4e, 2026-10-22): +// - the Builder input of the builder's own module, which build-agent builds (TrunkOf; novox/hq ADR 0453 clause 5, +// issue 1727); +// - the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no fingerprint. +// +// It lists no module and maps no directory to a module: the indexer's modules are those its own messages and bucket name +// (ADR 0501 clause 6, issue 1851). Safe for use from a watch and from the message handler at once. type Builds struct { mu sync.Mutex records map[string]delivery.BuildRecord // by bucket key - byDir map[string]delivery.BuildRecord // by repository and path; nil when stale - mods map[string]*moduleBuilds // by module; nil when stale -} - -// moduleBuilds is what the builds say of one module: its newest record of any kind, and its newest successful trunk -// build that recorded its inputs. -type moduleBuilds struct { - any delivery.BuildRecord - trunk *delivery.BuildRecord } // NewBuilds is an empty Builds. @@ -44,101 +37,19 @@ func (b *Builds) Put(key string, value []byte) { b.mu.Lock() defer b.mu.Unlock() b.records[key] = r - b.byDir, b.mods = nil, nil } // Delete takes a key out. func (b *Builds) Delete(key string) { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.records[key]; ok { - delete(b.records, key) - b.byDir, b.mods = nil, nil - } -} - -// dirKey names a directory of a repository whichever way the repository is spelled: a build record names it by its clone -// URL, a forge message by "owner/name" (issue 1281). -func dirKey(repository, dir string) string { - return strings.ToLower(delivery.RepositoryName(repository)) + "\x00" + strings.Trim(dir, "/") -} - -// Module is the module a build record of the repository names the directory as its path, if one does. -func (b *Builds) Module(repository, dir string) (string, bool) { - b.mu.Lock() - defer b.mu.Unlock() - if b.byDir == nil { - b.byDir = map[string]delivery.BuildRecord{} - for _, r := range b.records { - k := dirKey(r.Repository, r.Path) - if old, ok := b.byDir[k]; !ok || r.At.After(old.At) || (r.At.Equal(old.At) && r.Module < old.Module) { - b.byDir[k] = r - } - } - } - r, ok := b.byDir[dirKey(repository, dir)] - return r.Module, ok -} - -// ModuleOf is the module of a changed directory: the one a build record of the repository names it the path of; a -// directory no build record names is the directory's last element (the catalogue's rule for nearly every manifest). -func (b *Builds) ModuleOf(repository, dir string) string { - if m, ok := b.Module(repository, dir); ok { - return m - } - return path.Base(strings.Trim(dir, "/")) -} - -// Known is a module the builds know of in a repository: its directory there and its newest successful trunk build -// (delivery.BuildRecord.IsTrunkBuild) if any. The newest key of a module is any state and ref, so it is not what -// finds this (mesh-sdk go/delivery/buildrecord.go NewestKey); the records are. -type Known struct { - Module string - Repository string - Path string - Trunk *delivery.BuildRecord + delete(b.records, key) } func newer(a, b delivery.BuildRecord) bool { return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID) } -// Modules are the modules built from a repository, by name. A module whose newest successful trunk build is of the -// repository belongs to it with that record's path; a module with no such build belongs to the repository of its newest -// record of any kind. -func (b *Builds) Modules(repository string) []Known { - b.mu.Lock() - defer b.mu.Unlock() - if b.mods == nil { - b.mods = map[string]*moduleBuilds{} - for _, r := range b.records { - m := b.mods[r.Module] - if m == nil { - m = &moduleBuilds{any: r} - b.mods[r.Module] = m - } else if newer(r, m.any) { - m.any = r - } - if r.IsTrunkBuild() && (m.trunk == nil || newer(r, *m.trunk)) { - c := r - m.trunk = &c - } - } - } - var out []Known - for name, m := range b.mods { - k := Known{Module: name, Repository: m.any.Repository, Path: strings.Trim(m.any.Path, "/"), Trunk: m.trunk} - if m.trunk != nil { - k.Repository, k.Path = m.trunk.Repository, strings.Trim(m.trunk.Path, "/") - } - if delivery.SameRepository(k.Repository, repository) { - out = append(out, k) - } - } - sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) - return out -} - // TrunkOf is the newest successful trunk build record of a module, in whatever repository it was built from; nil when // there is none (issue 1727). func (b *Builds) TrunkOf(module string) *delivery.BuildRecord { @@ -153,3 +64,27 @@ func (b *Builds) TrunkOf(module string) *delivery.BuildRecord { } return best } + +// PathOf is the directory of a module of the repository as its newest build record of that repository names it, its +// newest successful trunk build's first ("." for the repository's root), and false when no record of the repository +// names the module (issue 1614). +func (b *Builds) PathOf(module, repository string) (string, bool) { + var best *delivery.BuildRecord + trunk := false + b.mu.Lock() + defer b.mu.Unlock() + for _, r := range b.records { + if r.Module != module || !delivery.SameRepository(r.Repository, repository) { + continue + } + t := r.IsTrunkBuild() + if best == nil || (t && !trunk) || (t == trunk && newer(r, *best)) { + c := r + best, trunk = &c, t + } + } + if best == nil { + return "", false + } + return pathOf(strings.Trim(best.Path, "/")), true +} diff --git a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go index a196a54..7aacda2 100644 --- a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go +++ b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go @@ -50,17 +50,17 @@ func liveIndexer(t *testing.T) (*Indexer, *memStore) { return ix, mods } -func TestChangedDirectoryMapsToTheModuleItsLiveRecordNamesByPath(t *testing.T) { // 1281/AC4 +func TestTheLiveRecordsGiveTheDirectoryOfAModuleOfTheirRepository(t *testing.T) { // 1614 (I3: the records name directories, never list modules) ix, _ := liveIndexer(t) - for dir, want := range map[string]string{"modules/zsh": "zsh", "modules/mesh-delivery": "mesh-delivery", "modules/mailu/": "mailu"} { - if got, ok := ix.Builds.Module("novox/mesh-catalog", dir); !ok || got != want { - t.Errorf("Module(novox/mesh-catalog, %q) = %q, %v; want %q", dir, got, ok, want) + for module, want := range map[string]string{"zsh": "modules/zsh", "mesh-delivery": "modules/mesh-delivery", "mailu": "modules/mailu"} { + if got, ok := ix.Builds.PathOf(module, "novox/mesh-catalog"); !ok || got != want { + t.Errorf("PathOf(%s, novox/mesh-catalog) = %q, %v; want %q", module, got, ok, want) } } - if got, ok := ix.Builds.Module("novox/mesh-catalog", "modules/sonarr"); ok { - t.Errorf("Module found %q for a directory of mesh-media-catalog under mesh-catalog", got) + if got, ok := ix.Builds.PathOf("sonarr", "novox/mesh-catalog"); ok { + t.Errorf("PathOf found %q for a module of mesh-media-catalog under mesh-catalog", got) } - if got := ix.Builds.ModuleOf("novox/mesh-media-catalog", "modules/sonarr"); got != "sonarr" { - t.Errorf("ModuleOf(mesh-media-catalog, modules/sonarr) = %q, want sonarr", got) + if got, ok := ix.Builds.PathOf("sonarr", "novox/mesh-media-catalog"); !ok || got != "modules/sonarr" { + t.Errorf("PathOf(sonarr, mesh-media-catalog) = %q, %v; want modules/sonarr", got, ok) } } diff --git a/modules/mesh-delivery-indexer/internal/index/paths_backfill_test.go b/modules/mesh-delivery-indexer/internal/index/paths_backfill_test.go new file mode 100644 index 0000000..48ee007 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/paths_backfill_test.go @@ -0,0 +1,67 @@ +package index + +import ( + "context" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Issue 1614: a modules key written before the indexer learned paths (mesh-tools and node-tools live) gets its directory +// from the module's newest build record without waiting for a merge of its repository, once, changing no fingerprint. The +// read of the controller's build records for it goes in R4g with build-agent (the lead's decision, 2026-10-22). + +func heldValue(t *testing.T, st *memStore, name, path string) { + t.Helper() + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-tools", Commit: "e0fadc43d9dda0bc24bd19bb05b021dd87da8582", + FP2: "src2:" + hex64('2'), Manifest: hex64('9'), Path: path}) + if err != nil { + t.Fatal(err) + } + st.put(name, data) +} + +func hex64(c byte) string { + b := make([]byte, 64) + for i := range b { + b[i] = c + } + return string(b) +} + +func TestAnEmptyPathIsFilledOnceFromTheModulesNewestBuildRecord(t *testing.T) { // 1614 + ix, mods := modulesIndexer(t, + [2]any{delivery.BuildKey("mesh-tools", "build-1"), buildRecord(t, "novox/mesh-tools", "mesh-tools", "", "2026-10-19T09:00:00Z")}, + [2]any{delivery.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "node-tools", "2026-10-19T09:00:00Z")}) + heldValue(t, mods, "mesh-tools", "") + heldValue(t, mods, "node-tools", "") + before, _ := moduleOf(t, mods, "node-tools") + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{"mesh-tools": delivery.RootPath, "node-tools": "node-tools"} { + m, _ := moduleOf(t, mods, name) + if m.Path != want || m.FP2 != before.FP2 || m.Commit != before.Commit { + t.Errorf("%s = %+v, want path %q and nothing else changed", name, m, want) + } + } + writes := mods.writes + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if mods.writes != writes { + t.Errorf("the paths were written again (%d writes, was %d): once only", mods.writes, writes) + } +} + +func TestASetPathIsNeverOverwrittenByABuildRecord(t *testing.T) { // 1614 + ix, mods := modulesIndexer(t, [2]any{delivery.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "elsewhere", "2026-10-19T09:00:00Z")}) + heldValue(t, mods, "node-tools", "node-tools") + writes := mods.writes + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "node-tools"); m.Path != "node-tools" || mods.writes != writes { + t.Errorf("module = %+v after %d writes (was %d), want its path kept and nothing written", m, mods.writes, writes) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index ab29381..5a93dcd 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -134,6 +134,9 @@ func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWit // of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason // (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module) error { + if err := ix.fillPaths(ctx); err != nil { + return err + } if err := ix.recomputeModules(ctx, overlay); err != nil { return err } @@ -420,3 +423,43 @@ func (ix *Indexer) RecomputeForBuild(ctx context.Context) error { } return ix.Recompute(ctx) } + +// fillPaths gives every module whose key names no directory (a key written before the indexer learned paths) the one its +// newest build record names, once, by compare-and-set, changing nothing else (issue 1614): the builder needs it, and a +// rarely merged repository would leave it without one for weeks. A key that names a directory is never written here. +// This read of the controller's build records goes in R4g with build-agent. +func (ix *Indexer) fillPaths(ctx context.Context) error { + if ix.Modules == nil || ix.Builds == nil { + return nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + for _, key := range slices.Sorted(maps.Keys(held)) { + if held[key].Path != "" { + continue + } + dir, ok := ix.Builds.PathOf(key, held[key].Repository) + if !ok { + continue + } + err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil + } + cur, err := delivery.DecodeModule(value) + if err != nil || cur.Path != "" { + return nil, false, nil + } + cur.Path = dir + data, err := delivery.EncodeModule(cur) + return data, false, err + }) + if err != nil { + return fmt.Errorf("modules.%s: %w", key, err) + } + ix.logf("modules.%s names no directory: %q filled from its newest build record (issue 1614)", key, dir) + } + return nil +} From e81ef8bb63dc8b9e71e30cb74febc7bc081f130d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:11:21 +0200 Subject: [PATCH 13/25] =?UTF-8?q?Read=20a=20manifest=20whose=20config=20va?= =?UTF-8?q?lue=20falls=20back=20to=20a=20predefined=20key,=20so=20claude-c?= =?UTF-8?q?ode,=20dunst=20and=20xterm=20get=20a=20new-form=20fingerprint?= =?UTF-8?q?=20(issue=201751,=20ADR=200348=20=C2=A77)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indexer holds no published list of predefined keys; each key a value names is taken as the value declares it. No fingerprint input comes from the list: the gate and the controller still judge the value. --- .../internal/index/inputs.go | 2 +- .../internal/index/predefined.go | 44 ++++ .../internal/index/predefined_test.go | 29 ++ ...catalogue-835bf22b-claude-code.module.json | 249 ++++++++++++++++++ .../catalogue-835bf22b-dunst.module.json | 123 +++++++++ .../catalogue-835bf22b-xterm.module.json | 236 +++++++++++++++++ .../internal/index/toolchains.go | 2 +- 7 files changed, 683 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/predefined.go create mode 100644 modules/mesh-delivery-indexer/internal/index/predefined_test.go create mode 100644 modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-claude-code.module.json create mode 100644 modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-dunst.module.json create mode 100644 modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-xterm.module.json diff --git a/modules/mesh-delivery-indexer/internal/index/inputs.go b/modules/mesh-delivery-indexer/internal/index/inputs.go index 250b78b..2a53587 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs.go @@ -55,7 +55,7 @@ type baseFP func(module string) (fp, none string) // here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve // nothing and have no toolchain. A base is a build.on entry that names a module. func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs { - m, err := catalogue.ParseManifest([]byte(raw)) + m, err := readManifest([]byte(raw)) if err != nil { return moduleInputs{None2: "the manifest cannot be read: " + err.Error()} } diff --git a/modules/mesh-delivery-indexer/internal/index/predefined.go b/modules/mesh-delivery-indexer/internal/index/predefined.go new file mode 100644 index 0000000..7c5905b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/predefined.go @@ -0,0 +1,44 @@ +package index + +import ( + "encoding/json" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// readManifest reads a manifest as the catalogue's parser does, for the inputs of its fingerprint (issue 1751). A config +// value falling back to a predefined key (novox/hq ADR 0348 §7) is judged against the list the mesh published when the +// process holds one; the indexer holds none, since the list is read where a manifest is judged (the merge gate, from the +// tree's root) and by the controller, so each key a value names is taken as the value declares it: of its type, trusted +// when it is. What the list adds is the judging of the value, which the gate and the controller do; no input of a +// fingerprint comes from it. Everything else the parser refuses is refused here too. +func readManifest(raw []byte) (catalogue.Manifest, error) { + list := catalogue.PublishedPredefined() + if list == nil { + list = fallbacksOf(raw) + } + return catalogue.ParseManifestWith(raw, list) +} + +// fallbacksOf is the list of the predefined keys the manifest's config values fall back to, each as the value declares +// it. A manifest that is not JSON gives an empty list, and the parser says what is wrong with it. +func fallbacksOf(raw []byte) catalogue.PredefinedList { + var m struct { + Config map[string]struct { + Type string `json:"type"` + Description string `json:"description"` + Predefined string `json:"predefined"` + Trusted bool `json:"trusted"` + } `json:"config"` + } + list := catalogue.PredefinedList{} + if json.Unmarshal(raw, &m) != nil { + return list + } + for _, d := range m.Config { + if d.Predefined != "" { + list[d.Predefined] = catalogue.PredefinedKey{Type: d.Type, Description: d.Description, Trusted: d.Trusted} + } + } + return list +} diff --git a/modules/mesh-delivery-indexer/internal/index/predefined_test.go b/modules/mesh-delivery-indexer/internal/index/predefined_test.go new file mode 100644 index 0000000..d83f31f --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/predefined_test.go @@ -0,0 +1,29 @@ +package index + +import ( + "strings" + "testing" +) + +// Issue 1751: a manifest whose config value falls back to a predefined key (novox/hq ADR 0348) is read by the indexer, so +// the module gets its new-form inputs. The manifests are claude-code's, dunst's and xterm's module.json from mesh-catalog +// at 835bf22b. +func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) { // 1751 + for _, module := range []string{"claude-code", "dunst", "xterm"} { + raw := fixture(t, "catalogue-835bf22b-"+module) + if !strings.Contains(raw, `"predefined"`) { + t.Fatalf("%s: the fixture names no predefined key", module) + } + in := inputsOf(module, raw, "newtree", nil, func(string) (string, string) { return "src2:" + hex64('1'), "" }) + if in.Kept == nil || strings.Contains(in.None2, "cannot be read") { + t.Errorf("%s: inputs = %+v, want the manifest read and its inputs kept", module, in) + } + } +} + +func TestAPredefinedKeyOfAnotherTypeThanItsValueIsStillRefused(t *testing.T) { // 1751: the read judges what it can + raw := strings.Replace(fixture(t, "catalogue-835bf22b-xterm"), `"predefined": "font-size"`, `"predefined": "font-size", "trusted": "yes"`, 1) + if in := inputsOf("xterm", raw, "newtree", nil, func(string) (string, string) { return "", "none" }); in.Kept != nil { + t.Errorf("inputs = %+v, want a manifest the catalogue refuses still refused", in) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-claude-code.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-claude-code.module.json new file mode 100644 index 0000000..3b62d21 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-claude-code.module.json @@ -0,0 +1,249 @@ +{ + "module": "claude-code", + "version": "1", + "moved": true, + "slug": "agent", + "capabilities": [ + "package-manager" + ], + "requires": [ + "mcp-endpoint", + { + "name": "gitea-read", + "optional": true + } + ], + "binds": { + "mcp-endpoint": "${dir:state}/mcp-endpoint.json" + }, + "secrets": { + "gitea-read": "${dir:state}/gitea-read.secret" + }, + "uses": [ + "operator-channel" + ], + "state": [ + "servers", + "holdings", + "config", + "proposals", + "questions", + "memory", + "waits" + ], + "reads": [ + "claude-licence-manager.bindings" + ], + "tools": [ + "claude_code_status", + "claude_code_render", + "claude_code_guard", + "claude_code_pull", + "claude_code_grant", + "claude_code_add_api_key", + "claude_code_registrations", + "claude_code_memory", + "claude_code_mcp_list", + "claude_code_mcp_register", + "claude_code_mcp_unregister", + "claude_code_skill_list", + "claude_code_skill_register", + "claude_code_skill_unregister", + "claude_code_agent_list", + "claude_code_agent_register", + "claude_code_agent_unregister", + "claude_code_command_list", + "claude_code_command_register", + "claude_code_command_unregister", + "claude_code_hook_list", + "claude_code_hook_register", + "claude_code_hook_unregister", + "claude_code_output_style_list", + "claude_code_output_style_register", + "claude_code_output_style_unregister", + "claude_code_instructions_list", + "claude_code_instructions_register", + "claude_code_instructions_unregister", + "claude_code_instructions_propose", + "claude_code_proposals", + "claude_code_ask", + "claude_code_answer", + "claude_code_wait", + "claude_code_settings_get", + "claude_code_settings_set", + "claude_code_settings_clear", + "claude_code_permission_add", + "claude_code_permission_remove", + "claude_code_config_list", + "claude_code_config_show", + "claude_code_config_status", + "claude_code_config_import", + "claude_code_home_show", + "claude_code_home_remove", + "claude_code_home_removed", + "claude_code_home_restore", + "claude_code_judge", + "claude_code_judged" + ], + "data": { + "own": [ + { + "id": "agent-home", + "path": "${dir:agent-home}", + "class": "valuable", + "why": "the operator's agent's own files: its memory, history and projects" + } + ] + }, + "config": { + "role": { + "type": "string", + "description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.", + "trusted": true, + "default": "" + }, + "mcp_servers": { + "type": "object", + "description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.", + "trusted": true, + "default": {} + }, + "managed_settings": { + "type": "object", + "description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.", + "trusted": true, + "predefined": "agent-managed-settings", + "default": {} + }, + "instructions": { + "type": "object", + "description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "output_styles": { + "type": "object", + "description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "skills": { + "type": "object", + "description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.", + "trusted": true, + "default": {} + }, + "commands": { + "type": "object", + "description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "agents": { + "type": "object", + "description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + } + }, + "resources": [ + { + "id": "package", + "type": "package", + "package": "claude-code" + }, + { + "id": "managed", + "type": "directory", + "path": "/etc/claude-code", + "mode": "0755", + "unknown": "program" + }, + { + "id": "start", + "type": "file", + "path": "/usr/local/bin/claude-agent", + "mode": "0755", + "template": "files/claude-agent" + }, + { + "id": "agent-account-name", + "type": "file", + "path": "/etc/claude-code/agent-account", + "mode": "0644", + "template": "files/agent-account" + }, + { + "id": "agent-account", + "type": "user", + "name": "${machine:agent-account}", + "home": "${machine:agent-home}", + "root": "${machine:agent-root}" + }, + { + "id": "agent-home", + "type": "directory", + "path": "${machine:agent-home}/.claude", + "mode": "0700", + "owner": "${machine:agent-account}", + "unknown": "module" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "${machine:account}", + "place": ".", + "unknown": "program" + }, + { + "id": "facts", + "type": "file", + "path": "${dir:state}/facts.json", + "mode": "0600", + "owner": "${machine:account}", + "template": "files/facts.json" + }, + { + "id": "settings", + "type": "file", + "path": "${dir:state}/settings.json", + "mode": "0600", + "owner": "${machine:account}", + "takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"], + "merge": "json", + "template": "files/settings.json" + } + ], + "items": [ + { + "collection": "shell:zsh:normal", + "template": "files/zsh-normal" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/claude-code", + "binary": "claude-code", + "loads": [ + "claude-code" + ], + "env": { + "MESH_CLAUDE_CODE_STATE": "${dir:state}", + "MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json", + "MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-dunst.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-dunst.module.json new file mode 100644 index 0000000..0c6983b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-dunst.module.json @@ -0,0 +1,123 @@ +{ + "module": "dunst", + "version": "1", + "moved": true, + "capabilities": [ + "package-manager" + ], + "claims": [ + { + "name": "node-notifier", + "scope": "node", + "serves": [ + "send", + "history" + ] + } + ], + "tools": [ + "dunst_pause", + "dunst_resume", + "dunst_close_all", + "dunst_rules", + "dunst_count", + "dunst_reload" + ], + "config": { + "font-size": { + "type": "number", + "description": "The size of a notification's text, in points of Inter: 10 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.", + "minimum": 6, + "maximum": 72, + "predefined": "font-size", + "default": 10 + }, + "width": { + "type": "number", + "description": "The width of a notification, in pixels: 250 fits a title of about forty characters at 10 points, and a larger font needs a wider notification.", + "minimum": 50, + "maximum": 4000, + "default": 250 + } + }, + "resources": [ + { + "id": "package", + "type": "package", + "package": "dunst" + }, + { + "id": "client", + "type": "package", + "package": "libnotify" + }, + { + "id": "configuration-dir", + "type": "directory", + "path": "${machine:account-home}/.config/dunst", + "owner": "${machine:account}", + "mode": "0755", + "unknown": "move-out" + }, + { + "id": "dropins", + "type": "directory", + "path": "${machine:account-home}/.config/dunst/dunstrc.d", + "owner": "${machine:account}", + "mode": "0755", + "unknown": "move-out", + "knows": ["50-slack.conf"] + }, + { + "id": "configuration", + "trusted": false, + "type": "file", + "path": "${machine:account-home}/.config/dunst/dunstrc", + "owner": "${machine:account}", + "mode": "0644", + "template": "files/dunstrc" + }, + { + "id": "notifier", + "type": "service", + "unit": "dunst.service", + "scope": "user", + "user": "${machine:account}", + "reload-on": [ + "configuration" + ] + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/dunst-tools", + "binary": "dunst-tools", + "loads": [ + "dunst-tools" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + }, + "emits": [ + "action-chosen", + "started", + "paused" + ], + "items": [ + { + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "files/i3/50-dunst.conf" + } + ] +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-xterm.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-xterm.module.json new file mode 100644 index 0000000..b65be8e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-xterm.module.json @@ -0,0 +1,236 @@ +{ + "module": "xterm", + "version": "1", + "moved": true, + "capabilities": [ + "package-manager" + ], + "requires": [ + "x11-display" + ], + "claims": [ + { + "name": "node-terminal-emulator", + "scope": "node", + "serves": [ + "open" + ] + } + ], + "tools": [ + "xterm_font", + "xterm_colours" + ], + "config": { + "face": { + "type": "string", + "description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).", + "pattern": "[A-Za-z0-9 ._-]{1,64}", + "default": "JetBrainsMono Nerd Font" + }, + "size": { + "type": "number", + "description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.", + "minimum": 6, + "maximum": 72, + "predefined": "font-size", + "default": 12 + }, + "save-lines": { + "type": "number", + "description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.", + "minimum": 0, + "maximum": 1000000, + "default": 10000 + }, + "term-name": { + "type": "string", + "description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.", + "pattern": "[A-Za-z0-9._-]{1,64}", + "default": "xterm-256color" + }, + "render-font": { + "type": "boolean", + "description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.", + "default": true + }, + "bold-mode": { + "type": "boolean", + "description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.", + "default": false + }, + "always-bold-mode": { + "type": "boolean", + "description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.", + "default": false + }, + "allow-bold-fonts": { + "type": "boolean", + "description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.", + "default": true + }, + "background": { + "type": "string", + "description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#000000" + }, + "foreground": { + "type": "string", + "description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#a8a8a8" + }, + "color0": { + "type": "string", + "description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#000000" + }, + "color1": { + "type": "string", + "description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#9e1828" + }, + "color2": { + "type": "string", + "description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#008800" + }, + "color3": { + "type": "string", + "description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#d2bb4b" + }, + "color4": { + "type": "string", + "description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#414171" + }, + "color5": { + "type": "string", + "description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#963c59" + }, + "color6": { + "type": "string", + "description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#418179" + }, + "color7": { + "type": "string", + "description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#bebebe" + }, + "color8": { + "type": "string", + "description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#666666" + }, + "color9": { + "type": "string", + "description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#bc5766" + }, + "color10": { + "type": "string", + "description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#61a171" + }, + "color11": { + "type": "string", + "description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#e7db52" + }, + "color12": { + "type": "string", + "description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#5085af" + }, + "color13": { + "type": "string", + "description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#a97a99" + }, + "color14": { + "type": "string", + "description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#6ba4a4" + }, + "color15": { + "type": "string", + "description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#ffffff" + }, + "copy-binding": { + "type": "string", + "description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.", + "pattern": "[^:\\\\\\n]+", + "default": "Ctrl Shift ~Alt C" + }, + "paste-binding": { + "type": "string", + "description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.", + "pattern": "[^:\\\\\\n]+", + "default": "Ctrl Shift ~Alt V" + } + }, + "items": [ + { + "collection": "environment", + "name": "terminal", + "attributes": { + "kind": "variable", + "name": "TERMINAL" + }, + "template": "files/env-terminal" + }, + { + "collection": "shell:xresources:normal", + "name": "xresources-normal", + "template": "files/xresources-normal" + } + ], + "resources": [ + { + "id": "package", + "type": "package", + "package": "xterm" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/xterm-tools", + "binary": "xterm-tools", + "loads": [ + "xterm-tools" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 5a93dcd..ffa49c2 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -369,7 +369,7 @@ func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Man if err != nil { return catalogue.Manifest{}, err } - return catalogue.ParseManifest([]byte(kept.JSON)) + return readManifest([]byte(kept.JSON)) } // hasContext is whether an artifact of the manifest reads another repository's tree. From 8ab8c5d04a0899a3d3945cef5c4ec22470ad7c87 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:13:25 +0200 Subject: [PATCH 14/25] =?UTF-8?q?Carry=20each=20named=20path's=20git=20obj?= =?UTF-8?q?ect=20in=20the=20merged=20and=20pull=20messages=20of=20a=20modu?= =?UTF-8?q?le=20whose=20every=20artifact=20names=20its=20paths,=20read=20a?= =?UTF-8?q?t=20its=20manifest's=20commit=20(issue=201758,=20I6,=20ADR=2004?= =?UTF-8?q?99=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asks gitea_commit_trees with paths (the interface agreed with the lead); a forge that cannot say is a retry. Pins mesh-sdk feat/r4e-sdk 9cb4d59. --- .../cmd/mesh-delivery-forge-bridge/call.go | 27 +++++ .../cmd/mesh-delivery-forge-bridge/main.go | 6 +- .../mesh-delivery-forge-bridge/main_test.go | 34 ++++++- modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 2 + .../internal/bridge/bridge.go | 20 +++- .../internal/bridge/named.go | 63 ++++++++++++ .../internal/bridge/named_test.go | 98 +++++++++++++++++++ .../testdata/mesh-sdk-9cb4d59.module.json | 16 +++ 9 files changed, 256 insertions(+), 12 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/named.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go index 9310a73..2c8789f 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go @@ -91,6 +91,33 @@ func treesVia(ctx context.Context, call caller) bridge.Trees { } } +// namedVia asks the forge for the git object of each named path through call: gitea_commit_trees with `paths`, by +// directory (the root spelled ""), the paths relative to it, answering {root, trees, paths} with paths by directory then +// path (the interface agreed in R4e between the lead, the catalogue's leaf lane and this lane; mesh-sdk +// delivery.Forge.Paths; novox/hq ADR 0499 clause 3). An answer without every path asked is an error. +func namedVia(ctx context.Context, call caller) bridge.Named { + return func(owner, repo, sha string, paths map[string][]string) (map[string]map[string]string, error) { + raw, err := call(ctx, commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "paths": paths}) + if err != nil { + return nil, err + } + var a struct { + Paths map[string]map[string]string `json:"paths"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s is not {root, trees, paths}: %w", commitTreesTool, err) + } + for dir, want := range paths { + for _, p := range want { + if a.Paths[dir][p] == "" { + return nil, fmt.Errorf("the answer of %s names no object for %q of %q", commitTreesTool, p, dir) + } + } + } + return a.Paths, nil + } +} + // getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file): // arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text // (client.ts getFile). A path the forge does not have fails with `Gitea API : 404 ` (client.ts request). diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index fc1aaff..595f185 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -115,8 +115,8 @@ type publisher interface { // handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never // redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server // delivers it again (the server's ack wait is the only clock). -func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) { - msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files) +func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, named bridge.Named, logger *log.Logger) { + msg, err := bridge.ConvertNamed(m.Subject(), m.Data(), trees, files, named) switch { case errors.Is(err, bridge.ErrRetry): logger.Printf("%v; heard again in %s", err, retryDelay) @@ -256,7 +256,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger) + handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), namedVia(hctx, call), logger) }) }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index c4ab010..bd0fe4c 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -68,7 +68,7 @@ func body(m map[string]any) []byte { func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" { t.Fatalf("published on %v", pub.subjects) } @@ -84,7 +84,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -99,7 +99,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} - handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) } @@ -121,7 +121,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/ call := func(context.Context, string, any) (json.RawMessage, error) { return nil, errors.New("the tree listing of novox/mesh-delivery was cut") } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), namedVia(context.Background(), call), log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -150,7 +150,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ } return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), namedVia(context.Background(), call), log.New(&logs, "", 0)) if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 { t.Fatalf("calls %v, want the trees once and first", keys) } @@ -230,3 +230,27 @@ func someFiles(_, _, path, _ string) ([]byte, error) { } return nil, bridge.ErrNotFound } + +func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T) { // I6, ADR 0499 clause 3 + var asked map[string]any + call := func(_ context.Context, key string, body any) (json.RawMessage, error) { + if key != commitTreesTool { + t.Errorf("called %s", key) + } + asked = body.(map[string]any) + return json.RawMessage(`{"root":"r0","trees":{"":"r0"},"paths":{"":{"src":"1111111111111111111111111111111111111111","package.json":"3333333333333333333333333333333333333333"}}}`), nil + } + got, err := namedVia(context.Background(), call)("novox", "mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}) + if err != nil || got[""]["src"] != "1111111111111111111111111111111111111111" || got[""]["package.json"] == "" { + t.Fatalf("objects = %v, %v", got, err) + } + if asked["sha"] != "s" || asked["paths"] == nil { + t.Errorf("asked %v, want the commit and the paths", asked) + } + call = func(context.Context, string, any) (json.RawMessage, error) { + return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil + } + if _, err := namedVia(context.Background(), call)("novox", "mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}); err == nil { + t.Error("an answer missing a path asked was taken") + } +} diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 383758b..7239d27 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 3515fb8..bb52af2 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index a6cbc7d..0447e53 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -88,6 +88,13 @@ func unreadable(subject, why string, args ...any) error { // the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the // pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules. func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) { + return ConvertNamed(subject, data, trees, files, nil) +} + +// ConvertNamed is Convert that also asks named for the git object of each path a module's manifest names for every +// artifact (novox/hq ADR 0499 clause 3), at the commit its manifest is read at, and carries them in the message's Paths. +// Nil named asks nothing, and the indexer gives such a module none. +func ConvertNamed(subject string, data []byte, trees Trees, files Files, named Named) (Message, error) { var action string switch subject { case UpdatedSubject: @@ -114,7 +121,7 @@ func Convert(subject string, data []byte, trees Trees, files Files) (Message, er return Message{}, unreadable(subject, "no branch") } if subject == MergedSubject && p.Base == Trunk { - return merged(subject, p, trees, files) + return merged(subject, p, trees, files, named) } f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: action} @@ -125,6 +132,9 @@ func Convert(subject string, data []byte, trees Trees, files Files) (Message, er if f.Manifests, f.Locks, err = inputsOf(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil { return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) } + if f.Paths, err = objectsOf(p.Owner, p.Repo, p.HeadSHA, f.Manifests, named); err != nil { + return Message{}, fmt.Errorf("%w: %s: %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) + } } event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull) if err != nil { @@ -227,7 +237,7 @@ func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests ma } // merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once. -func merged(subject string, p pull, trees Trees, files Files) (Message, error) { +func merged(subject string, p pull, trees Trees, files Files, named Named) (Message, error) { if !sha40.MatchString(p.MergeSHA) { return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA) } @@ -248,12 +258,16 @@ func merged(subject string, p pull, trees Trees, files Files) (Message, error) { if err != nil { return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) } + objects, err := objectsOf(p.Owner, p.Repo, p.MergeSHA, manifests, named) + if err != nil { + return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) + } event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged) if err != nil { return Message{}, unreadable(subject, "%v", err) } body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, - Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}) + Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks, Paths: objects}) if err != nil { return Message{}, unreadable(subject, "%v", err) } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/named.go b/modules/mesh-delivery-forge-bridge/internal/bridge/named.go new file mode 100644 index 0000000..fa31605 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/named.go @@ -0,0 +1,63 @@ +package bridge + +import ( + "encoding/json" + "fmt" + "slices" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Named asks the forge for the git object of each named path of the module directories given, at a commit: by directory +// (the root spelled ""), the paths relative to it; answered by directory, then path (gitea_commit_trees with `paths`, +// mesh-sdk delivery.Forge.Paths; novox/hq ADR 0499 clause 3). A path the forge does not hold fails the call. +type Named func(owner, repo, sha string, paths map[string][]string) (map[string]map[string]string, error) + +// namedPathsOf are, of each manifest that names the paths of every artifact, those paths by directory (catalogue +// Manifest.NamedPaths, ADR 0499 clause 2). Only the build section is read: what the paths are is the catalogue's rule, +// and a manifest the catalogue would refuse for another reason is judged by the indexer, which gives it none. +func namedPathsOf(manifests map[string]string) map[string][]string { + out := map[string][]string{} + for dir, raw := range manifests { + var m struct { + Build *catalogue.Build `json:"build"` + } + if json.Unmarshal([]byte(raw), &m) != nil || m.Build == nil { + continue + } + if paths, every := (catalogue.Manifest{Build: m.Build}).NamedPaths(); every { + out[dir] = paths + } + } + return out +} + +// objectsOf asks named for the objects of the named paths the manifests give at sha, and answers none when no manifest +// names paths. A forge that cannot say is an error. +func objectsOf(owner, repo, sha string, manifests map[string]string, named Named) (map[string]map[string]string, error) { + paths := namedPathsOf(manifests) + if len(paths) == 0 || named == nil { + return nil, nil + } + objects, err := named(owner, repo, sha, paths) + if err != nil { + return nil, fmt.Errorf("the objects of the named paths of %s: %w", dirsOf(paths), err) + } + for dir, want := range paths { + for _, p := range want { + if objects[dir][p] == "" { + return nil, fmt.Errorf("the forge named no object for the named path %q of %q", p, dir) + } + } + } + return objects, nil +} + +func dirsOf(paths map[string][]string) []string { + var dirs []string + for d := range paths { + dirs = append(dirs, d) + } + slices.Sort(dirs) + return dirs +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go new file mode 100644 index 0000000..5cf39c5 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go @@ -0,0 +1,98 @@ +package bridge + +import ( + "os" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ADR 0499 clause 3 (I6): the merged and the pull message carry each named path's git object of a module whose every +// artifact names its paths, read at the same commit as its manifest. The manifest is mesh-sdk's module.json at 9cb4d59 (its +// lib names src, test, package.json, package-lock.json and tsconfig.json). + +type namedCall struct { + owner, repo, sha string + paths map[string][]string +} + +func namedOf(calls *[]namedCall, answer map[string]map[string]string, err error) Named { + return func(owner, repo, sha string, paths map[string][]string) (map[string]map[string]string, error) { + *calls = append(*calls, namedCall{owner, repo, sha, paths}) + return answer, err + } +} + +func sdkFiles(t *testing.T) Files { + t.Helper() + raw, err := os.ReadFile("testdata/mesh-sdk-9cb4d59.module.json") + if err != nil { + t.Fatal(err) + } + return func(owner, repo, path, ref string) ([]byte, error) { + if path == "module.json" { + return raw, nil + } + return nil, ErrNotFound + } +} + +var sdkObjects = map[string]map[string]string{"": {"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222", + "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}} + +func TestAMergedMessageCarriesTheNamedPathsObjectsOfAModuleNamingItsPaths(t *testing.T) { // I6, ADR 0499 clause 3 + var calls []call + var named []namedCall + body := mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}) + msg, err := ConvertNamed(mergedSubject, body, treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), sdkFiles(t), namedOf(&named, sdkObjects, nil)) + if err != nil { + t.Fatal(err) + } + f, err := delivery.DecodeForge(msg.Data) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(f.Paths, sdkObjects) { + t.Errorf("paths = %v, want the forge's objects of lib's named paths", f.Paths) + } + if len(named) != 1 || named[0].sha != merge || named[0].repo != "mesh-sdk" || + strings.Join(named[0].paths[""], ",") != "package-lock.json,package.json,src,test,tsconfig.json" { + t.Errorf("asked %+v, want the root's named paths at the merge commit", named) + } +} + +func TestAMessageOfModulesNamingNoPathsAsksNoObjects(t *testing.T) { // I6 + var calls []call + var named []namedCall + if _, err := ConvertNamed(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), anyFiles, namedOf(&named, nil, nil)); err != nil { + t.Fatal(err) + } + if len(named) != 0 { + t.Errorf("asked %+v, want nothing for modules naming no paths", named) + } +} + +func TestAPullMessageCarriesTheNamedPathsObjectsAtItsHead(t *testing.T) { // I6, ADR 0499 clause 3 + var named []namedCall + body := mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}, "base": "feat/y"}) + msg, err := ConvertNamed(UpdatedSubject, body, nil, sdkFiles(t), namedOf(&named, sdkObjects, nil)) + if err != nil { + t.Fatal(err) + } + f, _ := delivery.DecodeForge(msg.Data) + if !reflect.DeepEqual(f.Paths, sdkObjects) || len(named) != 1 || named[0].sha != head { + t.Errorf("paths = %v, asked %+v; want the objects read at the head", f.Paths, named) + } +} + +func TestAForgeThatCannotSayTheObjectsIsARetry(t *testing.T) { // I6: a message without them would give the module none + var calls []call + var named []namedCall + body := mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}) + _, err := ConvertNamed(mergedSubject, body, treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), sdkFiles(t), namedOf(&named, nil, os.ErrDeadlineExceeded)) + if err == nil || !strings.Contains(err.Error(), "named paths") { + t.Errorf("err = %v, want a retry naming the named paths", err) + } +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json b/modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json new file mode 100644 index 0000000..404139a --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json @@ -0,0 +1,16 @@ +{ + "module": "mesh-sdk", + "version": "1", + "slug": "sdk", + "build": { + "artifacts": [ + { + "name": "lib", + "kind": "package", + "language": "typescript", + "paths": ["src", "test", "package.json", "package-lock.json", "tsconfig.json"] + } + ] + }, + "resources": [] +} From 23daa2c03ca23e4ec92301921e7105cea2a98162 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:14:30 +0200 Subject: [PATCH 15/25] =?UTF-8?q?Take=20a=20module's=20tree=20input=20from?= =?UTF-8?q?=20the=20paths=20its=20artifacts=20name,=20by=20mesh-sdk's=20Na?= =?UTF-8?q?medTree,=20and=20keep=20their=20objects=20and=20each=20artifact?= =?UTF-8?q?'s=20paths,=20so=20a=20module=20outside=20them=20moves=20for=20?= =?UTF-8?q?no=20change=20there=20(issue=201758,=20I7=20and=20the=20indexer?= =?UTF-8?q?'s=20I8,=20ADR=200499=20=C2=A72=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A merged message or a preview without the objects gives the module none, naming the paths. Every other module keeps its directory's tree. Module.ArtifactPaths is filled for the checker (the lead's interface). --- .../internal/index/inputs.go | 48 +++++++++- .../internal/index/modules.go | 4 +- .../internal/index/named_test.go | 87 +++++++++++++++++++ .../internal/index/previews.go | 2 +- .../testdata/sdk-9cb4d59-mesh-sdk.module.json | 16 ++++ 5 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/named_test.go create mode 100644 modules/mesh-delivery-indexer/internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json diff --git a/modules/mesh-delivery-indexer/internal/index/inputs.go b/modules/mesh-delivery-indexer/internal/index/inputs.go index 2a53587..75cee02 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs.go @@ -37,6 +37,8 @@ type moduleInputs struct { Languages map[string]string // None2 is why the module has no fingerprint of the new form. None2 string + // ArtifactPaths are the paths each artifact naming them names, as its manifest names them (delivery.Module.ArtifactPaths). + ArtifactPaths map[string][]string } // baseFP answers a base module's new-form fingerprint, or why it has none. @@ -55,6 +57,15 @@ type baseFP func(module string) (fp, none string) // here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve // nothing and have no toolchain. A base is a build.on entry that names a module. func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs { + return inputsOfNamed(module, raw, tree, locks, nil, base) +} + +// inputsOfNamed is inputsOf with the git object of each path the message names for the module's directory (delivery +// Forge.Paths, PreviewModule.Paths). A module whose every artifact names its paths takes its tree input from them and its +// manifest, by mesh-sdk's one function (delivery.NamedTree), and keeps their objects, so a computation from kept inputs +// gives the merge's value (novox/hq ADR 0499 clause 2); a message that does not carry the object of each such path gives it +// none, naming the paths (clause 3). Every other module keeps its directory's tree, whatever objects are given. +func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, base baseFP) moduleInputs { m, err := readManifest([]byte(raw)) if err != nil { return moduleInputs{None2: "the manifest cannot be read: " + err.Error()} @@ -62,10 +73,39 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo if m.Module != module { return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)} } - if tree == "" { - return moduleInputs{None2: "the merge names no tree for the module's directory"} + var artifactPaths map[string][]string + if m.Build != nil { + for _, a := range m.Build.Artifacts { + if len(a.Paths) > 0 { + if artifactPaths == nil { + artifactPaths = map[string][]string{} + } + artifactPaths[a.Name] = slices.Clone(a.Paths) + } + } } - kept := delivery.KeptInputs2{Tree: tree} + var named map[string]string + if paths, every := m.NamedPaths(); every { + var missing []string + named = map[string]string{} + for _, p := range paths { + if objects[p] == "" { + missing = append(missing, p) + continue + } + named[p] = objects[p] + } + if len(missing) > 0 { + return moduleInputs{None2: "the message carries no git object of the named paths " + strings.Join(missing, ", "), ArtifactPaths: artifactPaths} + } + if tree, err = delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil { + return moduleInputs{None2: err.Error(), ArtifactPaths: artifactPaths} + } + } + if tree == "" { + return moduleInputs{None2: "the merge names no tree for the module's directory", ArtifactPaths: artifactPaths} + } + kept := delivery.KeptInputs2{Tree: tree, Paths: named} contexts := false var missing []string langs := languagesOf(m) @@ -94,7 +134,7 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo } sort.Strings(kept.Bases) sort.Strings(missing) - out := moduleInputs{Kept: &kept} + out := moduleInputs{Kept: &kept, ArtifactPaths: artifactPaths} if len(langs) > 0 { out.Languages = langs } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 1503df6..137b035 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -60,8 +60,8 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" } else { - in := inputsOf(name, raw, f.Trees[dir], f.Locks[dir], ix.baseFP(ctx)) - next.Inputs2, next.None2 = in.Kept, in.None2 + in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], f.Paths[dir], ix.baseFP(ctx)) + next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths } overlay[key] = next } diff --git a/modules/mesh-delivery-indexer/internal/index/named_test.go b/modules/mesh-delivery-indexer/internal/index/named_test.go new file mode 100644 index 0000000..9fafd6b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/named_test.go @@ -0,0 +1,87 @@ +package index + +import ( + "context" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ADR 0499 clauses 2 and 3 (I7): a module whose every artifact names its paths takes its tree input from them, by mesh-sdk's +// one function, and keeps their objects, so a computation from kept inputs gives the merge's value. The manifest is +// mesh-sdk's module.json at 9cb4d59 (its lib names src, test, package.json, package-lock.json and tsconfig.json). + +var sdkObjects = map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222", + "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"} + +func TestAModuleNamingItsPathsTakesItsTreeFromThemAndKeepsTheirObjects(t *testing.T) { // I7, ADR 0499 clause 2 + ix, mods, _ := manifestsIndexer(t) + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, map[string]map[string]string{"": {"package-lock.json": avahiGoSum}}) + f.Paths = map[string]map[string]string{"": sdkObjects} + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "mesh-sdk") + if !ok || m.Inputs2 == nil { + t.Fatalf("module = %+v (%v)", m, ok) + } + want, err := delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects) + if err != nil { + t.Fatal(err) + } + if m.Inputs2.Tree != want || !reflect.DeepEqual(m.Inputs2.Paths, sdkObjects) { + t.Errorf("inputs = %+v, want the named tree %s and the objects kept", m.Inputs2, want) + } + if got := m.ArtifactPaths["lib"]; strings.Join(got, ",") != "src,test,package.json,package-lock.json,tsconfig.json" { + t.Errorf("artifact paths = %v, want lib's paths as its manifest names them", m.ArtifactPaths) + } + // Recomputed from what is kept, with no merge, the module is the same (ADR 0499 clause 2). + before := string(mods.vals["mesh-sdk"]) + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if string(mods.vals["mesh-sdk"]) != before { + t.Errorf("a recompute changed the module: %s, was %s", mods.vals["mesh-sdk"], before) + } +} + +func TestAMessageWithoutTheNamedPathsObjectsGivesTheModuleNoneNamingThem(t *testing.T) { // I7, ADR 0499 clause 3 + ix, mods, _ := manifestsIndexer(t) + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, _ := moduleOf(t, mods, "mesh-sdk") + if m.FP2 != "" || !strings.Contains(m.None2, "named paths") || !strings.Contains(m.None2, "tsconfig.json") { + t.Errorf("module = %+v, want none naming the paths the message did not carry", m) + } +} + +func TestAModuleNamingNoPathsKeepsItsDirectorysTree(t *testing.T) { // I7, ADR 0499 clause 2: every other module is unchanged + in := inputsOfNamed("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, map[string]string{"x": "1111111111111111111111111111111111111111"}, noBases) + if in.Kept == nil || in.Kept.Tree != "avahitree" || in.Kept.Paths != nil { + t.Errorf("inputs = %+v, want the directory's tree and no paths", in.Kept) + } +} + +func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T) { // I8, ADR 0499 clause 2 (the merge preview) + ix, _, _ := manifestsIndexer(t) + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + digest, err := ix.keepManifest(context.Background(), "mesh-sdk", raw) + if err != nil { + t.Fatal(err) + } + named, _ := delivery.NamedTree(digest, sdkObjects) + with := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: named, Manifest: digest, Paths: sdkObjects}}} + without := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: "roottree", Manifest: digest}}} + if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", without); !strings.Contains(got.None2, "named paths") { + t.Errorf("without the objects: %+v, want none naming the named paths", got) + } + if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", with); strings.Contains(got.None2, "named paths") || got.Manifest != digest { + t.Errorf("with the objects: %+v, want them taken (none only for what else is missing)", got) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index 1f804c4..55ec2a6 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -228,7 +228,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive out.None2 = NonePreviewNoManifest return out } - in := inputsOf(m, kept.JSON, pm.Tree, pm.Locks, ix.baseFP(ctx)) + in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, ix.baseFP(ctx)) if in.Kept == nil { out.None2 = in.None2 return out diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json new file mode 100644 index 0000000..404139a --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json @@ -0,0 +1,16 @@ +{ + "module": "mesh-sdk", + "version": "1", + "slug": "sdk", + "build": { + "artifacts": [ + { + "name": "lib", + "kind": "package", + "language": "typescript", + "paths": ["src", "test", "package.json", "package-lock.json", "tsconfig.json"] + } + ] + }, + "resources": [] +} From cc63123b78573fd1d9253de36b55111cd4e59aa0 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:15:09 +0200 Subject: [PATCH 16/25] =?UTF-8?q?Carry=20each=20named=20path's=20object=20?= =?UTF-8?q?from=20a=20fetch's=20module=20trees=20into=20the=20merge=20prev?= =?UTF-8?q?iew,=20so=20the=20indexer=20narrows=20a=20candidate's=20tree=20?= =?UTF-8?q?as=20the=20merge's=20(issue=201758,=20I8,=20ADR=200499=20=C2=A7?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins mesh-sdk feat/r4e-sdk 9cb4d59. --- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 2 ++ .../internal/check/preview.go | 4 ++-- .../internal/check/preview_named_test.go | 23 +++++++++++++++++++ .../internal/check/results.go | 7 ++++-- 5 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 modules/mesh-delivery-checker/internal/check/preview_named_test.go diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 9c911a8..8f008d1 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 3515fb8..bb52af2 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/internal/check/preview.go b/modules/mesh-delivery-checker/internal/check/preview.go index 5d4cf56..bc37e03 100644 --- a/modules/mesh-delivery-checker/internal/check/preview.go +++ b/modules/mesh-delivery-checker/internal/check/preview.go @@ -14,7 +14,7 @@ var digest64 = regexp.MustCompile(`^[0-9a-f]{64}$`) // PreviewOf is what the fetch steps said of a pull request's head merged onto its trunk head, as the value of a previews // key (ADR 0484 clause 3): the merge commit and, for each module directory the fetches named with a manifest, its tree, the -// digest of its manifest and the hashes of its lock files. The fetches of the repository's build modules merge the same +// digest of its manifest, the hashes of its lock files and its named paths' objects (ADR 0499 clause 2). The fetches of the repository's build modules merge the same // trees; the first one's merge commit is kept. A module directory with no tree or no manifest digest is left out, so the // indexer gives its module no fingerprint and says why. A head that does not merge has a preview of the files in conflict // alone, so the indexer says the candidate's modules have no fingerprint because it does not merge (issue 1445). It @@ -43,7 +43,7 @@ func PreviewOf(p Pull, fetched []Fetched) (delivery.Preview, bool) { if t.Tree == "" || !digest64.MatchString(t.Manifest) { continue } - mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks} + mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks, Paths: t.Paths} } } if merge == "" { diff --git a/modules/mesh-delivery-checker/internal/check/preview_named_test.go b/modules/mesh-delivery-checker/internal/check/preview_named_test.go new file mode 100644 index 0000000..fccc274 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/preview_named_test.go @@ -0,0 +1,23 @@ +package check + +import ( + "encoding/json" + "reflect" + "testing" +) + +// I8, ADR 0499 clause 2: the preview carries each named path's object a fetch's module trees answer (mesh-sdk +// buildstep.ModuleTree's paths), so the indexer computes the candidate's narrowed tree from them by the one function. +func TestAPreviewCarriesTheNamedPathsObjectsOfAFetchsModuleTrees(t *testing.T) { + var f Fetched + // As buildstep.ModuleTrees answers a module naming its paths: the named tree and the objects. + raw := `{"workspace":"w","merge":"6666666666666666666666666666666666666666","trees":{".":{"tree":"named:aa","manifest":"` + + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + `","paths":{"src":"1111111111111111111111111111111111111111"}}}}` + if err := json.Unmarshal([]byte(raw), &f); err != nil { + t.Fatal(err) + } + pv, ok := PreviewOf(Pull{Repository: "novox/mesh-sdk", Head: "h", Trunk: "t"}, []Fetched{f}) + if !ok || !reflect.DeepEqual(pv.Modules["."].Paths, map[string]string{"src": "1111111111111111111111111111111111111111"}) || pv.Modules["."].Tree != "named:aa" { + t.Errorf("preview = %+v, want the module's named tree and its objects", pv) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/results.go b/modules/mesh-delivery-checker/internal/check/results.go index 5583592..1c8d2a7 100644 --- a/modules/mesh-delivery-checker/internal/check/results.go +++ b/modules/mesh-delivery-checker/internal/check/results.go @@ -10,12 +10,15 @@ import ( // cmd/mesh-build-node check.go TestResult, fetch.go FetchResult) with the fields the fetch step adds for a merge (ADR 0484 // clause 3). Only the fields the checker reads are named; the rest is ignored. -// MergeTree is what a fetch that merged says of one module directory: the tree sha, the digest of its manifest and the -// hashes of its lock files by name. A bare string is the tree alone. +// MergeTree is what a fetch that merged says of one module directory: the tree sha, the digest of its manifest, the +// hashes of its lock files by name and the objects of its named paths. A bare string is the tree alone. type MergeTree struct { Tree string `json:"tree"` Manifest string `json:"manifest,omitempty"` Locks map[string]string `json:"locks,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's object; Tree is then their named + // tree (mesh-sdk buildstep.ModuleTree, novox/hq ADR 0499 clause 2). + Paths map[string]string `json:"paths,omitempty"` } // UnmarshalJSON takes a string as the tree alone, or an object. From 9bd84941ee5444839dcc1de58c73da6a2520389e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:16:18 +0200 Subject: [PATCH 17/25] Show a fingerprint of the new form against the builder's key of it, since the indexer no longer writes the old form and no build record names the new (issue 1851, I5) The view watches the builder's builds; a value written before R4e keeps today's comparison. Pins mesh-sdk feat/r4e-sdk 9cb4d59. --- .../cmd/mesh-delivery-view/main.go | 2 +- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 2 + .../view/fingerprints_newform_test.go | 53 +++++++++++++++++ .../mesh-delivery-view/internal/view/serve.go | 4 ++ .../mesh-delivery-view/internal/view/view.go | 59 ++++++++++++++++++- 6 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-view/internal/view/fingerprints_newform_test.go diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index 4f3cf6c..4452978 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -136,7 +136,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket} { + for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket} { go follow(ctx, js, bucket, model, logger) } select { diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index b8f5dd8..df79503 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 3686111..8904d85 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -8,6 +8,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/fingerprints_newform_test.go b/modules/mesh-delivery-view/internal/view/fingerprints_newform_test.go new file mode 100644 index 0000000..97fb6d1 --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/fingerprints_newform_test.go @@ -0,0 +1,53 @@ +package view + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// I5 (issue 1851): from R4e the indexer writes the new form alone, and a module's fingerprint is built by the builder's key +// of it; the page shows the indexer's fp2 against that key's state, or the reason it has none. + +const fp2A = "src2:3333333333333333333333333333333333333333333333333333333333333333" + +func builderKey(t *testing.T, module, fp2 string, b delivery.BuilderBuild) Entry { + t.Helper() + key, err := delivery.BuilderBuildKey(module, fp2) + if err != nil { + t.Fatal(err) + } + data, err := delivery.EncodeBuilderBuild(b) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.BuilderBuildsBucket, Key: key, Value: data, At: t0} +} + +func TestAFingerprintOfTheNewFormIsShownAgainstItsBuilderKey(t *testing.T) { // I5 + m := New() + manifest := strings.Repeat("a", 64) + d := "sha256:" + strings.Repeat("2", 64) + fill(m, + modEntry(t, "a", delivery.Module{FP2: fp2A, Manifest: manifest, Repository: "novox/mesh-delivery", Commit: hA}), + builderKey(t, "a", fp2A, delivery.BuilderBuild{State: delivery.BuildDone, By: "shanks.mesh-delivery-builder", Manifest: manifest, + Artifacts: []delivery.BuiltArtifact{{Name: "program", Kind: "bundle", Digest: d, Reference: "artifact-store://a/program/blobs/" + d}}}), + modEntry(t, "f", delivery.Module{FP2: fp2A, Manifest: manifest, Repository: "novox/mesh-delivery", Commit: hA}), + builderKey(t, "f", fp2A, delivery.BuilderBuild{State: delivery.BuildFailed, By: "shanks.mesh-delivery-builder", Manifest: manifest, Error: "not stored: program: 507"}), + modEntry(t, "n", delivery.Module{FP2: fp2A, Manifest: manifest, Repository: "novox/mesh-delivery", Commit: hA}), + modEntry(t, "r", delivery.Module{None2: "toolchain not yet reported", Repository: "novox/mesh-delivery", Commit: hA})) + s := m.Snapshot() + if r := fpRow(t, s, "a"); r.State != FPBuilt || r.Indexer != fp2A { + t.Errorf("a = %+v, want built with the indexer's fp2", r) + } + if r := fpRow(t, s, "f"); r.State != FPBuildFailed || !strings.Contains(r.Reason, "not stored") { + t.Errorf("f = %+v, want build failed with its error", r) + } + if r := fpRow(t, s, "n"); r.State != FPNoKey || r.Indexer != fp2A { + t.Errorf("n = %+v, want no key", r) + } + if r := fpRow(t, s, "r"); r.State != FPNone || r.Reason != "toolchain not yet reported" { + t.Errorf("r = %+v, want none with the new form's reason", r) + } +} diff --git a/modules/mesh-delivery-view/internal/view/serve.go b/modules/mesh-delivery-view/internal/view/serve.go index 3678c6c..d36b09d 100644 --- a/modules/mesh-delivery-view/internal/view/serve.go +++ b/modules/mesh-delivery-view/internal/view/serve.go @@ -146,6 +146,10 @@ func fingerprintText(s Snapshot) [][]string { what = f.Reason case FPNoBuild: what = "indexer " + f.Indexer + "; no successful trunk build recorded a fingerprint" + case FPBuildFailed, FPBuilding: + what = "indexer " + f.Indexer + "; builder: " + f.Reason + case FPNoKey: + what = "indexer " + f.Indexer + "; the builder holds no key of it" } out = append(out, []string{f.Module, f.State, what}) } diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index ad38cef..2cebc2e 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -101,6 +101,12 @@ const ( FPDiffers = "differs" FPNone = "none" FPNoBuild = "no build" + // The states of a fingerprint of the new form against the builder's key of it (I5, issue 1851): built (the key is done), + // build failed (Reason is its error), building (queued, claimed or building; Reason says what it waits for) and no key. + FPBuilt = "built" + FPBuildFailed = "build failed" + FPBuilding = "building" + FPNoKey = "no key" ) // FingerprintRow compares the indexer's fingerprint of a module at a trunk merge with the source fingerprint its newest @@ -147,6 +153,7 @@ type Model struct { builds map[string]buildEntry // by .build. groups map[string]groupEntry // the indexer's candidates, by group modules map[string]moduleEntry // the indexer's modules, by module + keys map[string]keyEntry // the builder's build keys, by . asks map[string]askRecord // the gate's asks, by key answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable) newest map[string]string // module -> id @@ -209,6 +216,11 @@ type groupEntry struct { err string } +type keyEntry struct { + value delivery.BuilderBuild + err string +} + type moduleEntry struct { value delivery.Module err string @@ -233,7 +245,7 @@ type nodeEntry struct { // New is an empty model. func New() *Model { - return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, asks: map[string]askRecord{}, answers: map[string]string{}, newest: map[string]string{}, + return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, answers: map[string]string{}, newest: map[string]string{}, haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{}, presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}} } @@ -321,6 +333,17 @@ func (m *Model) Apply(e Entry) { id, _, _ = strings.Cut(a.Warrant, "@") } m.answers[e.Key] = id + case delivery.BuilderBuildsBucket: + if e.Deleted { + delete(m.keys, e.Key) + return + } + b, err := delivery.DecodeBuilderBuild(e.Value) + if err != nil { + m.keys[e.Key] = keyEntry{err: err.Error()} + return + } + m.keys[e.Key] = keyEntry{value: b} case BuildsBucket: module, rest, ok := strings.Cut(e.Key, ".") if !ok { @@ -525,6 +548,11 @@ func (m *Model) fingerprintRows() []FingerprintRow { continue } row := FingerprintRow{Module: name, Repository: mod.value.Repository, Commit: mod.value.Commit} + if mod.value.FP2 != "" || mod.value.None2 != "" { + // The new form (R4e): the indexer's fp2 against the builder's key of it. + out = append(out, m.newFormRow(name, row, mod.value)) + continue + } b, hasBuild := newest[name] switch { case mod.value.None != "": @@ -542,6 +570,35 @@ func (m *Model) fingerprintRows() []FingerprintRow { return out } +// newFormRow is a module's row of the new form: none with its reason, or its fp2 against the state of the builder's key +// of it (I5, issue 1851; the build records name no fingerprint of the new form). +func (m *Model) newFormRow(name string, row FingerprintRow, v delivery.Module) FingerprintRow { + if v.FP2 == "" { + row.State, row.Reason = FPNone, v.None2 + return row + } + row.Indexer = v.FP2 + key, err := delivery.BuilderBuildKey(name, v.FP2) + if err != nil { + row.State, row.Reason = FPNoKey, err.Error() + return row + } + k, ok := m.keys[key] + switch { + case !ok: + row.State = FPNoKey + case k.err != "": + row.State, row.Reason = FPBuildFailed, "the key cannot be read: "+k.err + case k.value.State == delivery.BuildDone: + row.State, row.Build = FPBuilt, v.FP2 + case k.value.State == delivery.BuildFailed: + row.State, row.Reason = FPBuildFailed, k.value.Error + default: + row.State, row.Reason = FPBuilding, strings.TrimSpace(k.value.State+" "+k.value.Waits) + } + return row +} + // count is the number of items of a carried JSON array or object, and 0 for anything else. func count(raw json.RawMessage) int { var list []json.RawMessage From 9a3f38629fb929ca916bea102ea32aee9f4899de Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:16:54 +0200 Subject: [PATCH 18/25] Prove on a real bus that a merge of mesh-sdk changing only what its lib does not name leaves its tree input and new form unchanged (issue 1758, I9, ADR 0499 check 6 in miniature) --- .../mesh-delivery-indexer/integration_test.go | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index e65d5fd..448e24c 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "time" @@ -434,3 +435,46 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc t.Errorf("none2 = %q within 5 s of the preview, want %q (computed from the preview)", got, delivery.ToolchainNoReport) } } + +// TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual: two merged messages of mesh-sdk (its module.json at 9cb4d59, +// whose lib names its paths) differ in the root tree, as a change to go/ alone makes them, and carry the same objects of the +// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, and its new form +// (fingerprint or reason) does not move (issue 1758, I9, ADR 0499 check clause 6 in miniature). +func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) { + manifest, err := os.ReadFile("../../internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json") + if err != nil { + t.Fatal(err) + } + objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222", + "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"} + js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}) + merged := func(merge, root string) delivery.Module { + data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70, + Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root}, + Manifests: map[string]string{"": string(manifest)}, Paths: map[string]map[string]string{"": objects}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.merged", data); err != nil { + t.Fatal(err) + } + var m delivery.Module + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := modules.Get(ctx, "mesh-sdk"); err == nil { + if m, err = delivery.DecodeModule(e.Value()); err == nil && m.Commit == merge { + return m + } + } + } + t.Fatalf("modules.mesh-sdk = %+v within 5 s, want the merge %s", m, merge) + return m + } + first := merged("89abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + second := merged("99abcdef0123456789abcdef0123456789abcdef", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + if first.Inputs2 == nil || second.Inputs2 == nil || first.Inputs2.Tree != second.Inputs2.Tree || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) { + t.Fatalf("kept trees %+v and %+v, want one named tree", first.Inputs2, second.Inputs2) + } + if first.FP2 != second.FP2 || first.None2 != second.None2 { + t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2) + } +} From 4bb04a47f9d05073379d4b64fb46d058566a367a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:18:59 +0200 Subject: [PATCH 19/25] =?UTF-8?q?Build=20each=20fingerprint=20on=20one=20n?= =?UTF-8?q?ode=20and=20drop=20the=20second-node=20retake,=20as=20ADR=20047?= =?UTF-8?q?8=20=C2=A74=20has=20it=20from=20R4e=20(issue=201587,=20B9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A key done on one node before R4e is not built again for a second; a key failed for differing digests stays as it is. --- .../mesh-delivery-builder/integration_test.go | 55 ++++-------- .../cmd/mesh-delivery-builder/main.go | 3 - .../internal/build/candidates_test.go | 6 +- .../internal/build/claim.go | 31 +------ .../internal/build/followups_test.go | 37 -------- .../internal/build/r4e_test.go | 28 ++++++ .../internal/build/round4_test.go | 90 ++----------------- .../internal/build/run.go | 82 +++-------------- .../internal/build/run_test.go | 88 ++++-------------- 9 files changed, 84 insertions(+), 336 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 3dcd257..cca65e1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -56,10 +56,10 @@ type fakeBuildModules struct { failBuild atomic.Bool // refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2). refusePush atomic.Bool - mu sync.Mutex - calls map[string]int // by `.` - fetches []map[string]any - builds []map[string]any // the arguments of every build call + mu sync.Mutex + calls map[string]int // by `.` + fetches []map[string]any + builds []map[string]any // the arguments of every build call } func (f *fakeBuildModules) count(tool string) int { @@ -121,7 +121,7 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { merge = `,"merge":"` + strings.Repeat("9", 40) + `"` } step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"`+merge+`}`) - case f.failBuild.Load() && node == "n1": + case f.failBuild.Load() && tool != "build_image_copy": // every node's build fails: one node builds a fingerprint step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"]) case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4) d := digestOf(fmt.Sprint(args["artifact"])) @@ -374,7 +374,7 @@ func isState(state string) func(delivery.BuilderBuild) bool { // TestBuilderMakesADoneKeyWithTheDigests: modules.netcheck (a Go and a TypeScript bundle) and its manifest are in the // indexer's buckets, and two nodes report each build module's toolchain; the program writes builds.netcheck. done, -// the digests of both bundles, both build modules' toolchains, after a fetch and a build on each of the two nodes. +// the digests of both bundles, both build modules' toolchains, after a fetch and a build on one node (ADR 0478 clause 4). func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 b := startBed(t) digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") @@ -391,9 +391,9 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" { t.Errorf("toolchains = %+v", v.Toolchains) } - for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 2, "mesh-build-go.build_go_build": 2, "mesh-build-node.build_node_fetch": 2, "mesh-build-node.build_node_build": 2} { + for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} { if got := b.fake.count(tool); got != want { - t.Errorf("%s called %d times, want %d (once on each of two nodes)", tool, got, want) + t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want) } } } @@ -410,8 +410,8 @@ func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // if len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files") { t.Fatalf("build = %+v, want the archive's digest", v) } - if n := b.fake.count("mesh-build-go.build_go_build"); n < 2 || n%2 != 0 { // each instance that built, on two nodes - t.Errorf("build_go_build was called %d times, want two for each instance that built", n) + if n := b.fake.count("mesh-build-go.build_go_build"); n < 1 { // each instance that built, on one node + t.Errorf("build_go_build was called %d times, want one for each instance that built", n) } } @@ -423,32 +423,7 @@ func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) { b.start(t, "n1", nil) v := b.await(t, "ca-trust", isState(delivery.BuildDone)) if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") { - t.Errorf("build = %+v, want done on two nodes naming no toolchain", v) - } -} - -// TestAKeyBuiltOnOneNodeIsBuiltAgainWhenASecondNodeComesOnline: n2 is offline, so the key is done on n1 alone and marked; -// n2 coming online makes the builder build it again on both nodes, as a new revision (ADR 0478 clause 4). -func TestAKeyBuiltOnOneNodeIsBuiltAgainWhenASecondNodeComesOnline(t *testing.T) { - b := startBed(t) - b.setOnline(t, "n2", false) - b.archive(t, "ca-trust", "modules/ca-trust", commit) - b.start(t, "n1", nil) - one := b.await(t, "ca-trust", isState(delivery.BuildDone)) - if !strings.HasPrefix(one.Step, "one node (") { - t.Fatalf("build = %+v, want it marked as built on one node", one) - } - key := mustKey(t, "ca-trust") - e, err := b.builds.Get(b.ctx, key) - if err != nil { - t.Fatal(err) - } - b.setOnline(t, "n2", true) - two := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool { - return v.State == delivery.BuildDone && !strings.HasPrefix(v.Step, "one node (") - }) - if e2, _ := b.builds.Get(b.ctx, key); e2.Revision() <= e.Revision() || !strings.Contains(two.Step, "n1 and n2") { - t.Errorf("build = %+v at revision %d (was %d), want a new revision built on n1 and n2", two, e2.Revision(), e.Revision()) + t.Errorf("build = %+v, want done naming no toolchain", v) } } @@ -459,7 +434,7 @@ func TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed(t *testing.T) { b.archive(t, "ca-trust", "modules/ca-trust", commit) b.start(t, "n1", nil) v := b.await(t, "ca-trust", isState(delivery.BuildDone)) - if !strings.Contains(v.Step, "on one node, n1") { + if !strings.Contains(v.Step, "on n1") { t.Errorf("build = %+v, want it built on n1 alone", v) } } @@ -473,7 +448,7 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFing b.archive(t, "ca-trust", "modules/ca-trust", commit) b.start(t, "n1", nil) v := b.await(t, "ca-trust", isState(delivery.BuildFailed)) - if !strings.Contains(v.Error, "n1") || !strings.Contains(v.Error, "boom") { + if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") { t.Errorf("build = %+v, want failed naming the node and the step's reason", v) } key, _ := delivery.BuilderBuildKey("ca-trust", fp) @@ -510,8 +485,8 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil { t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2") } - if n := b.fake.count("mesh-build-go.build_go_build"); n != 2 { - t.Errorf("build_go_build was called %d times, want 2 (the new fingerprint on two nodes only)", n) + if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 { + t.Errorf("build_go_build was called %d times, want 1 (the new fingerprint on one node only)", n) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index f2b61e6..40b6e29 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -35,8 +35,6 @@ const ( // defaultStepBound is how long the builder waits for one step of a build module (a fetch downloads the Go modules, a // build compiles). defaultStepBound = 30 * time.Minute - // nodesPerFingerprint is how many build nodes build each fingerprint until release R4e (ADR 0478 clause 4). - nodesPerFingerprint = 2 ) func main() { @@ -178,7 +176,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Health: have, Places: &buildrun.Places{Capacity: capacity}, Manifests: manifestReader{manifests}, - Nodes: nodesPerFingerprint, Logf: logger.Printf, } // The toolchains first, so a build that starts from the first modules value finds the nodes. diff --git a/modules/mesh-delivery-builder/internal/build/candidates_test.go b/modules/mesh-delivery-builder/internal/build/candidates_test.go index 7c26902..97ff67a 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates_test.go +++ b/modules/mesh-delivery-builder/internal/build/candidates_test.go @@ -54,8 +54,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe t.Fatalf("build = %+v, want the candidate's fingerprint built", v) } fetches := r.bm.callsOf("build_go_fetch") - if len(fetches) != 2 { // one on each of two nodes - t.Fatalf("%d fetches, want one per node", len(fetches)) + if len(fetches) != 1 { // one node per fingerprint (ADR 0478 clause 4) + t.Fatalf("%d fetches, want one", len(fetches)) } f := fetches[0].body.(map[string]any) if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" { @@ -67,7 +67,7 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe // Two heads carrying one fingerprint are one build. r.b.Candidate(context.Background(), "feat%2Fy", candidateValue(t, "feat/y", 8, headY)) r.b.Wait() - if n := len(r.bm.callsOf("build_go_fetch")); n != 2 { + if n := len(r.bm.callsOf("build_go_fetch")); n != 1 { t.Errorf("%d fetches after a second head with the same fingerprint, want no more", n) } } diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index 6a6659f..21b1316 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -39,32 +39,11 @@ type Claim struct { Build delivery.BuilderBuild } -// OneNodePrefix opens the Step of a done build that a build module built on one node only, because no second node -// qualified (ADR 0478 clause 4); the build modules that did so follow in parentheses, then ": ". Claim reads it to build -// the key again, as a new revision, once a second node qualifies. -const OneNodePrefix = "one node (" - -// OneNodeModules are the build modules a done build built on one node only, or none when it is not such a build. -func OneNodeModules(b delivery.BuilderBuild) []string { - if b.State != delivery.BuildDone || !strings.HasPrefix(b.Step, OneNodePrefix) { - return nil - } - list, _, ok := strings.Cut(strings.TrimPrefix(b.Step, OneNodePrefix), "): ") - if !ok || list == "" { - return nil - } - return strings.Split(list, ", ") -} - // ClaimOptions say what the builder knows about the moment of a claim. type ClaimOptions struct { // TakeOver takes a key found building whoever holds it: the builder has just started and looks at the key for the first // time, and a build left building by a builder that is gone, on this node or another, is nobody's (ADR 0478 clause 6). TakeOver bool - // MoreNodes says whether one of these build modules, which built a done key on one node only, has a second node that - // qualifies now and the key may be built again (it is not held back after a failed second build, issue 1664). Nil means - // none does. - MoreNodes func(modules []string) bool // ImageBuilder says whether a node runs a healthy image build module now: a key failed for NoImageBuilderPrefix is taken // again only then. Nil means none does. ImageBuilder func() bool @@ -135,9 +114,10 @@ func Unreferenced(b delivery.BuilderBuild) bool { // - no key: it is created building, by this builder; a create that finds the key made meanwhile gets nothing; // - a done key is not taken again (the same source is never built twice), except one whose bundle, archive or upstream // copy names no reference (Unreferenced, a key done before R4e: ADR 0498 clause 8), which is claimed building as a new -// revision and built whole, and one built on one node only (OneNodePrefix) -// when a second node qualifies now (ClaimOptions.MoreNodes): that is taken as it stands, with the done value, and its -// builder writes the new revision when the second build ends (ADR 0478 clause 4); +// revision and built whole, and one whose image or package was built unpublished as a candidate's once the trunk +// names it (ClaimOptions.Trunk): that is taken as it stands, with the done value, and its builder writes the new +// revision when the publishing build ends (ADR 0498 clause 6). From R4e one node builds each fingerprint (ADR 0478 +// clause 4), so a key built on one node is never built again for a second; // - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no // builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478 // clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4), @@ -169,9 +149,6 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string if Unreferenced(cur) { return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) } - if modules := OneNodeModules(cur); len(modules) > 0 && opt.MoreNodes != nil && opt.MoreNodes(modules) { - return &Claim{Key: key, Rev: rev, Build: cur}, nil - } if opt.Trunk && len(publishable(cur, opt.ImageBuilder)) > 0 { return &Claim{Key: key, Rev: rev, Build: cur}, nil } diff --git a/modules/mesh-delivery-builder/internal/build/followups_test.go b/modules/mesh-delivery-builder/internal/build/followups_test.go index 6128d0c..ce0efd6 100644 --- a/modules/mesh-delivery-builder/internal/build/followups_test.go +++ b/modules/mesh-delivery-builder/internal/build/followups_test.go @@ -8,43 +8,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// 1664: a done key built on one node is not built again, over and over, while the second node's build keeps failing. -func TestASecondNodeBuildThatFailsIsNotRetriedUntilTheNodesChange(t *testing.T) { // issue 1664 - r := newRig(t, goBundle) - r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies - first := r.consider(t, "modules/m") - r.bm.fails["ace/build_go_fetch"] = "ace is out of disk" - r.pres.Put("ace", presenceValue(t, true), false) - r.b.NodesChanged(context.Background()) // ace came online: one more try - r.b.Wait() - tries := len(r.bm.callsOf("build_go_fetch")) - if tries < 2 { - t.Fatalf("%d fetches: the second node was never tried", tries) - } - if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) { - t.Fatalf("the done key changed: %+v", v) - } - // Nothing about the nodes changed: a modules value, a Reconsider and a start of looking again build nothing. - for i := 0; i < 3; i++ { - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Reconsider(context.Background()) - r.b.Wait() - } - if n := len(r.bm.callsOf("build_go_fetch")); n != tries { - t.Errorf("%d fetches after looking again, was %d: the failed second build was retried in a loop", n, tries) - } - // A node changing (ace goes and comes back) is a reason to try again, and ace now builds. - delete(r.bm.fails, "ace/build_go_fetch") - r.pres.Put("ace", presenceValue(t, false), false) - r.b.NodesChanged(context.Background()) - r.pres.Put("ace", presenceValue(t, true), false) - r.b.NodesChanged(context.Background()) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") { - t.Errorf("build = %+v, want it built on both nodes once ace could", v) - } -} - // 1660: at start the builder closes a key left building for a fingerprint the module no longer has. func TestAKeyLeftBuildingForAFingerprintTheModuleNoLongerHasIsClosedAtStart(t *testing.T) { // issue 1660 r := newRig(t, goBundle) diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index 9a03c7d..2998250 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -332,3 +332,31 @@ func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *test } func hex40(c byte) string { return strings.Repeat(string(c), 40) } + +func TestOneNodeBuildsEachFingerprint(t *testing.T) { // B9, ADR 0478 clause 4: "From R4e one node builds each fingerprint" + r := newRig(t, goBundle, archiveA, tsBundle) // ace and novox both qualify for both build modules + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone || strings.Contains(v.Step, " and ") || strings.HasPrefix(v.Step, "one node (") { + t.Fatalf("build = %+v, want done on one node of each build module, unmarked", v) + } + if f, b := len(r.bm.callsOf("build_go_fetch")), len(r.bm.callsOf("build_go_build")); f != 1 || b != 2 { + t.Errorf("go fetches %d, builds %d; want one fetch and one build of each artifact, on one node", f, b) + } + if f := len(r.bm.callsOf("build_node_fetch")); f != 1 { + t.Errorf("node fetches %d, want one", f) + } +} + +func TestAKeyDoneOnOneNodeIsNeverBuiltAgainForASecondNode(t *testing.T) { // B9: the second-node retake is gone + r := newRig(t, goBundle) + putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: manifestD, + Step: "one node (mesh-build-go): built mesh-build-go on one node, novox", + Artifacts: []delivery.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/tool/blobs/" + digest}}}) + writes := r.st.writes + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.NodesChanged(context.Background()) + r.b.Wait() + if r.st.writes != writes || len(r.bm.calls) != 0 { + t.Errorf("a key done on one node was built again (%d writes, was %d; %d calls)", r.st.writes, writes, len(r.bm.calls)) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/round4_test.go b/modules/mesh-delivery-builder/internal/build/round4_test.go index 24f962f..f2bcc51 100644 --- a/modules/mesh-delivery-builder/internal/build/round4_test.go +++ b/modules/mesh-delivery-builder/internal/build/round4_test.go @@ -34,8 +34,8 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || len(v.Toolchains) != 0 { t.Fatalf("build = %+v, want done with the archive and no toolchain named", v) } - if n := len(r.bm.callsOf("build_go_build")); n != 2 { - t.Errorf("%d go builds, want one on each of two nodes", n) + if n := len(r.bm.callsOf("build_go_build")); n != 1 { + t.Errorf("%d go builds, want one, on one node (ADR 0478 clause 4)", n) } } @@ -54,8 +54,8 @@ func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *te r.b.Reconsider(context.Background()) r.b.Wait() v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix) { - t.Errorf("build = %+v, want done on novox, the one node that qualifies, marked as one node", v) + if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") { + t.Errorf("build = %+v, want done on novox, the one node that qualifies", v) } } @@ -70,25 +70,6 @@ func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testi } } -// 2. The digests-differ failure opens with its prefix, and nothing builds it again. -func TestDifferingDigestsFailTheKeyWithThePrefixAndNothingBuildsItAgain(t *testing.T) { // review round 4, 2 - r := newRig(t, goBundle) - r.bm.digests["ace/tool"] = "sha256:" + hex64('9') - v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, "digests differ:") { - t.Fatalf("build = %+v, want a failure opening with exactly %q", v, "digests differ:") - } - calls, writes := len(r.bm.calls), r.st.writes - r.bm.digests = map[string]string{} // the nodes would now agree - r.b.Reconsider(context.Background()) - r.b.Wait() - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Wait() - if len(r.bm.calls) != calls || r.st.writes != writes { - t.Errorf("a failed-for-digests key was built again: %d calls (was %d), %d writes (was %d)", len(r.bm.calls), calls, r.st.writes, writes) - } -} - func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review round 4, 2 st := newMem() seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: buildrun.DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"}) @@ -99,67 +80,6 @@ func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review } } -// 3. A done key built on one node is built again on a second node once one qualifies. -func TestADoneKeyOfOneNodeIsBuiltAgainOnASecondNodeAndTheDigestsCompared(t *testing.T) { // review round 4, 3 - r := newRig(t, goBundle) - r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies - v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+buildrun.GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") { - t.Fatalf("build = %+v, want done on one node, marked", v) - } - if got := OneNodeModules(v); len(got) != 1 || got[0] != buildrun.GoBuildModule { - t.Errorf("OneNodeModules = %v", got) - } - rev, calls := r.st.revs[r.key], len(r.bm.callsOf("build_go_build")) - // Nothing changes while one node qualifies: looking again builds nothing. - r.b.Reconsider(context.Background()) - r.b.Wait() - if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_go_build")) != calls { - t.Fatal("a one-node key was built again with no second node") - } - // ace comes online: the key is built on ace and novox, as a new revision. - r.pres.Put("ace", presenceValue(t, true), false) - r.b.Reconsider(context.Background()) - r.b.Wait() - v = valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") { - t.Errorf("build = %+v, want done on ace and novox", v) - } - if r.st.revs[r.key] <= rev { - t.Errorf("revision %d, was %d: not a new revision", r.st.revs[r.key], rev) - } - builds := r.bm.callsOf("build_go_build") - if len(builds) != calls+2 || builds[len(builds)-2].node == builds[len(builds)-1].node { - t.Errorf("%d builds (was %d), want one on each of two nodes", len(builds), calls) - } -} - -func TestASecondNodeThatDisagreesFailsTheDoneKeyWithTheDigestsPrefix(t *testing.T) { // review round 4, 3 - r := newRig(t, goBundle) - r.pres.Put("ace", presenceValue(t, false), false) - r.consider(t, "modules/m") - r.bm.digests["ace/tool"] = "sha256:" + hex64('9') - r.pres.Put("ace", presenceValue(t, true), false) - r.b.Reconsider(context.Background()) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, buildrun.DigestsDifferPrefix) { - t.Errorf("build = %+v, want failed with digests differ", v) - } -} - -func TestASecondBuildThatFailsForAnotherReasonLeavesTheDoneKeyStanding(t *testing.T) { // review round 4, 3 - r := newRig(t, goBundle) - r.pres.Put("ace", presenceValue(t, false), false) - first := r.consider(t, "modules/m") - r.bm.fails["ace/build_go_fetch"] = "ace is out of disk" - r.pres.Put("ace", presenceValue(t, true), false) - r.b.Reconsider(context.Background()) - r.b.Wait() - if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) { - t.Errorf("build = %+v, want the first done build to stand: %+v", v, first) - } -} - // 4. A step never waits on a full node while another qualifying node has room. func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4 p := &buildrun.Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}} @@ -252,7 +172,7 @@ func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) r.b.Health = have v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on one node, novox") { + if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") { t.Fatalf("build = %+v, want done on novox alone", v) } for _, c := range r.bm.calls { diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index e725a4f..9c6f23e 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -42,8 +42,6 @@ type Builder struct { } // buildrun.Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node. Places *buildrun.Places - // Nodes is how many build nodes build each fingerprint: two until release R4e (ADR 0478 clause 4). - Nodes int // Limit is how many builds run at once; four when 0. Limit int Logf func(format string, args ...any) @@ -60,10 +58,6 @@ type Builder struct { via map[string]via // by build key: a candidate's fetch (candidates.go) trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint tried map[string]bool // by build key: looked at since this process started - // epoch counts the changes of the nodes (online, healthy, a toolchain) seen since this process started; a done key - // whose second-node build failed is held back until it moves (issue 1664). - epoch uint64 - secondAt map[string]uint64 // by build key: the epoch a second-node build of a one-node key failed at // published are the package versions `@` a done key of this bucket names as a package's reference: a key // waiting for one is built once it is here (ADR 0500 clause 6). published map[string]bool @@ -83,7 +77,7 @@ func (b *Builder) init() { b.sem = make(chan struct{}, limit) b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} - b.secondAt, b.published = map[string]uint64{}, map[string]bool{} + b.published = map[string]bool{} if b.Places == nil { b.Places = &buildrun.Places{} } @@ -120,15 +114,8 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { } // NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or -// stopped being, or a toolchain moved. A done key built on one node whose second-node build failed is tried again only -// after such a change, not at every modules value (issue 1664). -func (b *Builder) NodesChanged(ctx context.Context) { - b.init() - b.mu.Lock() - b.epoch++ - b.mu.Unlock() - b.Reconsider(ctx) -} +// stopped being, or a toolchain moved, so a build that waited for a node may go on. +func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) } // Sweep reads, at start, every key of the bucket: it keeps the package versions done keys name as published (a key // waiting for one is then looked at again, ADR 0500 clause 6), and closes every key left building for a fingerprint its @@ -323,14 +310,8 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { trunk := b.trunkFP[key] first := !b.tried[key] b.tried[key] = true - epoch := b.epoch - failedAt, failedBefore := b.secondAt[key] b.mu.Unlock() - moreNodes := func(modules []string) bool { - // A second build that failed is not tried again until the nodes change (issue 1664). - return (!failedBefore || epoch > failedAt) && b.moreNodes(modules) - } - cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, MoreNodes: moreNodes, ImageBuilder: b.imageBuilderRuns, Trunk: trunk, Published: b.isPublished}) + cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: b.imageBuilderRuns, Trunk: trunk, Published: b.isPublished}) if err != nil { b.logf("%s: %v", module, err) return @@ -338,16 +319,13 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if cl == nil { return } - // A done key taken again builds only what is missing: the build modules that ran on one node, and the image when it was - // built as a candidate's; the other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8). + // A done key taken again builds only what is missing: the image or the package built unpublished as a candidate's; the + // other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8, ADR 0498 clause 6). var prior *delivery.BuilderBuild var only map[string]bool publishing := false if cl.Build.State == delivery.BuildDone { prior, only = &cl.Build, map[string]bool{} - for _, bm := range OneNodeModules(cl.Build) { - only[bm] = true - } if trunk { for _, bm := range publishable(cl.Build, b.imageBuilderRuns) { only[bm] = true @@ -375,14 +353,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { // The trunk's image or package was built only as a candidate's and its push or publish failed: the key says so, so the builds view, the // checker and the failed-key retake see it, and a look does not build the image again for nothing. result.Error = keyError([]string{"publishing what the merged commit built failed: " + result.Error}) - } else if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, buildrun.DigestsDifferPrefix) { - // A done key built again on a second node keeps its done value when the new build fails for another reason than - // the digests: the first build stands, and a later change of the nodes looks again. - b.logf("%s: built again on a second node and failed, the done build stands: %s", cl.Key, result.Error) - b.mu.Lock() - b.secondAt[cl.Key] = epoch - b.mu.Unlock() - return } if err := b.Claimer.Finish(ctx, cl, result); err != nil { b.logf("%s: the build could not be written: %v", cl.Key, err) @@ -444,7 +414,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p repository := delivery.RepositoryName(m.Repository) var lanes []*lane used := map[string]delivery.ToolchainUsed{} - var steps, oneNode []string + var steps []string // Every node chosen holds a reservation until its lane takes the place; a build that ends before its lanes start gives them up. giveUp := func(nodes ...string) { for _, n := range nodes { @@ -477,7 +447,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } nodes = ns } else if plan.NeedsToolchain(bm) { - ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, b.Nodes) + ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, 1) if err != nil { giveUpLanes() return failed("%s", err) @@ -491,23 +461,15 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p nodes = ns } else { // An archive is packed with no toolchain (the fingerprint names none): any node of the build module that qualifies. - ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, b.Nodes) + ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, 1) if err != nil { giveUpLanes() return failed("%s", err) } nodes = ns } - if bm == ImageBuildModule { - steps = append(steps, fmt.Sprintf("%s on %s", bm, nodes[0])) - } else if len(nodes) == 1 { - oneNode = append(oneNode, bm) - steps = append(steps, fmt.Sprintf("%s on one node, %s", bm, nodes[0])) - } else { - sorted := append([]string(nil), nodes...) - sort.Strings(sorted) - steps = append(steps, fmt.Sprintf("%s on %s", bm, strings.Join(sorted, " and "))) - } + // One node builds each fingerprint (ADR 0478 clause 4, from R4e). + steps = append(steps, fmt.Sprintf("%s on %s", bm, nodes[0])) var tasks []Task for _, t := range plan.Tasks { if t.Module == bm { @@ -592,10 +554,6 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } } step := "built " + strings.Join(steps, "; ") - if len(oneNode) > 0 { - // Marked so that Claim builds the key again once a second node qualifies (ADR 0478 clause 4). - step = OneNodePrefix + strings.Join(oneNode, ", ") + "): " + step - } if len(used) == 0 { used = nil } @@ -696,24 +654,6 @@ func (b *Builder) imageBuilderRuns() bool { return false } -// moreNodes is whether any of these build modules has two nodes that qualify now. -func (b *Builder) moreNodes(modules []string) bool { - for _, bm := range modules { - seen := map[string]bool{} - n := 0 - for _, e := range b.Toolchains.Entries(buildrun.LanguageOf[bm]) { - if !seen[e.Node] && b.qualifies(e.Node, bm) { - seen[e.Node] = true - n++ - } - } - if n >= 2 { - return true - } - } - return false -} - // step runs one step of a build module; the lane that runs it holds its node's place. func (b *Builder) step(ctx context.Context, bm, node, tool string, args map[string]any) (json.RawMessage, error) { return b.Runner.Run(ctx, bm, node, toolPrefix[bm]+tool, args) diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index aef2d5a..4a8524c 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -139,7 +139,7 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { } r.b = &Builder{ Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}, - Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf, + Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf, } var err error if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil { @@ -180,7 +180,7 @@ var ( manifestD = mdigest('a') ) -func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 +func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 r := newRig(t, goBundle, archiveA, tsBundle) v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { @@ -200,15 +200,15 @@ func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *test if got := v.Toolchains["typescript"]; got.Version != "tsc5" { t.Errorf("toolchains = %+v", v.Toolchains) } - // Two fetches per build module, one on each node; every build names the module's directory. - if n := len(r.bm.callsOf("build_go_fetch")); n != 2 { - t.Errorf("%d go fetches, want one per node", n) + // One fetch per build module, on one node (ADR 0478 clause 4); every build names the module's directory. + if n := len(r.bm.callsOf("build_go_fetch")); n != 1 { + t.Errorf("%d go fetches, want one", n) } - if n := len(r.bm.callsOf("build_go_build")); n != 4 { // tool and files, each on two nodes - t.Errorf("%d go builds, want 4", n) + if n := len(r.bm.callsOf("build_go_build")); n != 2 { // tool and files + t.Errorf("%d go builds, want 2", n) } - if n := len(r.bm.callsOf("build_node_build")); n != 2 { - t.Errorf("%d node builds, want 2", n) + if n := len(r.bm.callsOf("build_node_build")); n != 1 { + t.Errorf("%d node builds, want 1", n) } c := r.bm.callsOf("build_go_build")[0] if args := c.body.(map[string]any); args["module"] != "modules/m" || !strings.HasPrefix(fmt.Sprint(args["workspace"]), "w-") { @@ -220,21 +220,12 @@ func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *test } } -func TestDifferingDigestsFailTheBuildNamingBothNodes(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 - r := newRig(t, goBundle) - r.bm.digests["ace/tool"] = "sha256:" + hex64('9') - v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "ace") || !strings.Contains(v.Error, "novox") || !strings.Contains(v.Error, digest) || !strings.Contains(v.Error, hex64('9')) { - t.Errorf("build = %+v, want failed naming both nodes and both digests", v) - } -} - -func TestOneQualifyingNodeBuildsOnceAndTheKeySaysSo(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 +func TestOneQualifyingNodeBuildsOnce(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 r := newRig(t, goBundle) r.tools.Put(buildrun.GoBuildModule, "ace.go", nil, true) v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || !strings.Contains(v.Step, "one node") || !strings.Contains(v.Step, "novox") { - t.Errorf("build = %+v, want done with its step saying one node, novox", v) + if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") { + t.Errorf("build = %+v, want done with its step saying novox", v) } if n := len(r.bm.callsOf("build_go_build")); n != 1 { t.Errorf("%d builds, want one", n) @@ -277,9 +268,6 @@ func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { / if v.Artifacts[0].Reference == "" { t.Errorf("a trunk image has no reference: %+v", v.Artifacts[0]) } - if strings.HasPrefix(v.Step, OneNodePrefix) || len(OneNodeModules(v)) != 0 { - t.Errorf("step = %q, want no one-node mark: an image is never built again on a second node", v.Step) - } if n := len(r.bm.callsOf("build_image_fetch")); n != 1 { t.Errorf("%d image fetches, want one", n) } @@ -300,22 +288,19 @@ func TestAnImageIsBuiltOnlyWhereMeshBuildImageIsHealthy(t *testing.T) { // 1705a } } -func TestABundleAndAnImageOfOneModuleBuildOnTwoNodesAndOne(t *testing.T) { // 1705a +func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a r := newRig(t, goBundle, imageArt) r.b.Health = imageHave(t, "ace", "novox") v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || len(v.Artifacts) != 2 { t.Fatalf("build = %+v", v) } - if n := len(r.bm.callsOf("build_go_build")); n != 2 { - t.Errorf("%d go builds, want 2", n) + if n := len(r.bm.callsOf("build_go_build")); n != 1 { + t.Errorf("%d go builds, want 1 (one node per fingerprint, ADR 0478 clause 4)", n) } if n := len(r.bm.callsOf("build_image_build")); n != 1 { t.Errorf("%d image builds, want 1", n) } - if modules := OneNodeModules(v); len(modules) != 0 { - t.Errorf("one-node modules = %v, want none", modules) - } } func TestWithNoImageNodeTheKeyFailsNamingTheProvisionAndIsNotRetakenUntilANodeReportsTheModule(t *testing.T) { // 1705b, lead's decision @@ -454,7 +439,7 @@ func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t * // whose compare-and-set comes first, and the other's write is refused, never overwriting it. func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7, review round 4 r := newRig(t, goBundle) - other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf} + other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf} value := moduleValue(t, "modules/m") var wg sync.WaitGroup for _, b := range []*Builder{r.b, other} { @@ -465,8 +450,8 @@ func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/A if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(v.Artifacts) != 1 { t.Errorf("build = %+v, want one done key", v) } - if n := len(r.bm.callsOf("build_go_build")); n < 2 || n%2 != 0 { - t.Errorf("%d builds, want two for each instance that built", n) + if n := len(r.bm.callsOf("build_go_build")); n < 1 { + t.Errorf("%d builds, want one for each instance that built", n) } } @@ -603,43 +588,6 @@ func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testi } } -func TestASecondGoNodeDoesNotBuildOrPushTheImageAgain(t *testing.T) { // review r1, mesh-delivery 2; ADR 0487 clause 8 - r := newRig(t, goBundle, imageArt) - r.b.Health = imageHave(t, "ace", "novox") - r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies for Go (and the image) - v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || len(OneNodeModules(v)) != 1 { - t.Fatalf("build = %+v, want done with the Go bundle on one node", v) - } - var image delivery.BuiltArtifact - for _, a := range v.Artifacts { - if a.Kind == "image" { - image = a - } - } - if image.Reference == "" { - t.Fatalf("artifacts = %+v, want a published image", v.Artifacts) - } - r.pres.Put("ace", presenceValue(t, true), false) - r.b.Reconsider(context.Background()) - r.b.Wait() - v = valueOf(t, r.st, r.key) - if len(OneNodeModules(v)) != 0 || len(v.Artifacts) != 2 { - t.Fatalf("build = %+v, want done on two Go nodes", v) - } - if n := len(r.bm.callsOf("build_image_build")); n != 1 { - t.Errorf("%d image builds, want the one of the first build only", n) - } - for _, a := range v.Artifacts { - if a.Kind == "image" && a != image { - t.Errorf("image = %+v, was %+v: the done key was rewritten with another image", a, image) - } - } - if n := len(r.bm.callsOf("build_go_build")); n != 3 { // one, then two - t.Errorf("%d go builds, want 3", n) - } -} - func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5 r := newRig(t, imageArt) r.b.Health = imageHave(t, "ace") From b39090aab9f6f35d3bbb8b435e70d0df9ed68e68 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:20:37 +0200 Subject: [PATCH 20/25] Read manifests with mesh-sdk's ParseManifestForInputs in the indexer and the builder, so claude-code, dunst and xterm are built too (issue 1751) The indexer's own copy goes. Pins mesh-sdk feat/r4e-sdk e76a5a3 in every module of the group. --- .../cmd/mesh-delivery-builder/bus.go | 2 +- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 2 + modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 2 + modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 2 + modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 2 + .../internal/index/inputs.go | 2 +- .../internal/index/predefined.go | 44 ------------------- .../internal/index/toolchains.go | 2 +- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 2 + 14 files changed, 18 insertions(+), 52 deletions(-) delete mode 100644 modules/mesh-delivery-indexer/internal/index/predefined.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go index d92318e..b456fbd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go @@ -155,5 +155,5 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue. if err != nil { return catalogue.Manifest{}, err } - return catalogue.ParseManifest([]byte(kept.JSON)) + return catalogue.ParseManifestForInputs([]byte(kept.JSON)) } diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 10090b3..81deb7b 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index a9c5711..61b8f11 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022154726-a90f523edc75 h1:e/aOJLHa git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022154726-a90f523edc75/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b h1:OjlhfiGr+wKLWxafV+BRjZx3TbBtwRYcm9mgoam/jv8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 8f008d1..761ca6b 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index bb52af2..a3076d3 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b h1:OjlhfiGr+wKLWxafV+BRjZx3TbBtwRYcm9mgoam/jv8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 7239d27..75f71c4 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index bb52af2..a3076d3 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b h1:OjlhfiGr+wKLWxafV+BRjZx3TbBtwRYcm9mgoam/jv8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 1f77135..4ae334c 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index bb52af2..a3076d3 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b h1:OjlhfiGr+wKLWxafV+BRjZx3TbBtwRYcm9mgoam/jv8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/inputs.go b/modules/mesh-delivery-indexer/internal/index/inputs.go index 75cee02..3af599e 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs.go @@ -66,7 +66,7 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo // gives the merge's value (novox/hq ADR 0499 clause 2); a message that does not carry the object of each such path gives it // none, naming the paths (clause 3). Every other module keeps its directory's tree, whatever objects are given. func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, base baseFP) moduleInputs { - m, err := readManifest([]byte(raw)) + m, err := catalogue.ParseManifestForInputs([]byte(raw)) if err != nil { return moduleInputs{None2: "the manifest cannot be read: " + err.Error()} } diff --git a/modules/mesh-delivery-indexer/internal/index/predefined.go b/modules/mesh-delivery-indexer/internal/index/predefined.go deleted file mode 100644 index 7c5905b..0000000 --- a/modules/mesh-delivery-indexer/internal/index/predefined.go +++ /dev/null @@ -1,44 +0,0 @@ -package index - -import ( - "encoding/json" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// readManifest reads a manifest as the catalogue's parser does, for the inputs of its fingerprint (issue 1751). A config -// value falling back to a predefined key (novox/hq ADR 0348 §7) is judged against the list the mesh published when the -// process holds one; the indexer holds none, since the list is read where a manifest is judged (the merge gate, from the -// tree's root) and by the controller, so each key a value names is taken as the value declares it: of its type, trusted -// when it is. What the list adds is the judging of the value, which the gate and the controller do; no input of a -// fingerprint comes from it. Everything else the parser refuses is refused here too. -func readManifest(raw []byte) (catalogue.Manifest, error) { - list := catalogue.PublishedPredefined() - if list == nil { - list = fallbacksOf(raw) - } - return catalogue.ParseManifestWith(raw, list) -} - -// fallbacksOf is the list of the predefined keys the manifest's config values fall back to, each as the value declares -// it. A manifest that is not JSON gives an empty list, and the parser says what is wrong with it. -func fallbacksOf(raw []byte) catalogue.PredefinedList { - var m struct { - Config map[string]struct { - Type string `json:"type"` - Description string `json:"description"` - Predefined string `json:"predefined"` - Trusted bool `json:"trusted"` - } `json:"config"` - } - list := catalogue.PredefinedList{} - if json.Unmarshal(raw, &m) != nil { - return list - } - for _, d := range m.Config { - if d.Predefined != "" { - list[d.Predefined] = catalogue.PredefinedKey{Type: d.Type, Description: d.Description, Trusted: d.Trusted} - } - } - return list -} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index ffa49c2..0257f1b 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -369,7 +369,7 @@ func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Man if err != nil { return catalogue.Manifest{}, err } - return readManifest([]byte(kept.JSON)) + return catalogue.ParseManifestForInputs([]byte(kept.JSON)) } // hasContext is whether an artifact of the manifest reads another repository's tree. diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index df79503..c5899b3 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 8904d85..78dc697 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -10,6 +10,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b h1:OjlhfiGr+wKLWxafV+BRjZx3TbBtwRYcm9mgoam/jv8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From ece328d0cc964e5110e81d3cc4042dd58f7bf3fe Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:20:37 +0200 Subject: [PATCH 21/25] =?UTF-8?q?Give=20a=20candidate's=20module=20its=20n?= =?UTF-8?q?amed=20paths'=20objects=20from=20the=20preview,=20so=20the=20bu?= =?UTF-8?q?ilder's=20fetch=20for=20it=20names=20them=20(issue=201758,=20AD?= =?UTF-8?q?R=200499=20=C2=A74;=20mesh-sdk=20CandidateModule.Paths)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/index/candidate_manifests_test.go | 5 +++-- .../mesh-delivery-indexer/internal/index/candidates.go | 3 ++- .../mesh-delivery-indexer/internal/index/named_test.go | 6 +++++- modules/mesh-delivery-indexer/internal/index/previews.go | 8 ++++++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go index 41c2f1b..a43f4e9 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go @@ -3,6 +3,7 @@ package index import ( "context" "errors" + "reflect" "strings" "testing" @@ -43,7 +44,7 @@ func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t } c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"} - if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || c.Pulls[0].Fps["avahi"] != want { + if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || !reflect.DeepEqual(c.Pulls[0].Fps["avahi"], want) { t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want) } if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) { @@ -55,7 +56,7 @@ func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t if err := ix.Apply(context.Background(), again); err != nil { t.Fatal(err) } - if c2 := candidateOf(t, ix.Store.(*memStore)); c2.Pulls[0].Fps["avahi"] != want || c2.FPSet != c.FPSet { + if c2 := candidateOf(t, ix.Store.(*memStore)); !reflect.DeepEqual(c2.Pulls[0].Fps["avahi"], want) || c2.FPSet != c.FPSet { t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet) } } diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index c322283..d0baf34 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "slices" "strings" "sync" @@ -284,7 +285,7 @@ func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error { first[module] = p continue } - if before.Fps[module] != m { + if !reflect.DeepEqual(before.Fps[module], m) { err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written", ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest) return err diff --git a/modules/mesh-delivery-indexer/internal/index/named_test.go b/modules/mesh-delivery-indexer/internal/index/named_test.go index 9fafd6b..91bec02 100644 --- a/modules/mesh-delivery-indexer/internal/index/named_test.go +++ b/modules/mesh-delivery-indexer/internal/index/named_test.go @@ -81,7 +81,11 @@ func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T) if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", without); !strings.Contains(got.None2, "named paths") { t.Errorf("without the objects: %+v, want none naming the named paths", got) } - if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", with); strings.Contains(got.None2, "named paths") || got.Manifest != digest { + got := ix.candidateModule(context.Background(), "mesh-sdk", ".", with) + if strings.Contains(got.None2, "named paths") || got.Manifest != digest { t.Errorf("with the objects: %+v, want them taken (none only for what else is missing)", got) } + if got.FP2 != "" && !reflect.DeepEqual(got.Paths, sdkObjects) { + t.Errorf("with a fingerprint: %+v, want the named paths' objects carried for the builder's fetch", got) + } } diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index 55ec2a6..5e8b454 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "maps" + "reflect" "slices" "strings" @@ -184,12 +185,12 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool if fps == nil { fps = map[string]delivery.CandidateModule{} } - if fps[m] != next { + if !reflect.DeepEqual(fps[m], next) { fps[m] = next changed = true } } - if changed || !maps.Equal(fps, p.Fps) { + if changed || !maps.EqualFunc(fps, p.Fps, func(a, b delivery.CandidateModule) bool { return reflect.DeepEqual(a, b) }) { p.Fps = fps } } @@ -269,5 +270,8 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive return out } out.FP2 = fp + if len(inputs.Paths) > 0 { + out.Paths = inputs.Paths // the builder's fetch for the candidate names them (ADR 0499 clause 4) + } return out } From 7cc97cd6b60f78b180353199123f6eb5cca64844 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:20:59 +0200 Subject: [PATCH 22/25] Name a candidate's named paths in its fetch, and prove on a real bus that claude-code's manifest is built (issues 1758 and 1751) --- .../mesh-delivery-builder/integration_test.go | 13 + ...catalogue-835bf22b-claude-code.module.json | 249 ++++++++++++++++++ .../internal/build/candidates.go | 4 + .../internal/build/r4e_test.go | 23 ++ 4 files changed, 289 insertions(+) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-835bf22b-claude-code.module.json diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index cca65e1..dcbf3e2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -698,3 +698,16 @@ func TestAPinnedUpstreamIsCopiedByTheImageBuildModule(t *testing.T) { t.Errorf("build_image_fetch called %d times, want none for a copy", n) } } + +// TestAManifestFallingBackToAPredefinedKeyIsBuilt: claude-code's manifest (mesh-catalog 835bf22b) has a config value falling +// back to a predefined key; the builder, which holds no published list of them, reads it and builds the key done (issue +// 1751, mesh-sdk catalogue.ParseManifestForInputs). +func TestAManifestFallingBackToAPredefinedKeyIsBuilt(t *testing.T) { + b := startBed(t) + b.module(t, "claude-code", "catalogue-835bf22b-claude-code.module.json", "modules/claude-code") + b.start(t, "n1", nil) + v := b.await(t, "claude-code", func(v delivery.BuilderBuild) bool { return v.State == delivery.BuildDone || v.State == delivery.BuildFailed }) + if v.State != delivery.BuildDone { + t.Errorf("build = %+v, want done: the manifest is read", v) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-835bf22b-claude-code.module.json b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-835bf22b-claude-code.module.json new file mode 100644 index 0000000..3b62d21 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-835bf22b-claude-code.module.json @@ -0,0 +1,249 @@ +{ + "module": "claude-code", + "version": "1", + "moved": true, + "slug": "agent", + "capabilities": [ + "package-manager" + ], + "requires": [ + "mcp-endpoint", + { + "name": "gitea-read", + "optional": true + } + ], + "binds": { + "mcp-endpoint": "${dir:state}/mcp-endpoint.json" + }, + "secrets": { + "gitea-read": "${dir:state}/gitea-read.secret" + }, + "uses": [ + "operator-channel" + ], + "state": [ + "servers", + "holdings", + "config", + "proposals", + "questions", + "memory", + "waits" + ], + "reads": [ + "claude-licence-manager.bindings" + ], + "tools": [ + "claude_code_status", + "claude_code_render", + "claude_code_guard", + "claude_code_pull", + "claude_code_grant", + "claude_code_add_api_key", + "claude_code_registrations", + "claude_code_memory", + "claude_code_mcp_list", + "claude_code_mcp_register", + "claude_code_mcp_unregister", + "claude_code_skill_list", + "claude_code_skill_register", + "claude_code_skill_unregister", + "claude_code_agent_list", + "claude_code_agent_register", + "claude_code_agent_unregister", + "claude_code_command_list", + "claude_code_command_register", + "claude_code_command_unregister", + "claude_code_hook_list", + "claude_code_hook_register", + "claude_code_hook_unregister", + "claude_code_output_style_list", + "claude_code_output_style_register", + "claude_code_output_style_unregister", + "claude_code_instructions_list", + "claude_code_instructions_register", + "claude_code_instructions_unregister", + "claude_code_instructions_propose", + "claude_code_proposals", + "claude_code_ask", + "claude_code_answer", + "claude_code_wait", + "claude_code_settings_get", + "claude_code_settings_set", + "claude_code_settings_clear", + "claude_code_permission_add", + "claude_code_permission_remove", + "claude_code_config_list", + "claude_code_config_show", + "claude_code_config_status", + "claude_code_config_import", + "claude_code_home_show", + "claude_code_home_remove", + "claude_code_home_removed", + "claude_code_home_restore", + "claude_code_judge", + "claude_code_judged" + ], + "data": { + "own": [ + { + "id": "agent-home", + "path": "${dir:agent-home}", + "class": "valuable", + "why": "the operator's agent's own files: its memory, history and projects" + } + ] + }, + "config": { + "role": { + "type": "string", + "description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.", + "trusted": true, + "default": "" + }, + "mcp_servers": { + "type": "object", + "description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.", + "trusted": true, + "default": {} + }, + "managed_settings": { + "type": "object", + "description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.", + "trusted": true, + "predefined": "agent-managed-settings", + "default": {} + }, + "instructions": { + "type": "object", + "description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "output_styles": { + "type": "object", + "description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "skills": { + "type": "object", + "description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.", + "trusted": true, + "default": {} + }, + "commands": { + "type": "object", + "description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "agents": { + "type": "object", + "description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + } + }, + "resources": [ + { + "id": "package", + "type": "package", + "package": "claude-code" + }, + { + "id": "managed", + "type": "directory", + "path": "/etc/claude-code", + "mode": "0755", + "unknown": "program" + }, + { + "id": "start", + "type": "file", + "path": "/usr/local/bin/claude-agent", + "mode": "0755", + "template": "files/claude-agent" + }, + { + "id": "agent-account-name", + "type": "file", + "path": "/etc/claude-code/agent-account", + "mode": "0644", + "template": "files/agent-account" + }, + { + "id": "agent-account", + "type": "user", + "name": "${machine:agent-account}", + "home": "${machine:agent-home}", + "root": "${machine:agent-root}" + }, + { + "id": "agent-home", + "type": "directory", + "path": "${machine:agent-home}/.claude", + "mode": "0700", + "owner": "${machine:agent-account}", + "unknown": "module" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "${machine:account}", + "place": ".", + "unknown": "program" + }, + { + "id": "facts", + "type": "file", + "path": "${dir:state}/facts.json", + "mode": "0600", + "owner": "${machine:account}", + "template": "files/facts.json" + }, + { + "id": "settings", + "type": "file", + "path": "${dir:state}/settings.json", + "mode": "0600", + "owner": "${machine:account}", + "takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"], + "merge": "json", + "template": "files/settings.json" + } + ], + "items": [ + { + "collection": "shell:zsh:normal", + "template": "files/zsh-normal" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/claude-code", + "binary": "claude-code", + "loads": [ + "claude-code" + ], + "env": { + "MESH_CLAUDE_CODE_STATE": "${dir:state}", + "MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json", + "MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-delivery-builder/internal/build/candidates.go b/modules/mesh-delivery-builder/internal/build/candidates.go index 93e7183..d96df2c 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates.go +++ b/modules/mesh-delivery-builder/internal/build/candidates.go @@ -69,6 +69,10 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) { } m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path, Toolchains: b.agreedFor(ctx, cm.Manifest)} + if len(cm.Paths) > 0 { + // The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4). + m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths} + } b.mu.Lock() if _, onTrunk := b.trunkFP[key]; !onTrunk { b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit} diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index 2998250..1c53597 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -360,3 +360,26 @@ func TestAKeyDoneOnOneNodeIsNeverBuiltAgainForASecondNode(t *testing.T) { // B9: t.Errorf("a key done on one node was built again (%d writes, was %d; %d calls)", r.st.writes, writes, len(r.bm.calls)) } } + +func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // ADR 0499 clause 4, mesh-sdk CandidateModule.Paths + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + trunkModule(t, r) + pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: headX}, Modules: []string{"m"}, + Fps: map[string]delivery.CandidateModule{"m": {FP2: fp1, Manifest: mdigest('a'), Paths: map[string]string{"src": hex40('1'), "package.json": hex40('2')}}}}} + hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head}) + data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}) + if err != nil { + t.Fatal(err) + } + r.b.Candidate(context.Background(), "feat%2Fx", data) + r.b.Wait() + f := r.bm.callsOf("build_node_fetch") + if len(f) == 0 { + t.Fatal("no fetch") + } + req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any)) + if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" { + t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err) + } +} From 12cf0cc1f57d74cae4d6896cf7cb05be6edcd4f0 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:21:04 +0200 Subject: [PATCH 23/25] Format the builder's integration test --- .../cmd/mesh-delivery-builder/integration_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index dcbf3e2..5ae38b7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -706,7 +706,9 @@ func TestAManifestFallingBackToAPredefinedKeyIsBuilt(t *testing.T) { b := startBed(t) b.module(t, "claude-code", "catalogue-835bf22b-claude-code.module.json", "modules/claude-code") b.start(t, "n1", nil) - v := b.await(t, "claude-code", func(v delivery.BuilderBuild) bool { return v.State == delivery.BuildDone || v.State == delivery.BuildFailed }) + v := b.await(t, "claude-code", func(v delivery.BuilderBuild) bool { + return v.State == delivery.BuildDone || v.State == delivery.BuildFailed + }) if v.State != delivery.BuildDone { t.Errorf("build = %+v, want done: the manifest is read", v) } From d801a6460e80dd9474e82c9be93050cd5d71f0e9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:23:29 +0200 Subject: [PATCH 24/25] =?UTF-8?q?Read=20no=20manifest's=20paths=20in=20the?= =?UTF-8?q?=20forge=20bridge:=20the=20indexer,=20the=20one=20reader=20of?= =?UTF-8?q?=20paths=20(ADR=200495=20=C2=A71,=20the=20lead's=20decision=20L?= =?UTF-8?q?8),=20asks=20the=20forge=20for=20their=20objects=20itself=20(is?= =?UTF-8?q?sue=201758,=20I6=20reworked)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Undoes 8ab8c5d's bridge change; the bridge keeps the mesh-sdk pin. --- .../cmd/mesh-delivery-forge-bridge/call.go | 27 ----- .../cmd/mesh-delivery-forge-bridge/main.go | 6 +- .../mesh-delivery-forge-bridge/main_test.go | 34 +------ .../internal/bridge/bridge.go | 20 +--- .../internal/bridge/named.go | 63 ------------ .../internal/bridge/named_test.go | 98 ------------------- .../internal/bridge/paths_test.go | 36 +++++++ 7 files changed, 47 insertions(+), 237 deletions(-) delete mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/named.go delete mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go index 2c8789f..9310a73 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go @@ -91,33 +91,6 @@ func treesVia(ctx context.Context, call caller) bridge.Trees { } } -// namedVia asks the forge for the git object of each named path through call: gitea_commit_trees with `paths`, by -// directory (the root spelled ""), the paths relative to it, answering {root, trees, paths} with paths by directory then -// path (the interface agreed in R4e between the lead, the catalogue's leaf lane and this lane; mesh-sdk -// delivery.Forge.Paths; novox/hq ADR 0499 clause 3). An answer without every path asked is an error. -func namedVia(ctx context.Context, call caller) bridge.Named { - return func(owner, repo, sha string, paths map[string][]string) (map[string]map[string]string, error) { - raw, err := call(ctx, commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "paths": paths}) - if err != nil { - return nil, err - } - var a struct { - Paths map[string]map[string]string `json:"paths"` - } - if err := json.Unmarshal(raw, &a); err != nil { - return nil, fmt.Errorf("the answer of %s is not {root, trees, paths}: %w", commitTreesTool, err) - } - for dir, want := range paths { - for _, p := range want { - if a.Paths[dir][p] == "" { - return nil, fmt.Errorf("the answer of %s names no object for %q of %q", commitTreesTool, p, dir) - } - } - } - return a.Paths, nil - } -} - // getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file): // arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text // (client.ts getFile). A path the forge does not have fails with `Gitea API : 404 ` (client.ts request). diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 595f185..fc1aaff 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -115,8 +115,8 @@ type publisher interface { // handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never // redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server // delivers it again (the server's ack wait is the only clock). -func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, named bridge.Named, logger *log.Logger) { - msg, err := bridge.ConvertNamed(m.Subject(), m.Data(), trees, files, named) +func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) { + msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files) switch { case errors.Is(err, bridge.ErrRetry): logger.Printf("%v; heard again in %s", err, retryDelay) @@ -256,7 +256,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), namedVia(hctx, call), logger) + handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger) }) }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index bd0fe4c..c4ab010 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -68,7 +68,7 @@ func body(m map[string]any) []byte { func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" { t.Fatalf("published on %v", pub.subjects) } @@ -84,7 +84,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -99,7 +99,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} - handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) } @@ -121,7 +121,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/ call := func(context.Context, string, any) (json.RawMessage, error) { return nil, errors.New("the tree listing of novox/mesh-delivery was cut") } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), namedVia(context.Background(), call), log.New(&logs, "", 0)) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -150,7 +150,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ } return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), namedVia(context.Background(), call), log.New(&logs, "", 0)) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 { t.Fatalf("calls %v, want the trees once and first", keys) } @@ -230,27 +230,3 @@ func someFiles(_, _, path, _ string) ([]byte, error) { } return nil, bridge.ErrNotFound } - -func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T) { // I6, ADR 0499 clause 3 - var asked map[string]any - call := func(_ context.Context, key string, body any) (json.RawMessage, error) { - if key != commitTreesTool { - t.Errorf("called %s", key) - } - asked = body.(map[string]any) - return json.RawMessage(`{"root":"r0","trees":{"":"r0"},"paths":{"":{"src":"1111111111111111111111111111111111111111","package.json":"3333333333333333333333333333333333333333"}}}`), nil - } - got, err := namedVia(context.Background(), call)("novox", "mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}) - if err != nil || got[""]["src"] != "1111111111111111111111111111111111111111" || got[""]["package.json"] == "" { - t.Fatalf("objects = %v, %v", got, err) - } - if asked["sha"] != "s" || asked["paths"] == nil { - t.Errorf("asked %v, want the commit and the paths", asked) - } - call = func(context.Context, string, any) (json.RawMessage, error) { - return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil - } - if _, err := namedVia(context.Background(), call)("novox", "mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}); err == nil { - t.Error("an answer missing a path asked was taken") - } -} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index 0447e53..a6cbc7d 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -88,13 +88,6 @@ func unreadable(subject, why string, args ...any) error { // the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the // pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules. func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) { - return ConvertNamed(subject, data, trees, files, nil) -} - -// ConvertNamed is Convert that also asks named for the git object of each path a module's manifest names for every -// artifact (novox/hq ADR 0499 clause 3), at the commit its manifest is read at, and carries them in the message's Paths. -// Nil named asks nothing, and the indexer gives such a module none. -func ConvertNamed(subject string, data []byte, trees Trees, files Files, named Named) (Message, error) { var action string switch subject { case UpdatedSubject: @@ -121,7 +114,7 @@ func ConvertNamed(subject string, data []byte, trees Trees, files Files, named N return Message{}, unreadable(subject, "no branch") } if subject == MergedSubject && p.Base == Trunk { - return merged(subject, p, trees, files, named) + return merged(subject, p, trees, files) } f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: action} @@ -132,9 +125,6 @@ func ConvertNamed(subject string, data []byte, trees Trees, files Files, named N if f.Manifests, f.Locks, err = inputsOf(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil { return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) } - if f.Paths, err = objectsOf(p.Owner, p.Repo, p.HeadSHA, f.Manifests, named); err != nil { - return Message{}, fmt.Errorf("%w: %s: %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) - } } event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull) if err != nil { @@ -237,7 +227,7 @@ func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests ma } // merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once. -func merged(subject string, p pull, trees Trees, files Files, named Named) (Message, error) { +func merged(subject string, p pull, trees Trees, files Files) (Message, error) { if !sha40.MatchString(p.MergeSHA) { return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA) } @@ -258,16 +248,12 @@ func merged(subject string, p pull, trees Trees, files Files, named Named) (Mess if err != nil { return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) } - objects, err := objectsOf(p.Owner, p.Repo, p.MergeSHA, manifests, named) - if err != nil { - return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) - } event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged) if err != nil { return Message{}, unreadable(subject, "%v", err) } body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, - Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks, Paths: objects}) + Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}) if err != nil { return Message{}, unreadable(subject, "%v", err) } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/named.go b/modules/mesh-delivery-forge-bridge/internal/bridge/named.go deleted file mode 100644 index fa31605..0000000 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/named.go +++ /dev/null @@ -1,63 +0,0 @@ -package bridge - -import ( - "encoding/json" - "fmt" - "slices" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// Named asks the forge for the git object of each named path of the module directories given, at a commit: by directory -// (the root spelled ""), the paths relative to it; answered by directory, then path (gitea_commit_trees with `paths`, -// mesh-sdk delivery.Forge.Paths; novox/hq ADR 0499 clause 3). A path the forge does not hold fails the call. -type Named func(owner, repo, sha string, paths map[string][]string) (map[string]map[string]string, error) - -// namedPathsOf are, of each manifest that names the paths of every artifact, those paths by directory (catalogue -// Manifest.NamedPaths, ADR 0499 clause 2). Only the build section is read: what the paths are is the catalogue's rule, -// and a manifest the catalogue would refuse for another reason is judged by the indexer, which gives it none. -func namedPathsOf(manifests map[string]string) map[string][]string { - out := map[string][]string{} - for dir, raw := range manifests { - var m struct { - Build *catalogue.Build `json:"build"` - } - if json.Unmarshal([]byte(raw), &m) != nil || m.Build == nil { - continue - } - if paths, every := (catalogue.Manifest{Build: m.Build}).NamedPaths(); every { - out[dir] = paths - } - } - return out -} - -// objectsOf asks named for the objects of the named paths the manifests give at sha, and answers none when no manifest -// names paths. A forge that cannot say is an error. -func objectsOf(owner, repo, sha string, manifests map[string]string, named Named) (map[string]map[string]string, error) { - paths := namedPathsOf(manifests) - if len(paths) == 0 || named == nil { - return nil, nil - } - objects, err := named(owner, repo, sha, paths) - if err != nil { - return nil, fmt.Errorf("the objects of the named paths of %s: %w", dirsOf(paths), err) - } - for dir, want := range paths { - for _, p := range want { - if objects[dir][p] == "" { - return nil, fmt.Errorf("the forge named no object for the named path %q of %q", p, dir) - } - } - } - return objects, nil -} - -func dirsOf(paths map[string][]string) []string { - var dirs []string - for d := range paths { - dirs = append(dirs, d) - } - slices.Sort(dirs) - return dirs -} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go deleted file mode 100644 index 5cf39c5..0000000 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package bridge - -import ( - "os" - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// ADR 0499 clause 3 (I6): the merged and the pull message carry each named path's git object of a module whose every -// artifact names its paths, read at the same commit as its manifest. The manifest is mesh-sdk's module.json at 9cb4d59 (its -// lib names src, test, package.json, package-lock.json and tsconfig.json). - -type namedCall struct { - owner, repo, sha string - paths map[string][]string -} - -func namedOf(calls *[]namedCall, answer map[string]map[string]string, err error) Named { - return func(owner, repo, sha string, paths map[string][]string) (map[string]map[string]string, error) { - *calls = append(*calls, namedCall{owner, repo, sha, paths}) - return answer, err - } -} - -func sdkFiles(t *testing.T) Files { - t.Helper() - raw, err := os.ReadFile("testdata/mesh-sdk-9cb4d59.module.json") - if err != nil { - t.Fatal(err) - } - return func(owner, repo, path, ref string) ([]byte, error) { - if path == "module.json" { - return raw, nil - } - return nil, ErrNotFound - } -} - -var sdkObjects = map[string]map[string]string{"": {"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222", - "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}} - -func TestAMergedMessageCarriesTheNamedPathsObjectsOfAModuleNamingItsPaths(t *testing.T) { // I6, ADR 0499 clause 3 - var calls []call - var named []namedCall - body := mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}) - msg, err := ConvertNamed(mergedSubject, body, treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), sdkFiles(t), namedOf(&named, sdkObjects, nil)) - if err != nil { - t.Fatal(err) - } - f, err := delivery.DecodeForge(msg.Data) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(f.Paths, sdkObjects) { - t.Errorf("paths = %v, want the forge's objects of lib's named paths", f.Paths) - } - if len(named) != 1 || named[0].sha != merge || named[0].repo != "mesh-sdk" || - strings.Join(named[0].paths[""], ",") != "package-lock.json,package.json,src,test,tsconfig.json" { - t.Errorf("asked %+v, want the root's named paths at the merge commit", named) - } -} - -func TestAMessageOfModulesNamingNoPathsAsksNoObjects(t *testing.T) { // I6 - var calls []call - var named []namedCall - if _, err := ConvertNamed(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), anyFiles, namedOf(&named, nil, nil)); err != nil { - t.Fatal(err) - } - if len(named) != 0 { - t.Errorf("asked %+v, want nothing for modules naming no paths", named) - } -} - -func TestAPullMessageCarriesTheNamedPathsObjectsAtItsHead(t *testing.T) { // I6, ADR 0499 clause 3 - var named []namedCall - body := mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}, "base": "feat/y"}) - msg, err := ConvertNamed(UpdatedSubject, body, nil, sdkFiles(t), namedOf(&named, sdkObjects, nil)) - if err != nil { - t.Fatal(err) - } - f, _ := delivery.DecodeForge(msg.Data) - if !reflect.DeepEqual(f.Paths, sdkObjects) || len(named) != 1 || named[0].sha != head { - t.Errorf("paths = %v, asked %+v; want the objects read at the head", f.Paths, named) - } -} - -func TestAForgeThatCannotSayTheObjectsIsARetry(t *testing.T) { // I6: a message without them would give the module none - var calls []call - var named []namedCall - body := mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}) - _, err := ConvertNamed(mergedSubject, body, treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), sdkFiles(t), namedOf(&named, nil, os.ErrDeadlineExceeded)) - if err == nil || !strings.Contains(err.Error(), "named paths") { - t.Errorf("err = %v, want a retry naming the named paths", err) - } -} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go new file mode 100644 index 0000000..57c06bc --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go @@ -0,0 +1,36 @@ +package bridge + +import ( + "os" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ADR 0495 clause 1 with the lead's decision L8 of R4e: the forge bridge is no reader of a manifest's `paths`. A merge of a +// module whose artifacts name their paths (mesh-sdk's module.json at 9cb4d59) gives a message that carries the manifest as +// the forge answered it and no named path's object: the indexer asks the forge for those itself. +func TestAMergedMessageCarriesNoNamedPathsTheBridgeDerived(t *testing.T) { + raw, err := os.ReadFile("testdata/mesh-sdk-9cb4d59.module.json") + if err != nil { + t.Fatal(err) + } + files := func(owner, repo, path, ref string) ([]byte, error) { + if path == "module.json" { + return raw, nil + } + return nil, ErrNotFound + } + var calls []call + msg, err := Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}), treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), files) + if err != nil { + t.Fatal(err) + } + f, err := delivery.DecodeForge(msg.Data) + if err != nil { + t.Fatal(err) + } + if len(f.Paths) != 0 || f.Manifests[""] != string(raw) || len(calls) != 1 { + t.Errorf("message paths %v, manifest carried %v, trees asked %d times; want no paths, the manifest, one ask", f.Paths, f.Manifests[""] == string(raw), len(calls)) + } +} From 612658e7880a8e0f18145e009ea77e6f79a8f4e0 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:24:38 +0200 Subject: [PATCH 25/25] =?UTF-8?q?Ask=20the=20forge=20for=20the=20objects?= =?UTF-8?q?=20of=20a=20merge's=20named=20paths=20in=20the=20indexer,=20the?= =?UTF-8?q?=20one=20reader=20of=20a=20manifest's=20paths,=20and=20narrow?= =?UTF-8?q?=20the=20module's=20tree=20from=20the=20answer=20(issue=201758,?= =?UTF-8?q?=20I6=20reworked;=20ADR=200499=20=C2=A72=20=C2=A73,=20ADR=20049?= =?UTF-8?q?5=20=C2=A71,=20L8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest invokes gitea.gitea_commit_trees. A forge that cannot say gives the module none with its words. --- .../cmd/mesh-delivery-indexer/main.go | 14 +++++- .../cmd/mesh-delivery-indexer/named.go | 48 +++++++++++++++++++ .../cmd/mesh-delivery-indexer/named_test.go | 31 ++++++++++++ .../internal/index/candidates.go | 7 ++- .../internal/index/modules.go | 46 +++++++++++++++++- .../internal/index/named_test.go | 42 ++++++++++++++++ .../mesh-delivery-indexer/manifest_test.go | 10 +++- modules/mesh-delivery-indexer/module.json | 3 ++ 8 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 54df569..d921599 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -25,6 +25,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) @@ -283,7 +284,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro errs := make(chan error, 1) go func() { errs <- follow(ctx, w, each) }() - ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf} + // The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3). + account, err := process.ReadAccount(path) + if err != nil { + return err + } + calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) + if err != nil { + return err + } + defer calls.Close() + ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf, + Named: namedVia(calls.Call)} // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go new file mode 100644 index 0000000..f8fd619 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory +// (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each +// path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk +// delivery.Forge.Paths; novox/hq ADR 0499 clause 3). +const commitTreesTool = "gitea.gitea_commit_trees" + +// toolCaller calls a module's tool, `.`, as the indexer's own account (mesh-sdk process.Conn.Call). +type toolCaller func(key string, body any) (json.RawMessage, error) + +// namedVia asks the forge for the objects of the named paths through call. An answer without the object of every path +// asked is an error naming it. +func namedVia(call toolCaller) func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) { + return func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) { + owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/") + if !ok { + return nil, fmt.Errorf("%q is not /", repository) + } + raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "paths": paths}) + if err != nil { + return nil, err + } + var a struct { + Paths map[string]map[string]string `json:"paths"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s is not {root, trees, paths}: %w", commitTreesTool, err) + } + for dir, want := range paths { + for _, p := range want { + if a.Paths[dir][p] == "" { + return nil, fmt.Errorf("the answer of %s names no object for %q of %q", commitTreesTool, p, dir) + } + } + } + return a.Paths, nil + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go new file mode 100644 index 0000000..9691911 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "encoding/json" + "testing" +) + +func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T) { // I6, ADR 0499 clause 3 + var asked map[string]any + call := func(key string, body any) (json.RawMessage, error) { + if key != commitTreesTool { + t.Errorf("called %s", key) + } + asked = body.(map[string]any) + return json.RawMessage(`{"root":"r0","trees":{"":"r0"},"paths":{"":{"src":"1111111111111111111111111111111111111111","package.json":"3333333333333333333333333333333333333333"}}}`), nil + } + got, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}) + if err != nil || got[""]["src"] != "1111111111111111111111111111111111111111" || got[""]["package.json"] == "" { + t.Fatalf("objects = %v, %v", got, err) + } + if asked["owner"] != "novox" || asked["repo"] != "mesh-sdk" || asked["sha"] != "s" || asked["paths"] == nil { + t.Errorf("asked %v, want the owner, repo, commit and paths", asked) + } + call = func(string, any) (json.RawMessage, error) { + return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil + } + if _, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}); err == nil { + t.Error("an answer missing a path asked was taken") + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index d0baf34..89a1ac9 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -47,7 +47,12 @@ type Indexer struct { // Manifests is the manifests bucket; nil writes no manifests (issue 1449). Manifests Store Builds *Builds - Logf func(format string, args ...any) + // Named asks the forge for the git object of each named path of the module directories given, at a commit + // (gitea.gitea_commit_trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a + // manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths + // has none. + Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) + Logf func(format string, args ...any) mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 137b035..7749238 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -5,6 +5,7 @@ import ( "maps" "slices" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -29,6 +30,7 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { if err != nil { return err } + objects, namedErr := ix.namedObjects(ctx, f) overlay := map[string]delivery.Module{} dirOf := map[string]string{} for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) { @@ -60,8 +62,11 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" } else { - in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], f.Paths[dir], ix.baseFP(ctx)) + in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], ix.baseFP(ctx)) next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths + if in.Kept == nil && namedErr[dir] != "" { + next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir] + } } overlay[key] = next } @@ -105,3 +110,42 @@ func pathOf(dir string) string { } func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m } + +// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact +// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call +// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them. +func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) { + objects := map[string]map[string]string{} + need := map[string][]string{} + for dir, raw := range f.Manifests { + m, err := catalogue.ParseManifestForInputs([]byte(raw)) + if err != nil { + continue // inputsOfNamed says why + } + paths, every := m.NamedPaths() + if !every { + continue + } + if carried := f.Paths[dir]; len(carried) > 0 { + objects[dir] = carried + continue + } + need[dir] = paths + } + if len(need) == 0 || ix.Named == nil { + return objects, nil + } + got, err := ix.Named(ctx, f.Repository, f.Merge, need) + if err != nil { + words := map[string]string{} + for dir := range need { + words[dir] = err.Error() + } + ix.logf("%s at %s: the objects of the named paths of %v: %v", f.Repository, f.Merge, slices.Sorted(maps.Keys(need)), err) + return objects, words + } + for dir := range need { + objects[dir] = got[dir] + } + return objects, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/named_test.go b/modules/mesh-delivery-indexer/internal/index/named_test.go index 91bec02..13f99c5 100644 --- a/modules/mesh-delivery-indexer/internal/index/named_test.go +++ b/modules/mesh-delivery-indexer/internal/index/named_test.go @@ -2,6 +2,7 @@ package index import ( "context" + "errors" "reflect" "strings" "testing" @@ -89,3 +90,44 @@ func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T) t.Errorf("with a fingerprint: %+v, want the named paths' objects carried for the builder's fetch", got) } } + +func TestTheIndexerAsksTheForgeForTheNamedPathsObjectsOfAMerge(t *testing.T) { // I6 as the lead has it: the indexer is the one reader of paths + ix, mods, _ := manifestsIndexer(t) + type ask struct { + repository, sha string + paths map[string][]string + } + var asked []ask + ix.Named = func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) { + asked = append(asked, ask{repository, sha, paths}) + return map[string]map[string]string{"": sdkObjects}, nil + } + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree", "modules/a": "t"}), map[string]string{"": raw, "modules/a": `{"module":"a","version":"1"}`}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if len(asked) != 1 || asked[0].repository != "novox/mesh-sdk" || asked[0].sha != merge || len(asked[0].paths) != 1 || + strings.Join(asked[0].paths[""], ",") != "package-lock.json,package.json,src,test,tsconfig.json" { + t.Fatalf("asked %+v, want the root's named paths of the merge commit, once, and nothing for a module naming none", asked) + } + m, _ := moduleOf(t, mods, "mesh-sdk") + want, _ := delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects) + if m.Inputs2 == nil || m.Inputs2.Tree != want { + t.Errorf("module = %+v, want the named tree %s", m, want) + } +} + +func TestAForgeThatCannotSayTheObjectsGivesTheModuleNoneWithItsWords(t *testing.T) { // I6 + ix, mods, _ := manifestsIndexer(t) + ix.Named = func(context.Context, string, string, map[string][]string) (map[string]map[string]string, error) { + return nil, errors.New("gitea: the path tsconfig.json of \"\" is not at the commit") + } + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, nil)); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "mesh-sdk"); m.FP2 != "" || !strings.Contains(m.None2, "is not at the commit") { + t.Errorf("module = %+v, want none with the forge's words", m) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 055782f..ce9784a 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -38,6 +38,7 @@ type manifest struct { Consumes []string `json:"consumes"` State []state `json:"state"` Reads []string `json:"reads"` + Invokes []string `json:"invokes"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -93,8 +94,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if m.SecretsOwner != "mesh-delivery-indexer" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // No invokes, no emits, no streams, no bus, no seat, no tool (ADR 0437 §4). - for _, k := range []string{"invokes", "streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { + // One invoke: the forge's trees with the named paths' objects, since the indexer is the one reader of a manifest's paths + // (ADR 0499 clauses 2 and 3, ADR 0495 clause 1, the lead's decision L8 of R4e). + if want := []string{"gitea.gitea_commit_trees"}; !reflect.DeepEqual(m.Invokes, want) { + t.Errorf("invokes = %v, want %v", m.Invokes, want) + } + // No emits, no streams, no bus, no seat, no tool (ADR 0437 §4). + for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the indexer has none", k) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index cb58b8b..706ee06 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -20,6 +20,9 @@ "mesh-delivery-checker.previews", "mesh-delivery-gate.source" ], + "invokes": [ + "gitea.gitea_commit_trees" + ], "own-secrets": { "broker": "${dir:state}/broker" },