From 46e53a81c29012e3191138330cab46a396ba155b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 18:45:45 +0100 Subject: [PATCH] mesh-delivery-checker: build mesh packages at the versions the indexer publishes instead of computing them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0593 §2: only the indexer computes a package's version; the checker follows its packages state and hands those commits to a Go check, so LatestPackages leaves repoModules and mesh-sdk's PackageVersionsOf has no reader here. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-delivery-checker/judge.go | 39 +++----------- .../cmd/mesh-delivery-checker/judge_test.go | 39 +++++++++----- .../cmd/mesh-delivery-checker/main.go | 16 +++++- .../cmd/mesh-delivery-checker/packages.go | 53 +++++++++++++++++++ .../mesh-delivery-checker/manifest_test.go | 2 +- modules/mesh-delivery-checker/module.json | 1 + 6 files changed, 103 insertions(+), 47 deletions(-) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/packages.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index d66feae..e4589b4 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -26,9 +26,11 @@ type judgeDeps struct { kept check.KV previews check.KV repos *repoModules - forge check.Forge - announce check.Announcer - builds check.BuildStates + // published is the indexer's packages state as it is followed (packages.go). + published *publishedPackages + forge check.Forge + announce check.Announcer + builds check.BuildStates // compositions is the assigner's bucket of compositions; nil leaves every composition not run. compositions check.CompositionReader // nodes names every node with a record; nil hands no names. @@ -95,7 +97,7 @@ func newJudge(d judgeDeps) check.Judge { nodes = d.nodes } return &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Predefined: d.predefined, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Predefined: d.predefined, Logf: d.logf} } // manifestKV reads a manifest by its digest. @@ -118,7 +120,6 @@ type repoModules struct { repoOf map[string]string // by module: its repository commit map[string]string // by module: the commit its record was computed at (the commit the mesh runs) typed map[string]string // by module: its manifest digest - uses map[string][]string // by module: the mesh packages it uses (Inputs2.Packages, ADR 0573 §2) } // Put takes a change of the indexer's modules bucket in: a module's repository and the languages its manifest names. @@ -127,7 +128,6 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted if r.repoOf == nil { r.repoOf, r.typed, r.langs, r.commit = map[string]string{}, map[string]string{}, map[string]map[string]bool{}, map[string]string{} r.tsPkgs, r.path, r.named = map[string][]string{}, map[string]string{}, map[string]map[string][]string{} - r.uses = map[string][]string{} } if deleted { delete(r.repoOf, key) @@ -135,7 +135,6 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted delete(r.commit, key) delete(r.path, key) delete(r.named, key) - delete(r.uses, key) r.mu.Unlock() return } @@ -150,10 +149,6 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted r.commit[key] = m.Commit r.path[key] = m.Path r.named[key] = m.ArtifactPaths - r.uses[key] = nil - if m.Inputs2 != nil { - r.uses[key] = m.Inputs2.Packages - } _, known := r.langs[m.Manifest] r.mu.Unlock() if m.Manifest == "" || known { @@ -185,28 +180,6 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted r.mu.Unlock() } -// LatestPackages is the latest published version of each mesh package a module of the mesh uses, and of mesh-sdk's Go -// module always (novox/hq ADR 0573 §2): the trunk merge of the module whose directory holds it, as the indexer's modules -// say. A package no module holds is left out: a check builds it at the version go.mod names. -func (r *repoModules) LatestPackages() delivery.PackageVersions { - r.mu.Lock() - defer r.mu.Unlock() - used := map[string]bool{check.SDKModulePath: true} - held := map[string]delivery.Module{} - for module, repo := range r.repoOf { - for _, p := range r.uses[module] { - used[p] = true - } - held[module] = delivery.Module{Repository: repo, Path: r.path[module], Commit: r.commit[module]} - } - pkgs := make([]string, 0, len(used)) - for p := range used { - pkgs = append(pkgs, p) - } - versions, _ := delivery.PackageVersionsOf(pkgs, held) - return versions -} - // PackagePaths are the paths, relative to the repository, that the TypeScript package artifacts of a repository's modules // are built from, sorted: the named paths the indexer keeps in each module's value (delivery.Module.ArtifactPaths; only the // indexer reads them from a manifest, the lead's decision L8), under the module's directory. None for a module whose value diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go index 7942122..24c64d2 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" "testing" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -113,19 +114,33 @@ func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *t if got := r.PackagePaths("novox/mesh-delivery"); len(got) != 0 { t.Errorf("another repository: %v", got) } - // Issue 1585 (ADR 0573 §2): the latest published version of mesh-sdk's Go module and of every package a module uses is - // the commit of the module that holds it; a package no module holds is left out. - put(".", nil) - const contract = "git.novox.be/novox/mesh-controller/modules/route-proxy/contract" - user, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: strings.Repeat("b", 40), FP: "src1:" + strings.Repeat("b", 64), - Manifest: delivery.ManifestDigest([]byte(sdkManifest)), Path: "modules/x", - Inputs2: &delivery.KeptInputs2{Tree: "t", Packages: []string{contract, "git.novox.be/novox/mesh-sdk/go"}}}) - if err != nil { - t.Fatal(err) +} + +// ADR 0593 §2 (D8): a Go check builds every mesh package at the version the indexer's packages state publishes, read +// there and computed nowhere here: a key's module at its commit, a deleted key gone, an unreadable value left out. +func TestThePublishedVersionsAreTheIndexersPackagesState(t *testing.T) { + var p publishedPackages + put := func(module, repository, dir, commit string) { + key, _ := delivery.PackagesKey(module) + raw, err := delivery.EncodePublishedPackages(delivery.PublishedPackages{Module: module, Repository: repository, Dir: dir, + Commit: commit, Requires: strings.Repeat("e", 64), + Packages: []delivery.PackageDigest{{Package: module, Digest: strings.Repeat("d", 64)}}, At: time.Now().UTC()}) + if err != nil { + t.Fatal(err) + } + p.Put(key, raw, false) } - r.Put(context.Background(), "x", user, false) - if got := r.LatestPackages(); !reflect.DeepEqual(got, delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": strings.Repeat("a", 40)}) { - t.Errorf("latest packages = %v, want mesh-sdk at its module's commit and no unheld contract", got) + const sdk, build = "git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-catalog/modules/mesh-build-go" + put(sdk, "novox/mesh-sdk", "go", strings.Repeat("a", 40)) + put(build, "novox/mesh-catalog", "modules/mesh-build-go", strings.Repeat("b", 40)) + p.Put("broken", []byte("{"), false) + if got := p.LatestPackages(); !reflect.DeepEqual(got, delivery.PackageVersions{sdk: strings.Repeat("a", 40), build: strings.Repeat("b", 40)}) { + t.Errorf("latest packages = %v", got) + } + key, _ := delivery.PackagesKey(build) + p.Put(key, nil, true) + if got := p.LatestPackages(); len(got) != 1 || got[sdk] == "" { + t.Errorf("after a delete: %v", got) } } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 7961a97..3296872 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -195,11 +195,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf} + // The indexer's packages state (ADR 0593 §2): followed once the bus has it, so a Go check builds every mesh package at + // its published version; until then a check builds each at the version its go.mod names. + published := &publishedPackages{} + process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "a Go check builds each mesh package at the version its go.mod names", + logger.Printf).Ready(func(kv jetstream.KeyValue) { + w, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + logger.Printf("%s cannot be watched, so a Go check builds each mesh package at the version its go.mod names: %v", delivery.PackagesBucket, err) + return + } + go func() { + _ = follow(ctx, w, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) }) + }() + }) ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6) ch.Judge = newJudge(judgeDeps{ asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks, kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, - repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf, + repos: repos, published: published, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf, announce: announce, compositions: compositions, nodes: openNodeNames(ctx, js, logger.Printf), predefined: keptPredefined{process.AwaitBucket(ctx, js, conn, delivery.IndexerModule+"_"+delivery.IndexerPredefinedState, diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/packages.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/packages.go new file mode 100644 index 0000000..165397e --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/packages.go @@ -0,0 +1,53 @@ +package main + +import ( + "sync" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// publishedPackages is the indexer's state `packages` as the checker follows it (novox/hq ADR 0593 §2): by mesh Go +// module, the latest published commit of its repository the indexer read its packages at. The indexer alone computes a +// package's version; the checker reads it, so a Go check builds every mesh package at the version the mesh publishes +// (check.Examiner.Published) and computes none. +type publishedPackages struct { + mu sync.Mutex + byKey map[string]delivery.PublishedPackages + unread map[string]string +} + +// Put takes one key of the state in: its value, or its deletion. A value that does not decode is kept out and remembered +// with why, never read as an older one. +func (p *publishedPackages) Put(key string, raw []byte, deleted bool) { + p.mu.Lock() + defer p.mu.Unlock() + if p.byKey == nil { + p.byKey, p.unread = map[string]delivery.PublishedPackages{}, map[string]string{} + } + delete(p.byKey, key) + delete(p.unread, key) + if deleted { + return + } + v, err := delivery.DecodePublishedPackages(raw) + if err != nil { + p.unread[key] = err.Error() + return + } + p.byKey[key] = v +} + +// LatestPackages is the latest published version of every mesh Go module the indexer holds, by module path: the commit +// its repository was published at. A module the state does not hold is left out: a check builds it at the version +// go.mod names. +func (p *publishedPackages) LatestPackages() delivery.PackageVersions { + p.mu.Lock() + defer p.mu.Unlock() + out := delivery.PackageVersions{} + for _, v := range p.byKey { + if v.Module != "" && v.Commit != "" { + out[v.Module] = v.Commit + } + } + return out +} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 3e6ac73..82cd884 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -103,7 +103,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-delivery-assigner.compositions", "mesh-delivery-assigner.placement", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates", - "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.predefined", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.packages", "mesh-delivery-indexer.predefined", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // 1887/A5b: the checker reads nothing of the controller's plans-building (ADR 0483 superseded by ADR 0525 §7), calls no diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index a8adc46..841c990 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -42,6 +42,7 @@ "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.packages", "mesh-delivery-indexer.predefined", "mesh-delivery-watchman.presence", "node-engine.have"