From 4c2914194be4fca58098730209c54370930d7cbd Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:20:23 +0100 Subject: [PATCH 1/7] mesh-delivery-indexer: a module's mesh packages are what its go.mod requires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At main's revision 21, 64 of 82 Go modules carried no package edge: the version line read the mesh packages go.sum names, and go.sum names none required at the zero version (ADR 0573 §2). The take-in now reads them from the go.mod's require lines, at any version, with no build needed; the go.sum is read only for the requirements state. The sibling rule (replaces by a directory) stands. Builds: design 47 H H-A-157, H-A-136 (sibling rule), G §47 (kind 1) --- .../cmd/mesh-delivery-indexer/named.go | 25 +++-- .../cmd/mesh-delivery-indexer/named_test.go | 7 +- .../packages_integration_test.go | 8 +- .../internal/fingerprint/fingerprint.go | 30 ++++-- .../internal/fingerprint/fingerprint_test.go | 14 +++ .../internal/index/modules.go | 91 +++++++++---------- .../internal/index/modules_test.go | 11 +++ .../internal/index/requirements.go | 2 +- .../internal/index/takenin.go | 2 +- 9 files changed, 120 insertions(+), 70 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 1dd3ce4a..04da28ad 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -182,9 +182,9 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit, var lockFiles = []string{"go.sum", "package-lock.json"} // readVia reads a repository at a commit as the forge bridge does for a merged message: the seat git's trees, then the -// module.json and lock files of the one directory dir by its file (a repository can hold hundreds). A manifest the forge -// does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does -// not have is left out. +// module.json and lock files of the one directory dir by its file (a repository can hold hundreds), and the go.mod beside a +// go.sum for its mesh packages (Packages). A manifest the forge does not have is an error; a directory with no module.json in +// the trees is answered with no manifest; a lock file it does not have is left out. func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) { return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) { a, err := treesOf(call, repository, git.TreesInput{Commit: sha}) @@ -222,13 +222,20 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str out.Locks[dir] = map[string]string{} } out.Locks[dir][name] = hex.EncodeToString(sum[:]) - if name == "go.sum" { - if p := fingerprint.MeshPackagesOf([]byte(c)); len(p) > 0 { - if out.Packages == nil { - out.Packages = map[string][]string{} - } - out.Packages[dir] = p // the mesh packages it uses, as the bridge reads them (ADR 0573 §2) + if name != "go.sum" { + continue + } + // A Go module's mesh packages are what its go.mod requires, at any version (design 47 H-A-157): its go.sum + // names none required at the zero version (ADR 0573 §2). + mod, absent, err := file(dir, "go.mod") + if err != nil { + return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "go.mod"), err) + } + if p := fingerprint.MeshRequiresOf([]byte(mod)); !absent && len(p) > 0 { + if out.Packages == nil { + out.Packages = map[string][]string{} } + out.Packages[dir] = p } } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index ab3faf53..c3d723f4 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -120,6 +120,8 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { return json.RawMessage(`{"path":"module.json","content":"{\"module\":\"snake\"}"}`), nil case "go.sum": return json.RawMessage(`{"path":"go.sum","content":"x"}`), nil + case "go.mod": // a mesh package required at the zero version, which no go.sum line names (design 47 H-A-157) + return json.RawMessage(`{"path":"go.mod","content":"module snake\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n)\n"}`), nil } return json.RawMessage(`{"path":"x","absent":true}`), nil } @@ -129,12 +131,15 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others + if len(got.Manifests) != 1 || len(files) != 4 { // the one directory asked: its manifest, two lock files and the go.mod, never the others t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files) } if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" { t.Fatalf("read = %+v", got) } + if p := got.Packages[""]; len(p) != 1 || p[0] != "git.novox.be/novox/mesh-sdk/go" { + t.Fatalf("mesh packages %v, want mesh-sdk, which the go.mod requires", got.Packages) + } } // fakeRuns records the run events handed to it and answers err. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index b50fe15a..603c875e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -58,7 +58,8 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name } // TestASourceRevisionMovingMeshSDKNamesItAndItsImporters (design 47 H §743; A-476a; ADR 0593 §2 to §4): the indexer whole against -// the bus. Two Go modules of the catalogue, importer, whose go.sum requires mesh-sdk's Go module, and plain, which requires +// the bus. Two Go modules of the catalogue, importer, whose go.mod requires mesh-sdk's Go module at the zero version (no +// go.sum line names it, ADR 0573 §2; design 47 H-A-157), and plain, which requires // nothing of it, are taken in with mesh-sdk as main's first source revision. A source revision moving mesh-sdk to a commit // changing its buildstep package is taken in as one revision whose movers are mesh-sdk and importer, of another repository, // and not plain; the packages key of mesh-sdk/go names that commit with a digest per package. @@ -67,7 +68,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { sdkGo = "git.novox.be/novox/mesh-sdk/go" m1, m2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222" catalogMerge = "3333333333333333333333333333333333333333" - goSumContent = "git.novox.be/novox/mesh-sdk/go v0.1.14 h1:x=\ngit.novox.be/novox/mesh-sdk/go v0.1.14/go.mod h1:y=\n" + goModContent = "module importer\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n" sdkRoot1, sdk2 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ) avahi, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") @@ -119,7 +120,8 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { forge.put(m2, sdkAt(sdk2, "f6", "7777777777777777777777777777777777777777")) forge.put(catalogMerge, repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, files: map[string]string{"modules/importer/module.json": named("importer"), "modules/plain/module.json": named("plain"), - "modules/importer/go.sum": goSumContent, "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}}) + "modules/importer/go.mod": goModContent, "modules/importer/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n", + "modules/plain/go.mod": "module plain\n\nrequire golang.org/x/sys v0.1.0\n", "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}}) regs := registerAvahi(t, ctx, js, "main") for _, r := range []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}}, {V: 1, Module: "plain", Origin: "novox/mesh-catalog", Path: "modules/plain", Branches: []string{"main"}}, diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go index b5309795..4a326972 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -135,14 +135,32 @@ func NamedTree(manifest string, paths map[string]string) (string, error) { return NamedTreePrefix + hex.EncodeToString(sum[:]), nil } -// MeshPackagesOf are the mesh Go modules a go.sum names (the module path of each line under -// trunk.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set -// takes the version line of (ADR 0593 §2). -func MeshPackagesOf(goSum []byte) []string { +// MeshRequiresOf are the mesh Go modules a go.mod requires (the module path of each `require` line under +// trunk.ForgeModulePrefix, alone or in a block, at any version, the zero version and `// indirect` included), sorted and +// each once; nil for none. They are a module's mesh packages from the take-in, with no build needed (design 47 H-A-157): +// a go.sum names none required at the zero version (ADR 0573 §2), so it is never read for them. +func MeshRequiresOf(goMod []byte) []string { seen := map[string]bool{} - for _, line := range strings.Split(string(goSum), "\n") { + inBlock := false + for _, line := range strings.Split(string(goMod), "\n") { + line = strings.TrimSpace(line) + if i := strings.Index(line, "//"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + switch { + case line == "require (": + inBlock = true + continue + case inBlock && line == ")": + inBlock = false + continue + case strings.HasPrefix(line, "require "): + line = strings.TrimSpace(strings.TrimPrefix(line, "require ")) + case !inBlock: + continue + } f := strings.Fields(line) - if len(f) < 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) { + if len(f) != 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) { continue } if _, _, ok := trunk.PackageRepository(f[0]); ok { diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go index f143ee14..6a1d68bb 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go @@ -31,6 +31,20 @@ func versionLineTakesASiblingDirectoryByItsContent(t *testing.T) { if got := ReplacedSiblings(goMod, repo); len(got) != 1 || got[0] != buildrun { t.Fatalf("siblings %v, want buildrun alone (mesh-sdk is another repository, replaced by a version)", got) } + // Its mesh packages are every mesh package its go.mod requires, at any version, the zero version included (design 47 + // H-A-157); a replace line or a module outside the forge is none. + for _, c := range []struct { + goMod string + want []string + }{ + {string(goMod), []string{buildrun, sdk}}, + {"module x\n\nrequire " + sdk + " v0.0.0-00010101000000-000000000000 // indirect\nrequire github.com/nats-io/nats.go v1.37.0\n", []string{sdk}}, + {"module x\n\nreplace " + sdk + " => ../sdk\n", nil}, + } { + if got := MeshRequiresOf([]byte(c.goMod)); strings.Join(got, ",") != strings.Join(c.want, ",") { + t.Errorf("mesh packages of %q: %v, want %v", c.goMod, got, c.want) + } + } published := func(digest string) map[string]meshcatalog.PackagesByModuleValue { return map[string]meshcatalog.PackagesByModuleValue{buildrun: {Module: buildrun, Repository: repo, Requires: "r1", Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: buildrun, Digest: digest}}}} diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 05638826..f259e2cc 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -108,7 +108,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" } else { - packages := ix.goSumPackages(ctx, f, dir, cur, has) + packages := ix.goPackages(ctx, f, dir, cur, has) in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx)) next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths if in.Kept != nil && has && cur.Inputs2 != nil { @@ -365,79 +365,72 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string return in } -// goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go -// modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory -// (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none. -func (ix *Indexer) goSumPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { - others := ix.otherPackages(ctx, f, dir, cur, has) +// goPackages are the version line's packages of the merge's directory dir (ADR 0593 §2; design 47 H-A-157): the mesh Go +// modules of other repositories its go.mod requires, at any version, and those of its own repository its go.mod replaces +// by a directory (siblingPackages, H-A-136's sibling rule). A directory with no go.sum is no Go module and has none. +func (ix *Indexer) goPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 { - return others + return nil + } + ix.requirementsOf(ctx, f, dir, cur, has) + goMod, read := ix.goModOf(ctx, f, dir) + others := fingerprint.OtherPackages(f.Packages[dir], f.Repository) // carried: what the go.mod requires, read with the tree + switch { + case len(others) > 0: + case read: + others = fingerprint.OtherPackages(fingerprint.MeshRequiresOf(goMod), f.Repository) + case has && cur.Inputs2 != nil: + others = fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) + } + var siblings []string + if read { + siblings = fingerprint.ReplacedSiblings(goMod, f.Repository) + } else if has && cur.Inputs2 != nil { + for _, p := range cur.Inputs2.Packages { + if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) { + siblings = append(siblings, p) + } + } } - siblings := ix.siblingPackages(ctx, f, dir, cur, has) if len(siblings) == 0 { return others } return slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(others), siblings...)))) } -// siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read -// from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said. -func (ix *Indexer) siblingPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { - held := func() []string { - if !has || cur.Inputs2 == nil { - return nil - } - var out []string - for _, p := range cur.Inputs2.Packages { - if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) { - out = append(out, p) - } - } - return out - } +// goModOf is the go.mod of the merge's directory dir, read from the forge (GoMod), and whether it was read; where it +// cannot be, the held module's packages stand, said. +func (ix *Indexer) goModOf(ctx context.Context, f whole, dir string) ([]byte, bool) { if ix.GoMod == nil { - return held() + return nil, false } raw, err := ix.GoMod(ctx, f.Repository, f.Merge, dir) if err != nil { - ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the directory replaces it held: %v", f.Repository, f.Merge, dir, err) - return held() + ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err) + return nil, false } - return fingerprint.ReplacedSiblings(raw, f.Repository) + return raw, true } -// otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the -// message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read -// from the forge (GoSum). -func (ix *Indexer) otherPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { - if carried := f.Packages[dir]; len(carried) > 0 { - return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree - } +// requirementsOf keeps what the go.sum of the merge's directory dir requires in the requirements state +// (keepRequirements): read from the forge (GoSum) when no mesh package was carried and its hash moved, or this process +// has not kept it yet. It gives no package: those are the go.mod's (H-A-157). +func (ix *Indexer) requirementsOf(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) { hash := f.Locks[dir]["go.sum"] - if hash == "" { - return nil + if len(f.Packages[dir]) > 0 || hash == "" || ix.GoSum == nil { + return } if has && cur.Inputs2 != nil && trunk.SameRepository(cur.Repository, f.Repository) && ix.requirementsKept(f.Repository, dir) { for _, h := range cur.Inputs2.Locks { if h == hash { - return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) + return } } } - if ix.GoSum == nil { - if has && cur.Inputs2 != nil { - return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) - } - return nil - } raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir) if err != nil { - ix.logf("%s at %s: the go.sum of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err) - if has && cur.Inputs2 != nil { - return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) - } - return nil + ix.logf("%s at %s: the go.sum of %q cannot be read, so what it requires is not kept: %v", f.Repository, f.Merge, dir, err) + return } ix.keepRequirements(ctx, f.Repository, dir, raw) - return fingerprint.OtherPackages(fingerprint.MeshPackagesOf(raw), f.Repository) } diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 43521238..445763d0 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -402,6 +402,14 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6 ix, mods, _ := manifestsIndexer(t) // no build record of any module + // Its go.mod requires mesh-sdk at the zero version, which its go.sum does not name (ADR 0573 §2): the go.mod's + // require lines are its mesh packages, with no build (design 47 H-A-157). + ix.GoMod = func(context.Context, string, string, string) ([]byte, error) { + return []byte("module avahi\n\nrequire " + meshSDK + " v0.0.0-00010101000000-000000000000\n"), nil + } + ix.GoSum = func(context.Context, string, string, string) ([]byte, error) { + return []byte("github.com/nats-io/nats.go v1.37.0 h1:x=\n"), nil + } avahi := catalogueManifest(t, "avahi") f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) @@ -412,6 +420,9 @@ func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t * if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != meshcatalogclient.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" { t.Fatalf("module = %+v (%v), want its directory, merge, manifest and inputs from the message alone", m, ok) } + if !slices.Equal(m.Inputs2.Packages, []string{meshSDK}) { + t.Errorf("mesh packages %v, want mesh-sdk, which its go.mod requires at the zero version", m.Inputs2.Packages) + } if m.Fp != "" || m.None != "" { t.Errorf("module = %+v, want no value of today's form (I4: it came from the build records)", m) } diff --git a/modules/mesh-delivery-indexer/internal/index/requirements.go b/modules/mesh-delivery-indexer/internal/index/requirements.go index e0624c9e..f1b2ff04 100644 --- a/modules/mesh-delivery-indexer/internal/index/requirements.go +++ b/modules/mesh-delivery-indexer/internal/index/requirements.go @@ -41,7 +41,7 @@ func (r Requirement) Versions() []string { return out } -// RequiredOf is, per mesh Go module a go.sum names (as fingerprint.MeshPackagesOf takes them), the highest version it +// RequiredOf is, per mesh Go module a go.sum names (a line under trunk.ForgeModulePrefix whose path names a repository), the highest version it // names, its module line or its go.mod line alike. func RequiredOf(goSum []byte) map[string]string { out := map[string]string{} diff --git a/modules/mesh-delivery-indexer/internal/index/takenin.go b/modules/mesh-delivery-indexer/internal/index/takenin.go index e899a25f..41a95ccd 100644 --- a/modules/mesh-delivery-indexer/internal/index/takenin.go +++ b/modules/mesh-delivery-indexer/internal/index/takenin.go @@ -9,7 +9,7 @@ type RepoRead struct { Trees map[string]string Manifests map[string]string Locks map[string]map[string]string - // Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2). + // Packages are the mesh packages each directory's go.mod requires (fingerprint.MeshRequiresOf, design 47 H-A-157). Packages map[string][]string } From e04321acaeebba50aef14e72fde53287c7c890f0 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:20:56 +0100 Subject: [PATCH 2/7] mesh-delivery-indexer: a build's reported imports refine the go.mod's packages, never empty them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An import form whose imports name no package of another repository, or a build that stands with versions held empty from the go.sum era, wrote an empty package set while the go.mod requires a mesh package: no kind-1 edge. The version line's versions of the go.mod's set stand then. Builds: design 47 H H-A-157, G §47 (kind 1) --- .../internal/index/form.go | 49 ++++++++++++++----- .../internal/index/imports_test.go | 32 ++++++++---- 2 files changed, 60 insertions(+), 21 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/form.go b/modules/mesh-delivery-indexer/internal/index/form.go index 96636838..e638c374 100644 --- a/modules/mesh-delivery-indexer/internal/index/form.go +++ b/modules/mesh-delivery-indexer/internal/index/form.go @@ -24,15 +24,16 @@ type form struct { // // - **The import form**, for a module whose last trunk build reported its imports (kept.Imports) when every import is // published (the packages state): the content digest of each imported package and the requirements' digest of each -// mesh Go module holding one. Its versions are each holding module's published commit (another repository's only). +// mesh Go module holding one. Its versions are each holding module's published commit (another repository's only); an +// import set naming none of another repository while the go.mod requires one keeps the version line's (refined). // - **A change of form never rebuilds by itself** (§4): kept.ImportForm is the import-form fingerprint of the inputs // the build that reported the imports was built from. While the import form computes to it, nothing the build was // built from changed: the module keeps the fingerprint it holds (prev), in whatever form, and its versions. // Otherwise the import form is written, and ImportForm moves with it. // - **The version line** otherwise (no import set, or an import none publishes): the version of each mesh Go module of -// another repository its go.sum names, read as the indexer always read it (fingerprint.HeldVersionsOf), and the -// content of each mesh Go module of its own repository its go.mod replaces by a directory (fingerprint.VersionLine), -// so a change of that directory alone moves it (review finding 2). Its versions are the other repositories' alone. +// another repository its go.mod requires, at any version (design 47 H-A-157), read as the indexer always read it +// (fingerprint.HeldVersionsOf), and the content of each mesh Go module of its own repository its go.mod replaces by a +// directory (fingerprint.VersionLine), so a change of that directory alone moves it (review finding 2). Its versions are the other repositories' alone. // A package of another repository none of them publishes takes the commit the gate's source names for its repository // (heads, design 47 G A-365). // - **An accepted series** (DLV-I6): published and pins come from acceptedSeries, so a mesh Go module of another @@ -51,10 +52,11 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp case fp == "": return form{none: none} case kept.ImportForm == fp && prev.Fp2 != "": - return form{fp: prev.Fp2, versions: prev.Packages} // the build that reported the imports stands + // The build that reported the imports stands, and so do its versions. + return form{fp: prev.Fp2, versions: refined(sdkstep.PackageVersions(prev.Packages), r.Versions, module, repository, kept, held, pins, heads)} } kept.ImportForm = fp - return form{fp: fp, versions: r.Versions} + return form{fp: fp, versions: refined(r.Versions, nil, module, repository, kept, held, pins, heads)} } if logf != nil { logf("%s: no published package holds %s, so its fingerprint takes the version line", module, strings.Join(r.Missing, ", ")) @@ -63,6 +65,34 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp // The fingerprint written here is the version line's, not the import form's: no import form is held for the inputs it was // computed from, so a later pass whose import form computes the same does not take this one for the build's (issue 1966). kept.ImportForm = "" + versions := lineVersions(module, repository, kept, held, pins, heads, logf) + in.Packages = fingerprint.VersionLine(repository, kept.Packages, versions, published) + fp, none := fingerprint.Of(in) + if fp == "" { + return form{none: none} + } + return form{fp: fp, versions: versions} +} + +// refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod +// requires a mesh package of another repository (kept.Packages), the version line's, so a build's reported imports +// refine the go.mod's set and never empty it (design 47 H-A-157); nil when nothing names one. +func refined(versions, then sdkstep.PackageVersions, module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) sdkstep.PackageVersions { + switch { + case len(versions) > 0: + return versions + case len(then) > 0: + return then + case len(fingerprint.OtherPackages(kept.Packages, repository)) > 0: + return lineVersions(module, repository, kept, held, pins, heads, nil) + } + return versions +} + +// lineVersions are the version line's versions of the mesh Go modules of other repositories the module's go.mod requires +// (kept.Packages): each as the indexer has always read it, an accepted series' pin over it, else the commit the gate's +// source names for its repository. +func lineVersions(module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, logf func(string, ...any)) sdkstep.PackageVersions { others := fingerprint.OtherPackages(kept.Packages, repository) versions, unpublished := fingerprint.HeldVersionsOf(others, held) for p, v := range pins { // an accepted series' newest published version (acceptedSeries, DLV-I6) @@ -90,12 +120,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp if len(unpublished) > 0 && logf != nil { logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", ")) } - in.Packages = fingerprint.VersionLine(repository, kept.Packages, versions, published) - fp, none := fingerprint.Of(in) - if fp == "" { - return form{none: none} - } - return form{fp: fp, versions: versions} + return versions } // inputsFrom is the fingerprint's inputs of module m from its kept inputs and the given toolchains, no package of diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 4f308265..0579ca8a 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -370,22 +370,36 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * head := strings.Repeat("c", 40) contracts := "git.novox.be/novox/mesh-contracts/go/seats/findings" held := map[string]meshcatalog.ModulesByModuleValue{"other": {Repository: "novox/mesh-sdk", Commit: strings.Repeat("d", 40), Path: "go"}} + // A build's imports of its own repository alone (design 47 H-A-157): they refine the go.mod's set and never empty it. + own := "git.novox.be/novox/mesh-catalog/modules/avahi/lib" + published := map[string]meshcatalog.PackagesByModuleValue{own: {Module: own, Repository: "novox/mesh-catalog", Commit: head, Requires: "r1", + Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: own, Digest: "d1"}}}} for _, c := range []struct { - name string - heads map[string]string - held map[string]meshcatalog.ModulesByModuleValue - want string + name string + heads map[string]string + held map[string]meshcatalog.ModulesByModuleValue + imports []string + want string }{ - {"the source names its repository", map[string]string{"novox/mesh-contracts": head}, held, head}, - {"the source names another repository only", map[string]string{"novox/mesh-sdk": head}, held, ""}, + {"the source names its repository", map[string]string{"novox/mesh-contracts": head}, held, nil, head}, + {"the source names another repository only", map[string]string{"novox/mesh-sdk": head}, held, nil, ""}, {"a module the indexer holds publishes it", map[string]string{"novox/mesh-contracts": head}, - map[string]meshcatalog.ModulesByModuleValue{"c": {Repository: "novox/mesh-contracts", Commit: strings.Repeat("e", 40), Path: "go/seats/findings"}}, strings.Repeat("e", 40)}, + map[string]meshcatalog.ModulesByModuleValue{"c": {Repository: "novox/mesh-contracts", Commit: strings.Repeat("e", 40), Path: "go/seats/findings"}}, nil, strings.Repeat("e", 40)}, + {"a build's imports naming none of another repository keep the go.mod's", map[string]string{"novox/mesh-contracts": head}, held, []string{own}, head}, } { - kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t", Packages: []string{contracts}} - f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, c.held, nil, nil, c.heads, nil) + kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t", Packages: []string{contracts}, Imports: c.imports} + f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, c.held, published, nil, c.heads, nil) if got := f.versions[contracts]; got != c.want { t.Errorf("%s: version of %s = %q, want %q", c.name, contracts, got, c.want) } + if c.imports == nil { + continue + } + // The build that reported them stands, its held versions empty (a version line read from go.sum). + f = formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("b", 64)}, c.held, published, nil, c.heads, nil) + if got := f.versions[contracts]; got != c.want || f.fp != "src2:"+strings.Repeat("b", 64) { + t.Errorf("%s, the build standing: version of %s = %q (fp %q), want %q", c.name, contracts, got, f.fp, c.want) + } } }) t.Run("formOf clears the import form when it writes the version line", func(t *testing.T) { From c43e7a5d1c9d946ecabc42481a4535a9e7404e57 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:31:16 +0100 Subject: [PATCH 3/7] mesh-delivery-indexer, mesh-delivery-builder: a package is pinned by its released version, never by a commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indexer filled a module's packages with the commit of a module value it holds, which Go turned into a pseudo-version of whatever that tree was (mesh-delivery-sdk at an 11-19 commit, nodeengine pre-H1). A pin is now the newest released version of the module holding the package (the versions state version-released says), an accepted series' version stands, and a package with none released is kept with no version: its kind-1 edge stands and the builder asks no pin for it. A commit of another repository moves a module only through a package of it the module imports (A-476a), now that pins name no commit. Builds: design 47 H H-A-161, G §370, H §746, H-A-157 (kind-1 edges) --- .../internal/build/run.go | 1 + .../internal/build/run_test.go | 8 +- .../packages_integration_test.go | 6 +- .../internal/fingerprint/fingerprint.go | 56 +++++++------ .../internal/index/imports_test.go | 8 +- .../internal/index/seed_steps.go | 2 +- .../internal/index/toolchains.go | 17 +++- .../internal/index/toolchains_test.go | 14 ++-- .../internal/index/versions.go | 78 +++++++++++++++++++ 9 files changed, 150 insertions(+), 40 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 7a329c8c..9eaed460 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -1114,6 +1114,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes // The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes // them into the workspace, whatever the repository's go.mod names. ask.Packages = maps.Clone(m.Packages) + maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161) for _, fb := range req.Builds { delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) } diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 41f4d44f..a13d4c37 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -332,10 +332,12 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { r := newRig(t, goBundle) r.asks.Answer = nil // no node has pulled it yet - // Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with. - sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')} + // Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none + // released, design 47 H-A-161) is no pin. + sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"} data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", - Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), Packages: sdk}) + Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}}) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index 603c875e..e2f46323 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -144,7 +144,11 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge}) heard.next(t, "main", 10) heard.acked(t, "main") - untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" && m.Packages[sdkGo] == m1 }) + // mesh-sdk has released no version: importer keeps the package for its kind-1 edge, with no pin (H-A-161). + untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { + pin, kept := m.Packages[sdkGo] + return m.Fp2 != "" && kept && pin == "" + }) plain := untilModule(t, ctx, js, "plain", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" }) if _, ok := plain.Packages[sdkGo]; ok { t.Fatalf("plain requires %s: %+v", sdkGo, plain.Packages) diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go index 4a326972..c5a4e5cb 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -423,41 +423,49 @@ func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys // commit or no directory, is in missing, sorted; versions is nil when packages is empty. func HeldVersionsOf(packages []string, modules map[string]meshcatalog.ModulesByModuleValue) (versions sdkstep.PackageVersions, missing []string) { for _, pkg := range packages { - repository, dir, ok := trunk.PackageRepository(pkg) - commit, depth := "", -1 - if ok { - for _, key := range slices.Sorted(maps.Keys(modules)) { - m := modules[key] - if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" { - continue - } - at := strings.Trim(m.Path, "/") - switch { - case at == "": - continue // a value that names no directory says nothing of where the package lies - case at == meshcatalogclient.RootPath: - at = "" - case dir != at && !strings.HasPrefix(dir, at+"/"): - continue - } - if d := len(at); d > depth { - commit, depth = m.Commit, d - } - } - } - if commit == "" { + key := HolderOf(pkg, modules) + if key == "" { missing = append(missing, pkg) continue } if versions == nil { versions = sdkstep.PackageVersions{} } - versions[pkg] = commit + versions[pkg] = modules[key].Commit } slices.Sort(missing) return versions, missing } +// HolderOf is the key of the held module of the package's repository whose directory holds the package's directory (the +// deepest, a module at the repository's root holding every directory) and that names a commit; "" for none. +func HolderOf(pkg string, modules map[string]meshcatalog.ModulesByModuleValue) string { + repository, dir, ok := trunk.PackageRepository(pkg) + if !ok { + return "" + } + holder, depth := "", -1 + for _, key := range slices.Sorted(maps.Keys(modules)) { + m := modules[key] + if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" { + continue + } + at := strings.Trim(m.Path, "/") + switch { + case at == "": + continue // a value that names no directory says nothing of where the package lies + case at == meshcatalogclient.RootPath: + at = "" + case dir != at && !strings.HasPrefix(dir, at+"/"): + continue + } + if d := len(at); d > depth { + holder, depth = key, d + } + } + return holder +} + // digestOf is the content digest of the package at import path pkg in a published module, and whether it holds it // (its packages are sorted by import path). func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) { diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 0579ca8a..222cf39f 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -118,8 +118,8 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T t.Fatal(err) } moved, _ := moduleOf(t, mods, "avahi") - if moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 { - t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3", + if pin, kept := moved.Packages[meshSDK]; moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || !kept || pin != "" { + t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form, mesh-sdk kept unpinned (none released, H-A-161)", moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages) } } @@ -256,8 +256,8 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) { } } before := step("trunk at c1", trunk(c1)) - if before.Packages[meshSDK] != c1 { - t.Fatalf("with no series published, versions %v, want the commit form c1", before.Packages) + if pin, kept := before.Packages[meshSDK]; !kept || pin != "" { + t.Fatalf("with no series published, versions %v, want mesh-sdk kept with no pin, never the commit (H-A-161)", before.Packages) } step("the build reports its imports", func() error { return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.Fp2, meshSDK, meshSDK+"/announce")) diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index 1aed0f99..9667902c 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -108,7 +108,7 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. seat := seatOf[a.Ref] sa := SeedArtifact{Name: name, Seat: seat} in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp, - Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: maps.Clone(e.Packages)}} + Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: PinsOf(e.Packages)}} // The module's directory and its named paths go together, only when the entry names its paths (ADR 0499 clause // 4); a request without them builds from the module's directory (sdk/buildstep FetchRequest). if paths := namedPaths(e, name); len(paths) > 0 { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index ae5b21e4..25975a4a 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -215,7 +215,8 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc ix.logf("no fingerprint is written by this pass: %v", err) return fmt.Errorf("no fingerprint is written: %w", err) } - series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6) + series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6) + released := ix.releasedOf(ctx) // each module's newest released version, what a package is pinned to (H-A-161) type result struct { fp, none string tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint @@ -297,8 +298,9 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions } } + r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161) if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && - maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { + !importsOf(r.pkgs, repository) { // A module registered for another repository is moved by this repository's commit only when it imports a package the // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's. r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} @@ -577,3 +579,14 @@ func (ix *Indexer) fillPaths(ctx context.Context) error { } return nil } + +// importsOf says whether pkgs name a package of repository: a pin is a released version (H-A-161), so a commit of +// repository moves a module of another only through a package of it the module imports (A-476a). +func importsOf(pkgs sdkstep.PackageVersions, repository string) bool { + for p := range pkgs { + if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, repository) { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 6357bc82..ec8c3e5e 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -757,7 +757,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.Fp2, u.None2) } tools, _ := moduleOf(t, mods, "mesh-tools") - sdkMerge := func(commit, tree string) meshcatalog.ModulesByModuleValue { + sdkMerge := func(commit, tree, pin string) meshcatalog.ModulesByModuleValue { t.Helper() f := whole{Repository: "novox/mesh-sdk", Branch: "feat/x", Head: h1, Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}} @@ -765,9 +765,10 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { t.Fatal(err) } // A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools, - // which imports nothing it moved. + // which imports nothing it moved. Its pin is mesh-sdk's newest released version, never the commit; with none + // released it has no pin, the package kept for its kind-1 edge (H-A-161). a, _ := moduleOf(t, mods, "avahi") - if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) { + if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: pin}) { t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages) } if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 || o.Packages != nil { @@ -781,8 +782,11 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { stale.Fp2 = "src2:" + strings.Repeat("5", 64) mods.put("mesh-tools", mustEncode(t, stale)) tools = stale - first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1") - second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2") + first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "") + ix.Versions = newMem() + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`)) + second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2", "v0.1.20") if second.Fp2 == first.Fp2 { t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2) } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 574a6fb5..1a14fb21 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -10,10 +10,14 @@ import ( "strconv" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" + + "mesh-delivery-indexer/internal/fingerprint" ) // The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's @@ -352,3 +356,77 @@ func isFloor(raw []byte) bool { var v meshcatalog.VersionEntry return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0 } + +// releasedOf is the newest released version of each module the state versions keeps (§349; design 47 H-A-161), by module; +// nil when the indexer keeps no such state or it cannot be listed, said. +func (ix *Indexer) releasedOf(ctx context.Context) map[string]string { + if ix.Versions == nil { + return nil + } + all, err := ix.Versions.Keys(ctx) + if err != nil { + ix.logf("versions: the state cannot be listed, so no package is pinned by this pass: %v", err) + return nil + } + out := map[string]string{} + for _, k := range all { + m, v, ok := meshcatalog.ParseVersionsByModuleVersionKey(k) + if _, parts := versionParts(v); !ok || !parts { + continue + } + if cur, has := out[m]; !has || compareVersions(v, cur) > 0 { + out[m] = v + } + } + return out +} + +// releasedPins are a module's pins (its catalog entry's packages, which the builder builds it against) from the versions its +// fingerprint was computed with (design 47 H-A-161, G §370): a package the held module holding it has released is pinned +// to that module's newest released version, as Go names it (v); a version already in that form (an accepted +// series' newest, DLV-I6) stands; any other, a commit, is no pin: its package is kept with no version, so its kind-1 edge +// stands (H §746) and no build is made against a commit. nil for none. +func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) sdkstep.PackageVersions { + if len(versions) == 0 { + return nil + } + out := sdkstep.PackageVersions{} + for p, v := range versions { + switch key := fingerprint.HolderOf(p, held); { + case sdkstep.PackageSemver.MatchString(v): + out[p] = v + case released[moduleOfKey(key)] != "": + out[p] = "v" + released[moduleOfKey(key)] + default: + out[p] = "" + } + } + return out +} + +// moduleOfKey is the module a modules key names; "" for none. +func moduleOfKey(key string) string { + if key == "" { + return "" + } + if m, err := meshcatalogclient.ParseModulesKey(key); err == nil { + return m + } + return key +} + +// PinsOf are the pins a build is asked with from a catalog entry's packages: each package with a version; one kept with +// none (no released version, H-A-161) is no pin. nil for none. +func PinsOf(packages map[string]string) sdkstep.PackageVersions { + var out sdkstep.PackageVersions + for p, v := range packages { + if v == "" { + continue + } + if out == nil { + out = sdkstep.PackageVersions{} + } + out[p] = v + } + return out +} From 5c5136d6d12cade75aea17f532c2e074c583a615 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:32:09 +0100 Subject: [PATCH 4/7] mesh-delivery-indexer: the requirements state reads the go.sum whatever packages the reader carried With the go.mod's mesh requires carried at every take-in, requirementsOf returned early for every module requiring a mesh package: its go.sum was never read and keepRequirements never cleared an old version it named (indexer_requirements for mesh-sdk still names v0.1.13). The go.sum is read whenever its hash moved or this process has not kept it yet. Builds: design 47 H H-A-157 (review round 1) --- .../internal/index/modules.go | 6 +-- .../internal/index/modules_test.go | 41 +++++++++++++------ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index f259e2cc..0b130d11 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -413,11 +413,11 @@ func (ix *Indexer) goModOf(ctx context.Context, f whole, dir string) ([]byte, bo } // requirementsOf keeps what the go.sum of the merge's directory dir requires in the requirements state -// (keepRequirements): read from the forge (GoSum) when no mesh package was carried and its hash moved, or this process -// has not kept it yet. It gives no package: those are the go.mod's (H-A-157). +// (keepRequirements): read from the forge (GoSum) when its hash moved, or this process has not kept it yet, whatever +// packages the reader carried. It gives no package: those are the go.mod's (H-A-157). func (ix *Indexer) requirementsOf(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) { hash := f.Locks[dir]["go.sum"] - if len(f.Packages[dir]) > 0 || hash == "" || ix.GoSum == nil { + if hash == "" || ix.GoSum == nil { return } if has && cur.Inputs2 != nil && trunk.SameRepository(cur.Repository, f.Repository) && ix.requirementsKept(f.Repository, dir) { diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 445763d0..d765110e 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -401,16 +401,36 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means } func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6 - ix, mods, _ := manifestsIndexer(t) // no build record of any module - // Its go.mod requires mesh-sdk at the zero version, which its go.sum does not name (ADR 0573 §2): the go.mod's - // require lines are its mesh packages, with no build (design 47 H-A-157). - ix.GoMod = func(context.Context, string, string, string) ([]byte, error) { - return []byte("module avahi\n\nrequire " + meshSDK + " v0.0.0-00010101000000-000000000000\n"), nil - } - ix.GoSum = func(context.Context, string, string, string) ([]byte, error) { - return []byte("github.com/nats-io/nats.go v1.37.0 h1:x=\n"), nil - } avahi := catalogueManifest(t, "avahi") + // Its go.mod requires mesh-sdk at the zero version (ADR 0573 §2): the go.mod's require lines are its mesh packages, with + // no build, carried by the reader or read through GoMod (design 47 H-A-157). Its go.sum is still read for the + // requirements state, carried packages or not, so an older version it names is kept there (and cleared when gone). + for _, carried := range [][]string{nil, {meshSDK}} { + ix, mods, _ := manifestsIndexer(t) + ix.Requirements = newMem() + ix.GoMod = func(context.Context, string, string, string) ([]byte, error) { + return []byte("module avahi\n\nrequire " + meshSDK + " v0.0.0-00010101000000-000000000000\n"), nil + } + ix.GoSum = func(context.Context, string, string, string) ([]byte, error) { + return []byte(meshSDK + " v0.1.14 h1:x=\n"), nil + } + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, + map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + if carried != nil { + f.Packages = map[string][]string{"modules/avahi": carried} + } + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, _ := moduleOf(t, mods, "avahi") + if m.Inputs2 == nil || !slices.Equal(m.Inputs2.Packages, []string{meshSDK}) { + t.Errorf("carried %v: mesh packages %+v, want mesh-sdk, which its go.mod requires at the zero version", carried, m.Inputs2) + } + if _, kept := ix.Requirements.(*memStore).vals[meshSDK]; !kept { + t.Errorf("carried %v: the requirements state holds no %s, which the go.sum names", carried, meshSDK) + } + } + ix, mods, _ := manifestsIndexer(t) // no build record of any module f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) if err := ix.Apply(context.Background(), f); err != nil { @@ -420,9 +440,6 @@ func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t * if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != meshcatalogclient.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" { t.Fatalf("module = %+v (%v), want its directory, merge, manifest and inputs from the message alone", m, ok) } - if !slices.Equal(m.Inputs2.Packages, []string{meshSDK}) { - t.Errorf("mesh packages %v, want mesh-sdk, which its go.mod requires at the zero version", m.Inputs2.Packages) - } if m.Fp != "" || m.None != "" { t.Errorf("module = %+v, want no value of today's form (I4: it came from the build records)", m) } From 037cb9555e8a0bd3e8cba7efdf74d5a1a3de090d Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:34:59 +0100 Subject: [PATCH 5/7] mesh-delivery-indexer: a version-released re-pins the modules holding the package Pins moved to a new released version only at the next merge, toolchain report or hand recompute. The release now starts the pass of H-A-161 for the modules whose packages hold a package of the released module alone; every other module stands as held. Builds: design 47 H H-A-165 (the indexer's part), H-A-161 --- .../internal/index/toolchains.go | 38 +++++++++++++++++-- .../internal/index/toolchains_test.go | 10 +++++ .../internal/index/versions.go | 7 ++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 25975a4a..94bec3d7 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -16,6 +16,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/catalogue" + "mesh-delivery-indexer/internal/fingerprint" "mesh-delivery-indexer/judge" ) @@ -152,11 +153,13 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]mesh } ix.recompute.Lock() defer ix.recompute.Unlock() - return ix.recomputeLocked(ctx, overlay, w, repository) + return ix.recomputeLocked(ctx, overlay, w, repository, nil) } // recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go). -func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { +// +// only, when not nil, names the modules the pass moves (a re-pin, H-A-165): every other stands as held. +func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string, only map[string]bool) error { if ix.Manifests == nil { return ix.writeOverlay(ctx, overlay, nil, w) } @@ -299,8 +302,8 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc } } r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161) - if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && - !importsOf(r.pkgs, repository) { + if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && + !importsOf(r.pkgs, repository) || only != nil && !only[module]) { // A module registered for another repository is moved by this repository's commit only when it imports a package the // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's. r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} @@ -590,3 +593,30 @@ func importsOf(pkgs sdkstep.PackageVersions, repository string) bool { } return false } + +// Repin moves the pins of every module whose packages hold a package of module (a held module's directory holds it) to +// module's newest released version, the pass of H-A-161 for those modules alone: a version-released of module starts it +// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It answers how many it named. +func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) { + if ix.Modules == nil || ix.Manifests == nil { + return 0, nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return 0, err + } + only := map[string]bool{} + for key, m := range held { + for p := range m.Packages { + if moduleOfKey(fingerprint.HolderOf(p, held)) == module { + only[key] = true + } + } + } + if len(only) == 0 { + return 0, nil + } + ix.recompute.Lock() + defer ix.recompute.Unlock() + return len(only), ix.recomputeLocked(ctx, nil, nil, "", only) +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index ec8c3e5e..b35cb8ec 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -786,6 +786,16 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { ix.Versions = newMem() ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`)) + // mesh-sdk's release re-pins avahi with no merge, and leaves mesh-tools, which holds none of it (H-A-165). + if n, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil || n != 1 { + t.Fatalf("re-pin on mesh-sdk's release: %d modules, %v; want avahi alone", n, err) + } + if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 != first.Fp2 || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) { + t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want the same fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages) + } + if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 { + t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2) + } second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2", "v0.1.20") if second.Fp2 == first.Fp2 { t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2) diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 1a14fb21..d82709e4 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -238,6 +238,13 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli } ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source) } + if !late { // the release re-pins every module whose packages hold one of it (design 47 H-A-165) + if n, err := ix.Repin(ctx, p.Module); err != nil { + ix.logf("versions: the modules pinned to %s are not re-pinned to %s: %v", p.Module, p.Version, err) + } else if n > 0 { + ix.logf("versions: %d modules re-pinned to %s %s", n, p.Module, p.Version) + } + } if err := ix.trimVersions(ctx, p.Module); err != nil { ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err) } From fe51379ee2514b3d03f98636a9d89c1ec09615e1 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:44:13 +0100 Subject: [PATCH 6/7] mesh-delivery-indexer: a release re-pins or is not acknowledged; pins enter the version line's fingerprint A re-pin that left the fingerprint as it was left the build made against the older pin as the one the mesh runs. The version line's fingerprint now takes the pins, so a re-pin moves the importer, which becomes a mover built against the release. An accepted series' older version no longer stands over a newer release. A re-pin that fails returns its error before the heard artifacts are forgotten, and a package-published heard again for a released version re-pins, so the redelivery recovers it. The release path and the seed's pins carry tests that fail without them (review round 2). Builds: design 47 H H-A-165, H-A-167, H-A-161 --- .../internal/fingerprint/fingerprint.go | 18 +++++++ .../internal/index/form.go | 3 +- .../internal/index/imports_test.go | 6 +++ .../internal/index/seed_steps_test.go | 7 ++- .../internal/index/toolchains.go | 15 ++++-- .../internal/index/toolchains_test.go | 33 ++++++++++--- .../internal/index/versions.go | 49 ++++++++++++++----- 7 files changed, 105 insertions(+), 26 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go index c5a4e5cb..64bf2d12 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -481,3 +481,21 @@ func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) { // NamedTreePrefix opens a module's tree when it is computed over the paths its artifacts name, not the directory's own // git tree (novox/hq ADR 0499 clause 2): the indexer alone computes it (NamedTree). const NamedTreePrefix = "named:" + +// WithPins is the fingerprint fp with the pins a build of it is made against (design 47 H-A-167): fp itself when no +// package is pinned, else `src2:` and the hex sha256 of fp and, sorted, the lines "pin " of each pin, +// joined by a newline. A package kept with no version (none released) is no pin. +func WithPins(fp string, pins sdkstep.PackageVersions) string { + var lines []string + for p, v := range pins { + if v != "" { + lines = append(lines, "pin "+p+" "+v) + } + } + if fp == "" || len(lines) == 0 { + return fp + } + sort.Strings(lines) + sum := sha256.Sum256([]byte(fp + "\n" + strings.Join(lines, "\n"))) + return meshcatalogclient.SourcePrefix2 + hex.EncodeToString(sum[:]) +} diff --git a/modules/mesh-delivery-indexer/internal/index/form.go b/modules/mesh-delivery-indexer/internal/index/form.go index e638c374..28b92d33 100644 --- a/modules/mesh-delivery-indexer/internal/index/form.go +++ b/modules/mesh-delivery-indexer/internal/index/form.go @@ -17,6 +17,7 @@ import ( type form struct { fp, none string versions sdkstep.PackageVersions + line bool // the version line's, whose fingerprint takes the pins too (pinnedOf, H-A-167) } // formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the @@ -71,7 +72,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp if fp == "" { return form{none: none} } - return form{fp: fp, versions: versions} + return form{fp: fp, versions: versions, line: true} } // refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 222cf39f..6371753a 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -272,6 +272,12 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) { if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" { t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages) } + // mesh-sdk releases 0.1.20: an accepted series' older version never stands over a newer release (H-A-167). + ix.Versions = newMem() + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + if m := step("0.1.20 released", func() error { return nil }); m.Packages[meshSDK] != "v0.1.20" { + t.Errorf("a newer release left avahi on its series' older version: %v", m.Packages) + } } // Issue 1966 (the indexer started while the bus's storage was down): a read of the packages state that failed, or that diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go index 22c7b3a9..6824bb67 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go @@ -24,7 +24,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { fp := "src2:" + strings.Repeat("a", 64) entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}}, - Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"}, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "v0.1.9", "git.novox.be/novox/mesh-contracts/go": ""}, // the second none released (H-A-161) Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}} rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) if err != nil { @@ -71,9 +71,12 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { t.Errorf("the image artifact is %+v", a) } in := got[0] - if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" { + if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "v0.1.9" { t.Errorf("the steps input is %+v", in) } + if _, pinned := in.Fetch.Packages["git.novox.be/novox/mesh-contracts/go"]; pinned { + t.Errorf("a package kept with no version is pinned in the fetch: %v", in.Fetch.Packages) + } if len(w.Held()) != 0 { t.Fatal("a call unanswered for no time wrote its finding") } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 94bec3d7..f3b72e2f 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -291,17 +291,18 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.none = strings.Join(missing, "; ") case noToolchain != "": r.none = noToolchain - if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads()); ok { + if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), released); ok { r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs} } default: f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.sourceHeads(), ix.logf) r.fp, r.none = f.fp, f.none if r.fp != "" { - r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions + // a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167) + r.fp, r.pkgs = pinnedOf(f, held, released) + r.tcs = toolchainsUsed(toolchains) } } - r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161) if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && !importsOf(r.pkgs, repository) || only != nil && !only[module]) { // A module registered for another repository is moved by this repository's commit only when it imports a package the @@ -497,7 +498,7 @@ func hasContext(m catalogue.Manifest) bool { // with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any // other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the // module has none until they agree. -func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) (string, sdkstep.PackageVersions, bool) { +func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, released map[string]string) (string, sdkstep.PackageVersions, bool) { if k.held.Fp2 == "" { return "", nil, false } @@ -514,7 +515,11 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} } f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, heads, nil) - return f.fp, f.versions, f.none == "" && f.fp == k.held.Fp2 + if f.none != "" || f.fp == "" { + return "", nil, false + } + fp, pkgs := pinnedOf(f, held, released) + return fp, pkgs, fp == k.held.Fp2 } // toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's). diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index b35cb8ec..281540a7 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -783,15 +784,33 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { mods.put("mesh-tools", mustEncode(t, stale)) tools = stale first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "") - ix.Versions = newMem() - ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + // mesh-sdk releases 0.1.20 (0.1.9 before it): the release re-pins avahi with no merge, its fingerprint moving with the + // pin so it is built again against the release, and leaves mesh-tools, which holds none of it (H-A-165, H-A-167). A + // re-pin that fails is no release acknowledged: the package-published delivered again re-pins. + ix.Versions, ix.Heard = newMem(), newMem() ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`)) - // mesh-sdk's release re-pins avahi with no merge, and leaves mesh-tools, which holds none of it (H-A-165). - if n, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil || n != 1 { - t.Fatalf("re-pin on mesh-sdk's release: %d modules, %v; want avahi alone", n, err) + revisions := ix.Revisions + ix.Revisions = newMemObjects(1 << 20) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "mesh-sdk:go"}}}}) + if err != nil { + t.Fatal(err) } - if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 != first.Fp2 || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) { - t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want the same fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages) + if err := KeepRevision(context.Background(), ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + release := meshbuilds.PackagePublished{Module: "mesh-sdk", Artifact: "mesh-sdk:go", Digest: "sha256:g", Version: "0.1.20", Source: 7, Branch: "main"} + mods.keysErr = errors.New("the bus did not answer") + if err := ix.HearPackagePublished(context.Background(), release); err == nil { + t.Fatal("a release whose re-pin failed was acknowledged") + } + mods.keysErr = nil + if err := ix.HearPackagePublished(context.Background(), release); err != nil { + t.Fatal(err) + } + ix.Revisions = revisions + if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == first.Fp2 || a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) { + t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want a new fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages) } if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 { t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2) diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index d82709e4..0587b99d 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -127,8 +127,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil { return err } else if found && !isFloor(raw) { - ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact) - return nil + // A package-published delivered again after a re-pin failed (H-A-167): the re-pin is made again, nothing else. + ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and only re-pins", p.Module, p.Version, p.Artifact) + return ix.repin(ctx, p.Module, p.Version) } entry, err := ix.entryOf(ctx, p.Module) if err != nil { @@ -238,11 +239,11 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli } ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source) } - if !late { // the release re-pins every module whose packages hold one of it (design 47 H-A-165) - if n, err := ix.Repin(ctx, p.Module); err != nil { - ix.logf("versions: the modules pinned to %s are not re-pinned to %s: %v", p.Module, p.Version, err) - } else if n > 0 { - ix.logf("versions: %d modules re-pinned to %s %s", n, p.Module, p.Version) + if !late { + // The release re-pins every module whose packages hold one of it (design 47 H-A-165); one that fails is not + // acknowledged, so the package-published is delivered again (H-A-167). + if err := ix.repin(ctx, p.Module, p.Version); err != nil { + return err } } if err := ix.trimVersions(ctx, p.Module); err != nil { @@ -399,11 +400,14 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog. } out := sdkstep.PackageVersions{} for p, v := range versions { - switch key := fingerprint.HolderOf(p, held); { - case sdkstep.PackageSemver.MatchString(v): + rel := released[moduleOfKey(fingerprint.HolderOf(p, held))] + series := sdkstep.PackageSemver.MatchString(v) + _, plain := versionParts(strings.TrimPrefix(v, "v")) + switch { + case rel != "" && (!series || !plain || compareVersions(rel, strings.TrimPrefix(v, "v")) > 0): + out[p] = "v" + rel // never an older series' version over a newer release (H-A-167) + case series: out[p] = v - case released[moduleOfKey(key)] != "": - out[p] = "v" + released[moduleOfKey(key)] default: out[p] = "" } @@ -411,6 +415,17 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog. return out } +// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so the +// version line's fingerprint takes them (design 47 H-A-167, ADR 0573 §4) and a re-pin moves it, the module a mover +// built again against the release; a fingerprint with no pin is the form's as it is. +func pinnedOf(f form, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) (string, sdkstep.PackageVersions) { + pins := releasedPins(f.versions, held, released) + if !f.line { + return f.fp, pins + } + return fingerprint.WithPins(f.fp, pins), pins +} + // moduleOfKey is the module a modules key names; "" for none. func moduleOfKey(key string) string { if key == "" { @@ -437,3 +452,15 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions { } return out } + +// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error. +func (ix *Indexer) repin(ctx context.Context, module, version string) error { + n, err := ix.Repin(ctx, module) + if err != nil { + return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err) + } + if n > 0 { + ix.logf("versions: %d modules re-pinned to %s %s", n, module, version) + } + return nil +} From d0436bb38c18f0cdafa99bcb497fd0eb682ed851 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:46:52 +0100 Subject: [PATCH 7/7] mesh-delivery-indexer: pins are in every form's fingerprint The import form's fingerprint left the pins out, so an importer whose trunk build reported its imports never moved on a re-pin and was never built again against the release. Every form now takes its sorted pins; a build that stands keeps its fingerprint while its pins are the ones it was built with, and a fingerprint with no pin stays as it was. Builds: design 47 H H-A-172, H-A-167 --- .../internal/index/form.go | 11 ++++++++--- .../internal/index/imports_test.go | 17 +++++++++++++++++ .../internal/index/versions.go | 15 ++++++++++----- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/form.go b/modules/mesh-delivery-indexer/internal/index/form.go index 28b92d33..133f42e1 100644 --- a/modules/mesh-delivery-indexer/internal/index/form.go +++ b/modules/mesh-delivery-indexer/internal/index/form.go @@ -17,7 +17,11 @@ import ( type form struct { fp, none string versions sdkstep.PackageVersions - line bool // the version line's, whose fingerprint takes the pins too (pinnedOf, H-A-167) + // stands: fp is the held fingerprint of the build that reported the imports, made with the pins built (its held + // packages) from the import form raw; pinnedOf keeps it while the pins are those (H-A-172). + stands bool + raw string + built map[string]string } // formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the @@ -54,7 +58,8 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp return form{none: none} case kept.ImportForm == fp && prev.Fp2 != "": // The build that reported the imports stands, and so do its versions. - return form{fp: prev.Fp2, versions: refined(sdkstep.PackageVersions(prev.Packages), r.Versions, module, repository, kept, held, pins, heads)} + return form{fp: prev.Fp2, versions: refined(sdkstep.PackageVersions(prev.Packages), r.Versions, module, repository, kept, held, pins, heads), + stands: true, raw: fp, built: prev.Packages} } kept.ImportForm = fp return form{fp: fp, versions: refined(r.Versions, nil, module, repository, kept, held, pins, heads)} @@ -72,7 +77,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp if fp == "" { return form{none: none} } - return form{fp: fp, versions: versions, line: true} + return form{fp: fp, versions: versions} } // refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 6371753a..0bcf5888 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -122,6 +122,23 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form, mesh-sdk kept unpinned (none released, H-A-161)", moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages) } + + // mesh-sdk releases 0.1.20: the import form's fingerprint takes the pin too and moves, once (H-A-172). + ix.Versions = newMem() + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + for range 2 { + if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } + } + if m, _ := moduleOf(t, mods, "avahi"); m.Fp2 == moved.Fp2 || m.Fp2 == "" || m.Packages[meshSDK] != "v0.1.20" || m.Inputs2.ImportForm != moved.Inputs2.ImportForm { + t.Fatalf("after mesh-sdk's release: fp2 %q (was %q), versions %v, import form %q; want a new fingerprint pinned to v0.1.20, the import form kept", + m.Fp2, moved.Fp2, m.Packages, m.Inputs2.ImportForm) + } else if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } else if again, _ := moduleOf(t, mods, "avahi"); again.Fp2 != m.Fp2 { + t.Fatalf("a pass with nothing moved moved the pinned fingerprint %q to %q", m.Fp2, again.Fp2) + } } // A build of a fingerprint the module no longer holds says nothing of its inputs now: its imports are not kept. diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 0587b99d..1fcae613 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "slices" "strconv" "strings" @@ -415,13 +416,17 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog. return out } -// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so the -// version line's fingerprint takes them (design 47 H-A-167, ADR 0573 §4) and a re-pin moves it, the module a mover -// built again against the release; a fingerprint with no pin is the form's as it is. +// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so every +// form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover +// built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that +// stands is kept while its pins are the ones it was built with, else its import form takes the new pins. func pinnedOf(f form, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) (string, sdkstep.PackageVersions) { pins := releasedPins(f.versions, held, released) - if !f.line { - return f.fp, pins + if f.stands { + if maps.Equal(f.built, map[string]string(pins)) { + return f.fp, pins + } + return fingerprint.WithPins(f.raw, pins), pins } return fingerprint.WithPins(f.fp, pins), pins }