diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 7a329c8c..9eaed460 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -1114,6 +1114,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes // The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes // them into the workspace, whatever the repository's go.mod names. ask.Packages = maps.Clone(m.Packages) + maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161) for _, fb := range req.Builds { delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) } diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 41f4d44f..a13d4c37 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -332,10 +332,12 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { r := newRig(t, goBundle) r.asks.Answer = nil // no node has pulled it yet - // Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with. - sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')} + // Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none + // released, design 47 H-A-161) is no pin. + sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"} data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", - Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), Packages: sdk}) + Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}}) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 1dd3ce4a..04da28ad 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -182,9 +182,9 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit, var lockFiles = []string{"go.sum", "package-lock.json"} // readVia reads a repository at a commit as the forge bridge does for a merged message: the seat git's trees, then the -// module.json and lock files of the one directory dir by its file (a repository can hold hundreds). A manifest the forge -// does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does -// not have is left out. +// module.json and lock files of the one directory dir by its file (a repository can hold hundreds), and the go.mod beside a +// go.sum for its mesh packages (Packages). A manifest the forge does not have is an error; a directory with no module.json in +// the trees is answered with no manifest; a lock file it does not have is left out. func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) { return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) { a, err := treesOf(call, repository, git.TreesInput{Commit: sha}) @@ -222,13 +222,20 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str out.Locks[dir] = map[string]string{} } out.Locks[dir][name] = hex.EncodeToString(sum[:]) - if name == "go.sum" { - if p := fingerprint.MeshPackagesOf([]byte(c)); len(p) > 0 { - if out.Packages == nil { - out.Packages = map[string][]string{} - } - out.Packages[dir] = p // the mesh packages it uses, as the bridge reads them (ADR 0573 §2) + if name != "go.sum" { + continue + } + // A Go module's mesh packages are what its go.mod requires, at any version (design 47 H-A-157): its go.sum + // names none required at the zero version (ADR 0573 §2). + mod, absent, err := file(dir, "go.mod") + if err != nil { + return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "go.mod"), err) + } + if p := fingerprint.MeshRequiresOf([]byte(mod)); !absent && len(p) > 0 { + if out.Packages == nil { + out.Packages = map[string][]string{} } + out.Packages[dir] = p } } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index ab3faf53..c3d723f4 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -120,6 +120,8 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { return json.RawMessage(`{"path":"module.json","content":"{\"module\":\"snake\"}"}`), nil case "go.sum": return json.RawMessage(`{"path":"go.sum","content":"x"}`), nil + case "go.mod": // a mesh package required at the zero version, which no go.sum line names (design 47 H-A-157) + return json.RawMessage(`{"path":"go.mod","content":"module snake\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n)\n"}`), nil } return json.RawMessage(`{"path":"x","absent":true}`), nil } @@ -129,12 +131,15 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others + if len(got.Manifests) != 1 || len(files) != 4 { // the one directory asked: its manifest, two lock files and the go.mod, never the others t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files) } if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" { t.Fatalf("read = %+v", got) } + if p := got.Packages[""]; len(p) != 1 || p[0] != "git.novox.be/novox/mesh-sdk/go" { + t.Fatalf("mesh packages %v, want mesh-sdk, which the go.mod requires", got.Packages) + } } // fakeRuns records the run events handed to it and answers err. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index b50fe15a..e2f46323 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -58,7 +58,8 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name } // TestASourceRevisionMovingMeshSDKNamesItAndItsImporters (design 47 H §743; A-476a; ADR 0593 §2 to §4): the indexer whole against -// the bus. Two Go modules of the catalogue, importer, whose go.sum requires mesh-sdk's Go module, and plain, which requires +// the bus. Two Go modules of the catalogue, importer, whose go.mod requires mesh-sdk's Go module at the zero version (no +// go.sum line names it, ADR 0573 §2; design 47 H-A-157), and plain, which requires // nothing of it, are taken in with mesh-sdk as main's first source revision. A source revision moving mesh-sdk to a commit // changing its buildstep package is taken in as one revision whose movers are mesh-sdk and importer, of another repository, // and not plain; the packages key of mesh-sdk/go names that commit with a digest per package. @@ -67,7 +68,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { sdkGo = "git.novox.be/novox/mesh-sdk/go" m1, m2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222" catalogMerge = "3333333333333333333333333333333333333333" - goSumContent = "git.novox.be/novox/mesh-sdk/go v0.1.14 h1:x=\ngit.novox.be/novox/mesh-sdk/go v0.1.14/go.mod h1:y=\n" + goModContent = "module importer\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n" sdkRoot1, sdk2 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ) avahi, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") @@ -119,7 +120,8 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { forge.put(m2, sdkAt(sdk2, "f6", "7777777777777777777777777777777777777777")) forge.put(catalogMerge, repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, files: map[string]string{"modules/importer/module.json": named("importer"), "modules/plain/module.json": named("plain"), - "modules/importer/go.sum": goSumContent, "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}}) + "modules/importer/go.mod": goModContent, "modules/importer/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n", + "modules/plain/go.mod": "module plain\n\nrequire golang.org/x/sys v0.1.0\n", "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}}) regs := registerAvahi(t, ctx, js, "main") for _, r := range []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}}, {V: 1, Module: "plain", Origin: "novox/mesh-catalog", Path: "modules/plain", Branches: []string{"main"}}, @@ -142,7 +144,11 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge}) heard.next(t, "main", 10) heard.acked(t, "main") - untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" && m.Packages[sdkGo] == m1 }) + // mesh-sdk has released no version: importer keeps the package for its kind-1 edge, with no pin (H-A-161). + untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { + pin, kept := m.Packages[sdkGo] + return m.Fp2 != "" && kept && pin == "" + }) plain := untilModule(t, ctx, js, "plain", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" }) if _, ok := plain.Packages[sdkGo]; ok { t.Fatalf("plain requires %s: %+v", sdkGo, plain.Packages) diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go index b5309795..64bf2d12 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -135,14 +135,32 @@ func NamedTree(manifest string, paths map[string]string) (string, error) { return NamedTreePrefix + hex.EncodeToString(sum[:]), nil } -// MeshPackagesOf are the mesh Go modules a go.sum names (the module path of each line under -// trunk.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set -// takes the version line of (ADR 0593 §2). -func MeshPackagesOf(goSum []byte) []string { +// MeshRequiresOf are the mesh Go modules a go.mod requires (the module path of each `require` line under +// trunk.ForgeModulePrefix, alone or in a block, at any version, the zero version and `// indirect` included), sorted and +// each once; nil for none. They are a module's mesh packages from the take-in, with no build needed (design 47 H-A-157): +// a go.sum names none required at the zero version (ADR 0573 §2), so it is never read for them. +func MeshRequiresOf(goMod []byte) []string { seen := map[string]bool{} - for _, line := range strings.Split(string(goSum), "\n") { + inBlock := false + for _, line := range strings.Split(string(goMod), "\n") { + line = strings.TrimSpace(line) + if i := strings.Index(line, "//"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + switch { + case line == "require (": + inBlock = true + continue + case inBlock && line == ")": + inBlock = false + continue + case strings.HasPrefix(line, "require "): + line = strings.TrimSpace(strings.TrimPrefix(line, "require ")) + case !inBlock: + continue + } f := strings.Fields(line) - if len(f) < 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) { + if len(f) != 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) { continue } if _, _, ok := trunk.PackageRepository(f[0]); ok { @@ -405,41 +423,49 @@ func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys // commit or no directory, is in missing, sorted; versions is nil when packages is empty. func HeldVersionsOf(packages []string, modules map[string]meshcatalog.ModulesByModuleValue) (versions sdkstep.PackageVersions, missing []string) { for _, pkg := range packages { - repository, dir, ok := trunk.PackageRepository(pkg) - commit, depth := "", -1 - if ok { - for _, key := range slices.Sorted(maps.Keys(modules)) { - m := modules[key] - if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" { - continue - } - at := strings.Trim(m.Path, "/") - switch { - case at == "": - continue // a value that names no directory says nothing of where the package lies - case at == meshcatalogclient.RootPath: - at = "" - case dir != at && !strings.HasPrefix(dir, at+"/"): - continue - } - if d := len(at); d > depth { - commit, depth = m.Commit, d - } - } - } - if commit == "" { + key := HolderOf(pkg, modules) + if key == "" { missing = append(missing, pkg) continue } if versions == nil { versions = sdkstep.PackageVersions{} } - versions[pkg] = commit + versions[pkg] = modules[key].Commit } slices.Sort(missing) return versions, missing } +// HolderOf is the key of the held module of the package's repository whose directory holds the package's directory (the +// deepest, a module at the repository's root holding every directory) and that names a commit; "" for none. +func HolderOf(pkg string, modules map[string]meshcatalog.ModulesByModuleValue) string { + repository, dir, ok := trunk.PackageRepository(pkg) + if !ok { + return "" + } + holder, depth := "", -1 + for _, key := range slices.Sorted(maps.Keys(modules)) { + m := modules[key] + if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" { + continue + } + at := strings.Trim(m.Path, "/") + switch { + case at == "": + continue // a value that names no directory says nothing of where the package lies + case at == meshcatalogclient.RootPath: + at = "" + case dir != at && !strings.HasPrefix(dir, at+"/"): + continue + } + if d := len(at); d > depth { + holder, depth = key, d + } + } + return holder +} + // digestOf is the content digest of the package at import path pkg in a published module, and whether it holds it // (its packages are sorted by import path). func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) { @@ -455,3 +481,21 @@ func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) { // NamedTreePrefix opens a module's tree when it is computed over the paths its artifacts name, not the directory's own // git tree (novox/hq ADR 0499 clause 2): the indexer alone computes it (NamedTree). const NamedTreePrefix = "named:" + +// WithPins is the fingerprint fp with the pins a build of it is made against (design 47 H-A-167): fp itself when no +// package is pinned, else `src2:` and the hex sha256 of fp and, sorted, the lines "pin " of each pin, +// joined by a newline. A package kept with no version (none released) is no pin. +func WithPins(fp string, pins sdkstep.PackageVersions) string { + var lines []string + for p, v := range pins { + if v != "" { + lines = append(lines, "pin "+p+" "+v) + } + } + if fp == "" || len(lines) == 0 { + return fp + } + sort.Strings(lines) + sum := sha256.Sum256([]byte(fp + "\n" + strings.Join(lines, "\n"))) + return meshcatalogclient.SourcePrefix2 + hex.EncodeToString(sum[:]) +} diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go index f143ee14..6a1d68bb 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go @@ -31,6 +31,20 @@ func versionLineTakesASiblingDirectoryByItsContent(t *testing.T) { if got := ReplacedSiblings(goMod, repo); len(got) != 1 || got[0] != buildrun { t.Fatalf("siblings %v, want buildrun alone (mesh-sdk is another repository, replaced by a version)", got) } + // Its mesh packages are every mesh package its go.mod requires, at any version, the zero version included (design 47 + // H-A-157); a replace line or a module outside the forge is none. + for _, c := range []struct { + goMod string + want []string + }{ + {string(goMod), []string{buildrun, sdk}}, + {"module x\n\nrequire " + sdk + " v0.0.0-00010101000000-000000000000 // indirect\nrequire github.com/nats-io/nats.go v1.37.0\n", []string{sdk}}, + {"module x\n\nreplace " + sdk + " => ../sdk\n", nil}, + } { + if got := MeshRequiresOf([]byte(c.goMod)); strings.Join(got, ",") != strings.Join(c.want, ",") { + t.Errorf("mesh packages of %q: %v, want %v", c.goMod, got, c.want) + } + } published := func(digest string) map[string]meshcatalog.PackagesByModuleValue { return map[string]meshcatalog.PackagesByModuleValue{buildrun: {Module: buildrun, Repository: repo, Requires: "r1", Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: buildrun, Digest: digest}}}} diff --git a/modules/mesh-delivery-indexer/internal/index/form.go b/modules/mesh-delivery-indexer/internal/index/form.go index 96636838..133f42e1 100644 --- a/modules/mesh-delivery-indexer/internal/index/form.go +++ b/modules/mesh-delivery-indexer/internal/index/form.go @@ -17,6 +17,11 @@ import ( type form struct { fp, none string versions sdkstep.PackageVersions + // stands: fp is the held fingerprint of the build that reported the imports, made with the pins built (its held + // packages) from the import form raw; pinnedOf keeps it while the pins are those (H-A-172). + stands bool + raw string + built map[string]string } // formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the @@ -24,15 +29,16 @@ type form struct { // // - **The import form**, for a module whose last trunk build reported its imports (kept.Imports) when every import is // published (the packages state): the content digest of each imported package and the requirements' digest of each -// mesh Go module holding one. Its versions are each holding module's published commit (another repository's only). +// mesh Go module holding one. Its versions are each holding module's published commit (another repository's only); an +// import set naming none of another repository while the go.mod requires one keeps the version line's (refined). // - **A change of form never rebuilds by itself** (§4): kept.ImportForm is the import-form fingerprint of the inputs // the build that reported the imports was built from. While the import form computes to it, nothing the build was // built from changed: the module keeps the fingerprint it holds (prev), in whatever form, and its versions. // Otherwise the import form is written, and ImportForm moves with it. // - **The version line** otherwise (no import set, or an import none publishes): the version of each mesh Go module of -// another repository its go.sum names, read as the indexer always read it (fingerprint.HeldVersionsOf), and the -// content of each mesh Go module of its own repository its go.mod replaces by a directory (fingerprint.VersionLine), -// so a change of that directory alone moves it (review finding 2). Its versions are the other repositories' alone. +// another repository its go.mod requires, at any version (design 47 H-A-157), read as the indexer always read it +// (fingerprint.HeldVersionsOf), and the content of each mesh Go module of its own repository its go.mod replaces by a +// directory (fingerprint.VersionLine), so a change of that directory alone moves it (review finding 2). Its versions are the other repositories' alone. // A package of another repository none of them publishes takes the commit the gate's source names for its repository // (heads, design 47 G A-365). // - **An accepted series** (DLV-I6): published and pins come from acceptedSeries, so a mesh Go module of another @@ -51,10 +57,12 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp case fp == "": return form{none: none} case kept.ImportForm == fp && prev.Fp2 != "": - return form{fp: prev.Fp2, versions: prev.Packages} // the build that reported the imports stands + // The build that reported the imports stands, and so do its versions. + return form{fp: prev.Fp2, versions: refined(sdkstep.PackageVersions(prev.Packages), r.Versions, module, repository, kept, held, pins, heads), + stands: true, raw: fp, built: prev.Packages} } kept.ImportForm = fp - return form{fp: fp, versions: r.Versions} + return form{fp: fp, versions: refined(r.Versions, nil, module, repository, kept, held, pins, heads)} } if logf != nil { logf("%s: no published package holds %s, so its fingerprint takes the version line", module, strings.Join(r.Missing, ", ")) @@ -63,6 +71,34 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp // The fingerprint written here is the version line's, not the import form's: no import form is held for the inputs it was // computed from, so a later pass whose import form computes the same does not take this one for the build's (issue 1966). kept.ImportForm = "" + versions := lineVersions(module, repository, kept, held, pins, heads, logf) + in.Packages = fingerprint.VersionLine(repository, kept.Packages, versions, published) + fp, none := fingerprint.Of(in) + if fp == "" { + return form{none: none} + } + return form{fp: fp, versions: versions} +} + +// refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod +// requires a mesh package of another repository (kept.Packages), the version line's, so a build's reported imports +// refine the go.mod's set and never empty it (design 47 H-A-157); nil when nothing names one. +func refined(versions, then sdkstep.PackageVersions, module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) sdkstep.PackageVersions { + switch { + case len(versions) > 0: + return versions + case len(then) > 0: + return then + case len(fingerprint.OtherPackages(kept.Packages, repository)) > 0: + return lineVersions(module, repository, kept, held, pins, heads, nil) + } + return versions +} + +// lineVersions are the version line's versions of the mesh Go modules of other repositories the module's go.mod requires +// (kept.Packages): each as the indexer has always read it, an accepted series' pin over it, else the commit the gate's +// source names for its repository. +func lineVersions(module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, logf func(string, ...any)) sdkstep.PackageVersions { others := fingerprint.OtherPackages(kept.Packages, repository) versions, unpublished := fingerprint.HeldVersionsOf(others, held) for p, v := range pins { // an accepted series' newest published version (acceptedSeries, DLV-I6) @@ -90,12 +126,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp if len(unpublished) > 0 && logf != nil { logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", ")) } - in.Packages = fingerprint.VersionLine(repository, kept.Packages, versions, published) - fp, none := fingerprint.Of(in) - if fp == "" { - return form{none: none} - } - return form{fp: fp, versions: versions} + return versions } // inputsFrom is the fingerprint's inputs of module m from its kept inputs and the given toolchains, no package of diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 4f308265..0bcf5888 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -118,10 +118,27 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T t.Fatal(err) } moved, _ := moduleOf(t, mods, "avahi") - if moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 { - t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3", + if pin, kept := moved.Packages[meshSDK]; moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || !kept || pin != "" { + t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form, mesh-sdk kept unpinned (none released, H-A-161)", moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages) } + + // mesh-sdk releases 0.1.20: the import form's fingerprint takes the pin too and moves, once (H-A-172). + ix.Versions = newMem() + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + for range 2 { + if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } + } + if m, _ := moduleOf(t, mods, "avahi"); m.Fp2 == moved.Fp2 || m.Fp2 == "" || m.Packages[meshSDK] != "v0.1.20" || m.Inputs2.ImportForm != moved.Inputs2.ImportForm { + t.Fatalf("after mesh-sdk's release: fp2 %q (was %q), versions %v, import form %q; want a new fingerprint pinned to v0.1.20, the import form kept", + m.Fp2, moved.Fp2, m.Packages, m.Inputs2.ImportForm) + } else if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } else if again, _ := moduleOf(t, mods, "avahi"); again.Fp2 != m.Fp2 { + t.Fatalf("a pass with nothing moved moved the pinned fingerprint %q to %q", m.Fp2, again.Fp2) + } } // A build of a fingerprint the module no longer holds says nothing of its inputs now: its imports are not kept. @@ -256,8 +273,8 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) { } } before := step("trunk at c1", trunk(c1)) - if before.Packages[meshSDK] != c1 { - t.Fatalf("with no series published, versions %v, want the commit form c1", before.Packages) + if pin, kept := before.Packages[meshSDK]; !kept || pin != "" { + t.Fatalf("with no series published, versions %v, want mesh-sdk kept with no pin, never the commit (H-A-161)", before.Packages) } step("the build reports its imports", func() error { return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.Fp2, meshSDK, meshSDK+"/announce")) @@ -272,6 +289,12 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) { if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" { t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages) } + // mesh-sdk releases 0.1.20: an accepted series' older version never stands over a newer release (H-A-167). + ix.Versions = newMem() + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + if m := step("0.1.20 released", func() error { return nil }); m.Packages[meshSDK] != "v0.1.20" { + t.Errorf("a newer release left avahi on its series' older version: %v", m.Packages) + } } // Issue 1966 (the indexer started while the bus's storage was down): a read of the packages state that failed, or that @@ -370,22 +393,36 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * head := strings.Repeat("c", 40) contracts := "git.novox.be/novox/mesh-contracts/go/seats/findings" held := map[string]meshcatalog.ModulesByModuleValue{"other": {Repository: "novox/mesh-sdk", Commit: strings.Repeat("d", 40), Path: "go"}} + // A build's imports of its own repository alone (design 47 H-A-157): they refine the go.mod's set and never empty it. + own := "git.novox.be/novox/mesh-catalog/modules/avahi/lib" + published := map[string]meshcatalog.PackagesByModuleValue{own: {Module: own, Repository: "novox/mesh-catalog", Commit: head, Requires: "r1", + Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: own, Digest: "d1"}}}} for _, c := range []struct { - name string - heads map[string]string - held map[string]meshcatalog.ModulesByModuleValue - want string + name string + heads map[string]string + held map[string]meshcatalog.ModulesByModuleValue + imports []string + want string }{ - {"the source names its repository", map[string]string{"novox/mesh-contracts": head}, held, head}, - {"the source names another repository only", map[string]string{"novox/mesh-sdk": head}, held, ""}, + {"the source names its repository", map[string]string{"novox/mesh-contracts": head}, held, nil, head}, + {"the source names another repository only", map[string]string{"novox/mesh-sdk": head}, held, nil, ""}, {"a module the indexer holds publishes it", map[string]string{"novox/mesh-contracts": head}, - map[string]meshcatalog.ModulesByModuleValue{"c": {Repository: "novox/mesh-contracts", Commit: strings.Repeat("e", 40), Path: "go/seats/findings"}}, strings.Repeat("e", 40)}, + map[string]meshcatalog.ModulesByModuleValue{"c": {Repository: "novox/mesh-contracts", Commit: strings.Repeat("e", 40), Path: "go/seats/findings"}}, nil, strings.Repeat("e", 40)}, + {"a build's imports naming none of another repository keep the go.mod's", map[string]string{"novox/mesh-contracts": head}, held, []string{own}, head}, } { - kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t", Packages: []string{contracts}} - f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, c.held, nil, nil, c.heads, nil) + kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t", Packages: []string{contracts}, Imports: c.imports} + f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, c.held, published, nil, c.heads, nil) if got := f.versions[contracts]; got != c.want { t.Errorf("%s: version of %s = %q, want %q", c.name, contracts, got, c.want) } + if c.imports == nil { + continue + } + // The build that reported them stands, its held versions empty (a version line read from go.sum). + f = formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("b", 64)}, c.held, published, nil, c.heads, nil) + if got := f.versions[contracts]; got != c.want || f.fp != "src2:"+strings.Repeat("b", 64) { + t.Errorf("%s, the build standing: version of %s = %q (fp %q), want %q", c.name, contracts, got, f.fp, c.want) + } } }) t.Run("formOf clears the import form when it writes the version line", func(t *testing.T) { diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 05638826..0b130d11 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -108,7 +108,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" } else { - packages := ix.goSumPackages(ctx, f, dir, cur, has) + packages := ix.goPackages(ctx, f, dir, cur, has) in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx)) next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths if in.Kept != nil && has && cur.Inputs2 != nil { @@ -365,79 +365,72 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string return in } -// goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go -// modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory -// (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none. -func (ix *Indexer) goSumPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { - others := ix.otherPackages(ctx, f, dir, cur, has) +// goPackages are the version line's packages of the merge's directory dir (ADR 0593 §2; design 47 H-A-157): the mesh Go +// modules of other repositories its go.mod requires, at any version, and those of its own repository its go.mod replaces +// by a directory (siblingPackages, H-A-136's sibling rule). A directory with no go.sum is no Go module and has none. +func (ix *Indexer) goPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 { - return others + return nil + } + ix.requirementsOf(ctx, f, dir, cur, has) + goMod, read := ix.goModOf(ctx, f, dir) + others := fingerprint.OtherPackages(f.Packages[dir], f.Repository) // carried: what the go.mod requires, read with the tree + switch { + case len(others) > 0: + case read: + others = fingerprint.OtherPackages(fingerprint.MeshRequiresOf(goMod), f.Repository) + case has && cur.Inputs2 != nil: + others = fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) + } + var siblings []string + if read { + siblings = fingerprint.ReplacedSiblings(goMod, f.Repository) + } else if has && cur.Inputs2 != nil { + for _, p := range cur.Inputs2.Packages { + if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) { + siblings = append(siblings, p) + } + } } - siblings := ix.siblingPackages(ctx, f, dir, cur, has) if len(siblings) == 0 { return others } return slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(others), siblings...)))) } -// siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read -// from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said. -func (ix *Indexer) siblingPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { - held := func() []string { - if !has || cur.Inputs2 == nil { - return nil - } - var out []string - for _, p := range cur.Inputs2.Packages { - if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) { - out = append(out, p) - } - } - return out - } +// goModOf is the go.mod of the merge's directory dir, read from the forge (GoMod), and whether it was read; where it +// cannot be, the held module's packages stand, said. +func (ix *Indexer) goModOf(ctx context.Context, f whole, dir string) ([]byte, bool) { if ix.GoMod == nil { - return held() + return nil, false } raw, err := ix.GoMod(ctx, f.Repository, f.Merge, dir) if err != nil { - ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the directory replaces it held: %v", f.Repository, f.Merge, dir, err) - return held() + ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err) + return nil, false } - return fingerprint.ReplacedSiblings(raw, f.Repository) + return raw, true } -// otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the -// message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read -// from the forge (GoSum). -func (ix *Indexer) otherPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { - if carried := f.Packages[dir]; len(carried) > 0 { - return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree - } +// requirementsOf keeps what the go.sum of the merge's directory dir requires in the requirements state +// (keepRequirements): read from the forge (GoSum) when its hash moved, or this process has not kept it yet, whatever +// packages the reader carried. It gives no package: those are the go.mod's (H-A-157). +func (ix *Indexer) requirementsOf(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) { hash := f.Locks[dir]["go.sum"] - if hash == "" { - return nil + if hash == "" || ix.GoSum == nil { + return } if has && cur.Inputs2 != nil && trunk.SameRepository(cur.Repository, f.Repository) && ix.requirementsKept(f.Repository, dir) { for _, h := range cur.Inputs2.Locks { if h == hash { - return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) + return } } } - if ix.GoSum == nil { - if has && cur.Inputs2 != nil { - return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) - } - return nil - } raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir) if err != nil { - ix.logf("%s at %s: the go.sum of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err) - if has && cur.Inputs2 != nil { - return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) - } - return nil + ix.logf("%s at %s: the go.sum of %q cannot be read, so what it requires is not kept: %v", f.Repository, f.Merge, dir, err) + return } ix.keepRequirements(ctx, f.Repository, dir, raw) - return fingerprint.OtherPackages(fingerprint.MeshPackagesOf(raw), f.Repository) } diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 43521238..d765110e 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -401,8 +401,36 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means } func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6 - ix, mods, _ := manifestsIndexer(t) // no build record of any module avahi := catalogueManifest(t, "avahi") + // Its go.mod requires mesh-sdk at the zero version (ADR 0573 §2): the go.mod's require lines are its mesh packages, with + // no build, carried by the reader or read through GoMod (design 47 H-A-157). Its go.sum is still read for the + // requirements state, carried packages or not, so an older version it names is kept there (and cleared when gone). + for _, carried := range [][]string{nil, {meshSDK}} { + ix, mods, _ := manifestsIndexer(t) + ix.Requirements = newMem() + ix.GoMod = func(context.Context, string, string, string) ([]byte, error) { + return []byte("module avahi\n\nrequire " + meshSDK + " v0.0.0-00010101000000-000000000000\n"), nil + } + ix.GoSum = func(context.Context, string, string, string) ([]byte, error) { + return []byte(meshSDK + " v0.1.14 h1:x=\n"), nil + } + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, + map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + if carried != nil { + f.Packages = map[string][]string{"modules/avahi": carried} + } + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, _ := moduleOf(t, mods, "avahi") + if m.Inputs2 == nil || !slices.Equal(m.Inputs2.Packages, []string{meshSDK}) { + t.Errorf("carried %v: mesh packages %+v, want mesh-sdk, which its go.mod requires at the zero version", carried, m.Inputs2) + } + if _, kept := ix.Requirements.(*memStore).vals[meshSDK]; !kept { + t.Errorf("carried %v: the requirements state holds no %s, which the go.sum names", carried, meshSDK) + } + } + ix, mods, _ := manifestsIndexer(t) // no build record of any module f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) if err := ix.Apply(context.Background(), f); err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/requirements.go b/modules/mesh-delivery-indexer/internal/index/requirements.go index e0624c9e..f1b2ff04 100644 --- a/modules/mesh-delivery-indexer/internal/index/requirements.go +++ b/modules/mesh-delivery-indexer/internal/index/requirements.go @@ -41,7 +41,7 @@ func (r Requirement) Versions() []string { return out } -// RequiredOf is, per mesh Go module a go.sum names (as fingerprint.MeshPackagesOf takes them), the highest version it +// RequiredOf is, per mesh Go module a go.sum names (a line under trunk.ForgeModulePrefix whose path names a repository), the highest version it // names, its module line or its go.mod line alike. func RequiredOf(goSum []byte) map[string]string { out := map[string]string{} diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index 1aed0f99..9667902c 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -108,7 +108,7 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. seat := seatOf[a.Ref] sa := SeedArtifact{Name: name, Seat: seat} in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp, - Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: maps.Clone(e.Packages)}} + Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: PinsOf(e.Packages)}} // The module's directory and its named paths go together, only when the entry names its paths (ADR 0499 clause // 4); a request without them builds from the module's directory (sdk/buildstep FetchRequest). if paths := namedPaths(e, name); len(paths) > 0 { diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go index 22c7b3a9..6824bb67 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go @@ -24,7 +24,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { fp := "src2:" + strings.Repeat("a", 64) entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}}, - Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"}, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "v0.1.9", "git.novox.be/novox/mesh-contracts/go": ""}, // the second none released (H-A-161) Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}} rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) if err != nil { @@ -71,9 +71,12 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { t.Errorf("the image artifact is %+v", a) } in := got[0] - if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" { + if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "v0.1.9" { t.Errorf("the steps input is %+v", in) } + if _, pinned := in.Fetch.Packages["git.novox.be/novox/mesh-contracts/go"]; pinned { + t.Errorf("a package kept with no version is pinned in the fetch: %v", in.Fetch.Packages) + } if len(w.Held()) != 0 { t.Fatal("a call unanswered for no time wrote its finding") } diff --git a/modules/mesh-delivery-indexer/internal/index/takenin.go b/modules/mesh-delivery-indexer/internal/index/takenin.go index e899a25f..41a95ccd 100644 --- a/modules/mesh-delivery-indexer/internal/index/takenin.go +++ b/modules/mesh-delivery-indexer/internal/index/takenin.go @@ -9,7 +9,7 @@ type RepoRead struct { Trees map[string]string Manifests map[string]string Locks map[string]map[string]string - // Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2). + // Packages are the mesh packages each directory's go.mod requires (fingerprint.MeshRequiresOf, design 47 H-A-157). Packages map[string][]string } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index ae5b21e4..f3b72e2f 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -16,6 +16,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/catalogue" + "mesh-delivery-indexer/internal/fingerprint" "mesh-delivery-indexer/judge" ) @@ -152,11 +153,13 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]mesh } ix.recompute.Lock() defer ix.recompute.Unlock() - return ix.recomputeLocked(ctx, overlay, w, repository) + return ix.recomputeLocked(ctx, overlay, w, repository, nil) } // recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go). -func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { +// +// only, when not nil, names the modules the pass moves (a re-pin, H-A-165): every other stands as held. +func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string, only map[string]bool) error { if ix.Manifests == nil { return ix.writeOverlay(ctx, overlay, nil, w) } @@ -215,7 +218,8 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc ix.logf("no fingerprint is written by this pass: %v", err) return fmt.Errorf("no fingerprint is written: %w", err) } - series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6) + series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6) + released := ix.releasedOf(ctx) // each module's newest released version, what a package is pinned to (H-A-161) type result struct { fp, none string tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint @@ -287,18 +291,20 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.none = strings.Join(missing, "; ") case noToolchain != "": r.none = noToolchain - if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads()); ok { + if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), released); ok { r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs} } default: f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.sourceHeads(), ix.logf) r.fp, r.none = f.fp, f.none if r.fp != "" { - r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions + // a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167) + r.fp, r.pkgs = pinnedOf(f, held, released) + r.tcs = toolchainsUsed(toolchains) } } - if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && - maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { + if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && + !importsOf(r.pkgs, repository) || only != nil && !only[module]) { // A module registered for another repository is moved by this repository's commit only when it imports a package the // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's. r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} @@ -492,7 +498,7 @@ func hasContext(m catalogue.Manifest) bool { // with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any // other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the // module has none until they agree. -func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) (string, sdkstep.PackageVersions, bool) { +func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, released map[string]string) (string, sdkstep.PackageVersions, bool) { if k.held.Fp2 == "" { return "", nil, false } @@ -509,7 +515,11 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} } f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, heads, nil) - return f.fp, f.versions, f.none == "" && f.fp == k.held.Fp2 + if f.none != "" || f.fp == "" { + return "", nil, false + } + fp, pkgs := pinnedOf(f, held, released) + return fp, pkgs, fp == k.held.Fp2 } // toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's). @@ -577,3 +587,41 @@ func (ix *Indexer) fillPaths(ctx context.Context) error { } return nil } + +// importsOf says whether pkgs name a package of repository: a pin is a released version (H-A-161), so a commit of +// repository moves a module of another only through a package of it the module imports (A-476a). +func importsOf(pkgs sdkstep.PackageVersions, repository string) bool { + for p := range pkgs { + if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, repository) { + return true + } + } + return false +} + +// Repin moves the pins of every module whose packages hold a package of module (a held module's directory holds it) to +// module's newest released version, the pass of H-A-161 for those modules alone: a version-released of module starts it +// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It answers how many it named. +func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) { + if ix.Modules == nil || ix.Manifests == nil { + return 0, nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return 0, err + } + only := map[string]bool{} + for key, m := range held { + for p := range m.Packages { + if moduleOfKey(fingerprint.HolderOf(p, held)) == module { + only[key] = true + } + } + } + if len(only) == 0 { + return 0, nil + } + ix.recompute.Lock() + defer ix.recompute.Unlock() + return len(only), ix.recomputeLocked(ctx, nil, nil, "", only) +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 6357bc82..281540a7 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -757,7 +758,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.Fp2, u.None2) } tools, _ := moduleOf(t, mods, "mesh-tools") - sdkMerge := func(commit, tree string) meshcatalog.ModulesByModuleValue { + sdkMerge := func(commit, tree, pin string) meshcatalog.ModulesByModuleValue { t.Helper() f := whole{Repository: "novox/mesh-sdk", Branch: "feat/x", Head: h1, Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}} @@ -765,9 +766,10 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { t.Fatal(err) } // A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools, - // which imports nothing it moved. + // which imports nothing it moved. Its pin is mesh-sdk's newest released version, never the commit; with none + // released it has no pin, the package kept for its kind-1 edge (H-A-161). a, _ := moduleOf(t, mods, "avahi") - if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) { + if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: pin}) { t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages) } if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 || o.Packages != nil { @@ -781,8 +783,39 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { stale.Fp2 = "src2:" + strings.Repeat("5", 64) mods.put("mesh-tools", mustEncode(t, stale)) tools = stale - first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1") - second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2") + first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "") + // mesh-sdk releases 0.1.20 (0.1.9 before it): the release re-pins avahi with no merge, its fingerprint moving with the + // pin so it is built again against the release, and leaves mesh-tools, which holds none of it (H-A-165, H-A-167). A + // re-pin that fails is no release acknowledged: the package-published delivered again re-pins. + ix.Versions, ix.Heard = newMem(), newMem() + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`)) + revisions := ix.Revisions + ix.Revisions = newMemObjects(1 << 20) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "mesh-sdk:go"}}}}) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(context.Background(), ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + release := meshbuilds.PackagePublished{Module: "mesh-sdk", Artifact: "mesh-sdk:go", Digest: "sha256:g", Version: "0.1.20", Source: 7, Branch: "main"} + mods.keysErr = errors.New("the bus did not answer") + if err := ix.HearPackagePublished(context.Background(), release); err == nil { + t.Fatal("a release whose re-pin failed was acknowledged") + } + mods.keysErr = nil + if err := ix.HearPackagePublished(context.Background(), release); err != nil { + t.Fatal(err) + } + ix.Revisions = revisions + if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == first.Fp2 || a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) { + t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want a new fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages) + } + if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 { + t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2) + } + second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2", "v0.1.20") if second.Fp2 == first.Fp2 { t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2) } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 574a6fb5..1fcae613 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -6,14 +6,19 @@ import ( "encoding/json" "errors" "fmt" + "maps" "slices" "strconv" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" + + "mesh-delivery-indexer/internal/fingerprint" ) // The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's @@ -123,8 +128,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil { return err } else if found && !isFloor(raw) { - ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact) - return nil + // A package-published delivered again after a re-pin failed (H-A-167): the re-pin is made again, nothing else. + ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and only re-pins", p.Module, p.Version, p.Artifact) + return ix.repin(ctx, p.Module, p.Version) } entry, err := ix.entryOf(ctx, p.Module) if err != nil { @@ -234,6 +240,13 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli } ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source) } + if !late { + // The release re-pins every module whose packages hold one of it (design 47 H-A-165); one that fails is not + // acknowledged, so the package-published is delivered again (H-A-167). + if err := ix.repin(ctx, p.Module, p.Version); err != nil { + return err + } + } if err := ix.trimVersions(ctx, p.Module); err != nil { ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err) } @@ -352,3 +365,107 @@ func isFloor(raw []byte) bool { var v meshcatalog.VersionEntry return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0 } + +// releasedOf is the newest released version of each module the state versions keeps (§349; design 47 H-A-161), by module; +// nil when the indexer keeps no such state or it cannot be listed, said. +func (ix *Indexer) releasedOf(ctx context.Context) map[string]string { + if ix.Versions == nil { + return nil + } + all, err := ix.Versions.Keys(ctx) + if err != nil { + ix.logf("versions: the state cannot be listed, so no package is pinned by this pass: %v", err) + return nil + } + out := map[string]string{} + for _, k := range all { + m, v, ok := meshcatalog.ParseVersionsByModuleVersionKey(k) + if _, parts := versionParts(v); !ok || !parts { + continue + } + if cur, has := out[m]; !has || compareVersions(v, cur) > 0 { + out[m] = v + } + } + return out +} + +// releasedPins are a module's pins (its catalog entry's packages, which the builder builds it against) from the versions its +// fingerprint was computed with (design 47 H-A-161, G §370): a package the held module holding it has released is pinned +// to that module's newest released version, as Go names it (v); a version already in that form (an accepted +// series' newest, DLV-I6) stands; any other, a commit, is no pin: its package is kept with no version, so its kind-1 edge +// stands (H §746) and no build is made against a commit. nil for none. +func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) sdkstep.PackageVersions { + if len(versions) == 0 { + return nil + } + out := sdkstep.PackageVersions{} + for p, v := range versions { + rel := released[moduleOfKey(fingerprint.HolderOf(p, held))] + series := sdkstep.PackageSemver.MatchString(v) + _, plain := versionParts(strings.TrimPrefix(v, "v")) + switch { + case rel != "" && (!series || !plain || compareVersions(rel, strings.TrimPrefix(v, "v")) > 0): + out[p] = "v" + rel // never an older series' version over a newer release (H-A-167) + case series: + out[p] = v + default: + out[p] = "" + } + } + return out +} + +// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so every +// form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover +// built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that +// stands is kept while its pins are the ones it was built with, else its import form takes the new pins. +func pinnedOf(f form, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) (string, sdkstep.PackageVersions) { + pins := releasedPins(f.versions, held, released) + if f.stands { + if maps.Equal(f.built, map[string]string(pins)) { + return f.fp, pins + } + return fingerprint.WithPins(f.raw, pins), pins + } + return fingerprint.WithPins(f.fp, pins), pins +} + +// moduleOfKey is the module a modules key names; "" for none. +func moduleOfKey(key string) string { + if key == "" { + return "" + } + if m, err := meshcatalogclient.ParseModulesKey(key); err == nil { + return m + } + return key +} + +// PinsOf are the pins a build is asked with from a catalog entry's packages: each package with a version; one kept with +// none (no released version, H-A-161) is no pin. nil for none. +func PinsOf(packages map[string]string) sdkstep.PackageVersions { + var out sdkstep.PackageVersions + for p, v := range packages { + if v == "" { + continue + } + if out == nil { + out = sdkstep.PackageVersions{} + } + out[p] = v + } + return out +} + +// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error. +func (ix *Indexer) repin(ctx context.Context, module, version string) error { + n, err := ix.Repin(ctx, module) + if err != nil { + return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err) + } + if n > 0 { + ix.logf("versions: %d modules re-pinned to %s %s", n, module, version) + } + return nil +}