diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 1dd3ce4a..04da28ad 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -182,9 +182,9 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit, var lockFiles = []string{"go.sum", "package-lock.json"} // readVia reads a repository at a commit as the forge bridge does for a merged message: the seat git's trees, then the -// module.json and lock files of the one directory dir by its file (a repository can hold hundreds). A manifest the forge -// does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does -// not have is left out. +// module.json and lock files of the one directory dir by its file (a repository can hold hundreds), and the go.mod beside a +// go.sum for its mesh packages (Packages). A manifest the forge does not have is an error; a directory with no module.json in +// the trees is answered with no manifest; a lock file it does not have is left out. func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) { return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) { a, err := treesOf(call, repository, git.TreesInput{Commit: sha}) @@ -222,13 +222,20 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str out.Locks[dir] = map[string]string{} } out.Locks[dir][name] = hex.EncodeToString(sum[:]) - if name == "go.sum" { - if p := fingerprint.MeshPackagesOf([]byte(c)); len(p) > 0 { - if out.Packages == nil { - out.Packages = map[string][]string{} - } - out.Packages[dir] = p // the mesh packages it uses, as the bridge reads them (ADR 0573 §2) + if name != "go.sum" { + continue + } + // A Go module's mesh packages are what its go.mod requires, at any version (design 47 H-A-157): its go.sum + // names none required at the zero version (ADR 0573 §2). + mod, absent, err := file(dir, "go.mod") + if err != nil { + return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "go.mod"), err) + } + if p := fingerprint.MeshRequiresOf([]byte(mod)); !absent && len(p) > 0 { + if out.Packages == nil { + out.Packages = map[string][]string{} } + out.Packages[dir] = p } } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index ab3faf53..c3d723f4 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -120,6 +120,8 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { return json.RawMessage(`{"path":"module.json","content":"{\"module\":\"snake\"}"}`), nil case "go.sum": return json.RawMessage(`{"path":"go.sum","content":"x"}`), nil + case "go.mod": // a mesh package required at the zero version, which no go.sum line names (design 47 H-A-157) + return json.RawMessage(`{"path":"go.mod","content":"module snake\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n)\n"}`), nil } return json.RawMessage(`{"path":"x","absent":true}`), nil } @@ -129,12 +131,15 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others + if len(got.Manifests) != 1 || len(files) != 4 { // the one directory asked: its manifest, two lock files and the go.mod, never the others t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files) } if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" { t.Fatalf("read = %+v", got) } + if p := got.Packages[""]; len(p) != 1 || p[0] != "git.novox.be/novox/mesh-sdk/go" { + t.Fatalf("mesh packages %v, want mesh-sdk, which the go.mod requires", got.Packages) + } } // fakeRuns records the run events handed to it and answers err. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index b50fe15a..603c875e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -58,7 +58,8 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name } // TestASourceRevisionMovingMeshSDKNamesItAndItsImporters (design 47 H §743; A-476a; ADR 0593 §2 to §4): the indexer whole against -// the bus. Two Go modules of the catalogue, importer, whose go.sum requires mesh-sdk's Go module, and plain, which requires +// the bus. Two Go modules of the catalogue, importer, whose go.mod requires mesh-sdk's Go module at the zero version (no +// go.sum line names it, ADR 0573 §2; design 47 H-A-157), and plain, which requires // nothing of it, are taken in with mesh-sdk as main's first source revision. A source revision moving mesh-sdk to a commit // changing its buildstep package is taken in as one revision whose movers are mesh-sdk and importer, of another repository, // and not plain; the packages key of mesh-sdk/go names that commit with a digest per package. @@ -67,7 +68,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { sdkGo = "git.novox.be/novox/mesh-sdk/go" m1, m2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222" catalogMerge = "3333333333333333333333333333333333333333" - goSumContent = "git.novox.be/novox/mesh-sdk/go v0.1.14 h1:x=\ngit.novox.be/novox/mesh-sdk/go v0.1.14/go.mod h1:y=\n" + goModContent = "module importer\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n" sdkRoot1, sdk2 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ) avahi, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") @@ -119,7 +120,8 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { forge.put(m2, sdkAt(sdk2, "f6", "7777777777777777777777777777777777777777")) forge.put(catalogMerge, repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, files: map[string]string{"modules/importer/module.json": named("importer"), "modules/plain/module.json": named("plain"), - "modules/importer/go.sum": goSumContent, "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}}) + "modules/importer/go.mod": goModContent, "modules/importer/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n", + "modules/plain/go.mod": "module plain\n\nrequire golang.org/x/sys v0.1.0\n", "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}}) regs := registerAvahi(t, ctx, js, "main") for _, r := range []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}}, {V: 1, Module: "plain", Origin: "novox/mesh-catalog", Path: "modules/plain", Branches: []string{"main"}}, diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go index b5309795..4a326972 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -135,14 +135,32 @@ func NamedTree(manifest string, paths map[string]string) (string, error) { return NamedTreePrefix + hex.EncodeToString(sum[:]), nil } -// MeshPackagesOf are the mesh Go modules a go.sum names (the module path of each line under -// trunk.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set -// takes the version line of (ADR 0593 §2). -func MeshPackagesOf(goSum []byte) []string { +// MeshRequiresOf are the mesh Go modules a go.mod requires (the module path of each `require` line under +// trunk.ForgeModulePrefix, alone or in a block, at any version, the zero version and `// indirect` included), sorted and +// each once; nil for none. They are a module's mesh packages from the take-in, with no build needed (design 47 H-A-157): +// a go.sum names none required at the zero version (ADR 0573 §2), so it is never read for them. +func MeshRequiresOf(goMod []byte) []string { seen := map[string]bool{} - for _, line := range strings.Split(string(goSum), "\n") { + inBlock := false + for _, line := range strings.Split(string(goMod), "\n") { + line = strings.TrimSpace(line) + if i := strings.Index(line, "//"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + switch { + case line == "require (": + inBlock = true + continue + case inBlock && line == ")": + inBlock = false + continue + case strings.HasPrefix(line, "require "): + line = strings.TrimSpace(strings.TrimPrefix(line, "require ")) + case !inBlock: + continue + } f := strings.Fields(line) - if len(f) < 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) { + if len(f) != 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) { continue } if _, _, ok := trunk.PackageRepository(f[0]); ok { diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go index f143ee14..6a1d68bb 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go @@ -31,6 +31,20 @@ func versionLineTakesASiblingDirectoryByItsContent(t *testing.T) { if got := ReplacedSiblings(goMod, repo); len(got) != 1 || got[0] != buildrun { t.Fatalf("siblings %v, want buildrun alone (mesh-sdk is another repository, replaced by a version)", got) } + // Its mesh packages are every mesh package its go.mod requires, at any version, the zero version included (design 47 + // H-A-157); a replace line or a module outside the forge is none. + for _, c := range []struct { + goMod string + want []string + }{ + {string(goMod), []string{buildrun, sdk}}, + {"module x\n\nrequire " + sdk + " v0.0.0-00010101000000-000000000000 // indirect\nrequire github.com/nats-io/nats.go v1.37.0\n", []string{sdk}}, + {"module x\n\nreplace " + sdk + " => ../sdk\n", nil}, + } { + if got := MeshRequiresOf([]byte(c.goMod)); strings.Join(got, ",") != strings.Join(c.want, ",") { + t.Errorf("mesh packages of %q: %v, want %v", c.goMod, got, c.want) + } + } published := func(digest string) map[string]meshcatalog.PackagesByModuleValue { return map[string]meshcatalog.PackagesByModuleValue{buildrun: {Module: buildrun, Repository: repo, Requires: "r1", Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: buildrun, Digest: digest}}}} diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 05638826..f259e2cc 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -108,7 +108,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" } else { - packages := ix.goSumPackages(ctx, f, dir, cur, has) + packages := ix.goPackages(ctx, f, dir, cur, has) in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx)) next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths if in.Kept != nil && has && cur.Inputs2 != nil { @@ -365,79 +365,72 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string return in } -// goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go -// modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory -// (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none. -func (ix *Indexer) goSumPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { - others := ix.otherPackages(ctx, f, dir, cur, has) +// goPackages are the version line's packages of the merge's directory dir (ADR 0593 §2; design 47 H-A-157): the mesh Go +// modules of other repositories its go.mod requires, at any version, and those of its own repository its go.mod replaces +// by a directory (siblingPackages, H-A-136's sibling rule). A directory with no go.sum is no Go module and has none. +func (ix *Indexer) goPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 { - return others + return nil + } + ix.requirementsOf(ctx, f, dir, cur, has) + goMod, read := ix.goModOf(ctx, f, dir) + others := fingerprint.OtherPackages(f.Packages[dir], f.Repository) // carried: what the go.mod requires, read with the tree + switch { + case len(others) > 0: + case read: + others = fingerprint.OtherPackages(fingerprint.MeshRequiresOf(goMod), f.Repository) + case has && cur.Inputs2 != nil: + others = fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) + } + var siblings []string + if read { + siblings = fingerprint.ReplacedSiblings(goMod, f.Repository) + } else if has && cur.Inputs2 != nil { + for _, p := range cur.Inputs2.Packages { + if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) { + siblings = append(siblings, p) + } + } } - siblings := ix.siblingPackages(ctx, f, dir, cur, has) if len(siblings) == 0 { return others } return slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(others), siblings...)))) } -// siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read -// from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said. -func (ix *Indexer) siblingPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { - held := func() []string { - if !has || cur.Inputs2 == nil { - return nil - } - var out []string - for _, p := range cur.Inputs2.Packages { - if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) { - out = append(out, p) - } - } - return out - } +// goModOf is the go.mod of the merge's directory dir, read from the forge (GoMod), and whether it was read; where it +// cannot be, the held module's packages stand, said. +func (ix *Indexer) goModOf(ctx context.Context, f whole, dir string) ([]byte, bool) { if ix.GoMod == nil { - return held() + return nil, false } raw, err := ix.GoMod(ctx, f.Repository, f.Merge, dir) if err != nil { - ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the directory replaces it held: %v", f.Repository, f.Merge, dir, err) - return held() + ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err) + return nil, false } - return fingerprint.ReplacedSiblings(raw, f.Repository) + return raw, true } -// otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the -// message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read -// from the forge (GoSum). -func (ix *Indexer) otherPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { - if carried := f.Packages[dir]; len(carried) > 0 { - return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree - } +// requirementsOf keeps what the go.sum of the merge's directory dir requires in the requirements state +// (keepRequirements): read from the forge (GoSum) when no mesh package was carried and its hash moved, or this process +// has not kept it yet. It gives no package: those are the go.mod's (H-A-157). +func (ix *Indexer) requirementsOf(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) { hash := f.Locks[dir]["go.sum"] - if hash == "" { - return nil + if len(f.Packages[dir]) > 0 || hash == "" || ix.GoSum == nil { + return } if has && cur.Inputs2 != nil && trunk.SameRepository(cur.Repository, f.Repository) && ix.requirementsKept(f.Repository, dir) { for _, h := range cur.Inputs2.Locks { if h == hash { - return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) + return } } } - if ix.GoSum == nil { - if has && cur.Inputs2 != nil { - return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) - } - return nil - } raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir) if err != nil { - ix.logf("%s at %s: the go.sum of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err) - if has && cur.Inputs2 != nil { - return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) - } - return nil + ix.logf("%s at %s: the go.sum of %q cannot be read, so what it requires is not kept: %v", f.Repository, f.Merge, dir, err) + return } ix.keepRequirements(ctx, f.Repository, dir, raw) - return fingerprint.OtherPackages(fingerprint.MeshPackagesOf(raw), f.Repository) } diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 43521238..445763d0 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -402,6 +402,14 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6 ix, mods, _ := manifestsIndexer(t) // no build record of any module + // Its go.mod requires mesh-sdk at the zero version, which its go.sum does not name (ADR 0573 §2): the go.mod's + // require lines are its mesh packages, with no build (design 47 H-A-157). + ix.GoMod = func(context.Context, string, string, string) ([]byte, error) { + return []byte("module avahi\n\nrequire " + meshSDK + " v0.0.0-00010101000000-000000000000\n"), nil + } + ix.GoSum = func(context.Context, string, string, string) ([]byte, error) { + return []byte("github.com/nats-io/nats.go v1.37.0 h1:x=\n"), nil + } avahi := catalogueManifest(t, "avahi") f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) @@ -412,6 +420,9 @@ func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t * if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != meshcatalogclient.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" { t.Fatalf("module = %+v (%v), want its directory, merge, manifest and inputs from the message alone", m, ok) } + if !slices.Equal(m.Inputs2.Packages, []string{meshSDK}) { + t.Errorf("mesh packages %v, want mesh-sdk, which its go.mod requires at the zero version", m.Inputs2.Packages) + } if m.Fp != "" || m.None != "" { t.Errorf("module = %+v, want no value of today's form (I4: it came from the build records)", m) } diff --git a/modules/mesh-delivery-indexer/internal/index/requirements.go b/modules/mesh-delivery-indexer/internal/index/requirements.go index e0624c9e..f1b2ff04 100644 --- a/modules/mesh-delivery-indexer/internal/index/requirements.go +++ b/modules/mesh-delivery-indexer/internal/index/requirements.go @@ -41,7 +41,7 @@ func (r Requirement) Versions() []string { return out } -// RequiredOf is, per mesh Go module a go.sum names (as fingerprint.MeshPackagesOf takes them), the highest version it +// RequiredOf is, per mesh Go module a go.sum names (a line under trunk.ForgeModulePrefix whose path names a repository), the highest version it // names, its module line or its go.mod line alike. func RequiredOf(goSum []byte) map[string]string { out := map[string]string{} diff --git a/modules/mesh-delivery-indexer/internal/index/takenin.go b/modules/mesh-delivery-indexer/internal/index/takenin.go index e899a25f..41a95ccd 100644 --- a/modules/mesh-delivery-indexer/internal/index/takenin.go +++ b/modules/mesh-delivery-indexer/internal/index/takenin.go @@ -9,7 +9,7 @@ type RepoRead struct { Trees map[string]string Manifests map[string]string Locks map[string]map[string]string - // Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2). + // Packages are the mesh packages each directory's go.mod requires (fingerprint.MeshRequiresOf, design 47 H-A-157). Packages map[string][]string }