diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 0b130d11..6efa9c20 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -63,6 +63,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma return w, err } objects, namedErr := ix.namedObjects(ctx, f) + trees := ix.goModTree(ctx, f) overlay := map[string]meshcatalog.ModulesByModuleValue{} dirOf := map[string]string{} named, every := map[string]bool{}, true // every module the message's manifests name, and whether each names one @@ -110,6 +111,17 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma } else { packages := ix.goPackages(ctx, f, dir, cur, has) in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx)) + if in.Kept != nil { + // A Go build resolves the same-repository modules its go.mod files require, transitively, from the tree: their + // directories' objects join the tree input of every form, never the entry's paths (design 47 H-A-202, H-A-172). + if m, err := judge.ForInputs([]byte(raw)); err == nil && usesGo(languagesOf(m)) { + if tree, none := ix.derivedTree(ctx, f, trees, m, dir, in.Kept.Tree); none != "" { + in.Kept, in.None2 = nil, none + } else { + in.Kept.Tree = tree + } + } + } next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths if in.Kept != nil && has && cur.Inputs2 != nil { // The imports its last trunk build reported stand until its next build reports again (ADR 0593 §2, §4). diff --git a/modules/mesh-delivery-indexer/internal/index/package_trees.go b/modules/mesh-delivery-indexer/internal/index/package_trees.go new file mode 100644 index 00000000..c5ffe23b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/package_trees.go @@ -0,0 +1,173 @@ +package index + +import ( + "context" + "fmt" + "path" + "path/filepath" + "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "mesh-delivery-indexer/internal/fingerprint" +) + +// goModTree reads the go.mod files of one merge's tree, each directory once (Indexer.GoMod), for the walk of the module +// graph a Go build resolves from the tree (design 47 H-A-202, H-A-157). +type goModTree struct { + ix *Indexer + ctx context.Context + f whole + read map[string][]byte // by directory from the repository's root; nil for none +} + +func (ix *Indexer) goModTree(ctx context.Context, f whole) *goModTree { + return &goModTree{ix: ix, ctx: ctx, f: f, read: map[string][]byte{}} +} + +// goMod is the go.mod of the directory dir (from the root, "" for the root) at the merge commit; nil when there is none +// or it cannot be read, said. +func (t *goModTree) goMod(dir string) []byte { + if raw, ok := t.read[dir]; ok { + return raw + } + var raw []byte + if t.ix.GoMod != nil { + var err error + if raw, err = t.ix.GoMod(t.ctx, t.f.Repository, t.f.Merge, dir); err != nil { + t.ix.logf("%s at %s: the go.mod of %q cannot be read, so no module of the tree is found there: %v", t.f.Repository, t.f.Merge, dir, err) + raw = nil + } + } + t.read[dir] = raw + return raw +} + +// sameRepositoryDir is the directory of the merge's tree that holds the Go module required: the required path starts with +// the repository's module prefix, and the directory it names holds a go.mod declaring that path (H-A-198, H-A-202). false +// for a module of another repository, or one the tree holds no go.mod of. +func (t *goModTree) sameRepositoryDir(required string) (string, bool) { + repo, dir, ok := trunk.PackageRepository(required) + if !ok || dir == "" || !trunk.SameRepository(repo, t.f.Repository) { + return "", false + } + if goModModule(t.goMod(dir)) != required { + return "", false + } + return dir, true +} + +// sameRepositoryRequired are the directories (from the root, sorted, each once) of every same-repository Go module the +// go.mod of each start directory requires, transitively through those modules' own go.mod files (H-A-202, H-A-157). The +// start directories themselves are not among them. +func (t *goModTree) sameRepositoryRequired(start []string) []string { + seen := map[string]bool{} + for _, s := range start { + seen[s] = true + } + queue := slices.Clone(start) + var out []string + for len(queue) > 0 { + dir := queue[0] + queue = queue[1:] + for _, req := range fingerprint.MeshRequiresOf(t.goMod(dir)) { + d, ok := t.sameRepositoryDir(req) + if !ok || seen[d] { + continue + } + seen[d] = true + out = append(out, d) + queue = append(queue, d) + } + } + slices.Sort(out) + return out +} + +// goModuleDirs are the directories (from the root) a module's Go build starts from: its own directory and the directory +// of each path of each Go package artifact of its manifest m. +func goModuleDirs(m catalogue.Manifest, dir string) []string { + out := []string{dir} + if m.Build == nil { + return out + } + for _, a := range m.Build.Artifacts { + if a.Kind != catalogue.ArtifactPackage || a.Language != "go" { + continue + } + for _, p := range a.Paths { + if d := path.Join(dir, strings.TrimSuffix(filepath.ToSlash(p), "/")); !slices.Contains(out, d) { + out = append(out, d) + } + } + } + return out +} + +// derivedDirs are the same-repository module directories (from the root, sorted) the module's Go build resolves from the +// tree beyond its own tree input (H-A-202): every one its go.mod files require, transitively, that its tree input does +// not hold already, which is the module's directory for the directory's tree, and its named paths for the named tree. +func (t *goModTree) derivedDirs(m catalogue.Manifest, dir string) []string { + var held []string + if paths, every := m.NamedPaths(); every { + for _, p := range paths { + held = append(held, path.Join(dir, filepath.ToSlash(p))) + } + } else { + held = []string{dir} + } + var out []string + for _, d := range t.sameRepositoryRequired(goModuleDirs(m, dir)) { + if !slices.ContainsFunc(held, func(h string) bool { return h == "" || h == "." || d == h || strings.HasPrefix(d, h+"/") }) { + out = append(out, d) + } + } + return out +} + +// DerivedLine opens each line of the tree input naming a derived directory and its git object (H-A-202, H-A-172). +const DerivedLine = "derived " + +// derivedTree is the module's tree input with a line per derived directory and its object appended, sorted by directory +// (H-A-202, H-A-172: what a build reads moves its fingerprint), so a change of such a directory moves the fingerprint of +// every form. The objects are asked of the forge at the merge commit by their paths from the root (Named); an answer that +// lacks one gives the reason the module has no inputs, naming them. A module with no derived directory keeps its tree. +func (ix *Indexer) derivedTree(ctx context.Context, f whole, t *goModTree, m catalogue.Manifest, dir, tree string) (string, string) { + dirs := t.derivedDirs(m, dir) + if len(dirs) == 0 { + return tree, "" + } + if ix.Named == nil { + return "", "the forge cannot be asked the git objects of the derived directories " + strings.Join(dirs, ", ") + } + got, err := ix.Named(ctx, f.Repository, f.Merge, map[string][]string{"": dirs}) + if err != nil { + return "", fmt.Sprintf("the git objects of the derived directories %s: %v", strings.Join(dirs, ", "), err) + } + lines := []string{tree} + var missing []string + for _, d := range dirs { + o := got[""][d] + if o == "" { + missing = append(missing, d) + continue + } + lines = append(lines, DerivedLine+d+" "+o) + } + if len(missing) > 0 { + return "", "the forge names no git object of the derived directories " + strings.Join(missing, ", ") + } + return strings.Join(lines, "\n"), "" +} + +// goModModule is the module path a go.mod declares; "" for none. +func goModModule(raw []byte) string { + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) == 2 && f[0] == "module" { + return strings.Trim(f[1], `"`) + } + } + return "" +} diff --git a/modules/mesh-delivery-indexer/internal/index/package_trees_test.go b/modules/mesh-delivery-indexer/internal/index/package_trees_test.go new file mode 100644 index 00000000..ecdfeeb0 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/package_trees_test.go @@ -0,0 +1,108 @@ +package index + +import ( + "context" + "strings" + "testing" + + "mesh-delivery-indexer/internal/fingerprint" +) + +// A fixture repository novox/mesh-delivery: the Go package state of modules/g requires the same-repository module +// modules/b/builds at the zero version, which requires modules/c/client in turn; mesh-sdk is of another repository, and +// modules/b/gone names a directory the tree holds no go.mod of. +const ptpZero = " v0.0.0-00010101000000-000000000000" + +var ptpGoMods = map[string]string{ + "modules/g/state": "module git.novox.be/novox/mesh-delivery/modules/g/state\n\nrequire (\n\tgit.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + + "\n\tgit.novox.be/novox/mesh-sdk/go" + ptpZero + "\n\tgit.novox.be/novox/mesh-delivery/modules/b/gone" + ptpZero + "\n)\n", + "modules/b/builds": "module git.novox.be/novox/mesh-delivery/modules/b/builds\n\nrequire git.novox.be/novox/mesh-delivery/modules/c/client" + ptpZero + " // indirect\n", + "modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n", +} + +// ptpProgram is modules/g's manifest: a program naming no paths beside the package state. +const ptpProgram = `{"module":"g","version":"1.0","build":{"artifacts":[{"name":"program","kind":"bundle","language":"go","system":"arch","from":"cmd/g","binary":"g","loads":["g"]},` + + `{"name":"state","kind":"package","language":"go","paths":["state"],"series":"0.1"}]}}` + +// ptpIndexer answers the fixture's go.mod files and, for a path asked from the root, the object objects names for it. +func ptpIndexer(t *testing.T, objects map[string]string) (*Indexer, *memStore, *[]map[string][]string) { + ix, mods, _ := manifestsIndexer(t) + ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) { + if c, ok := ptpGoMods[dir]; ok { + return []byte(c), nil + } + return nil, nil + } + var asked []map[string][]string + ix.Named = func(_ context.Context, _, _ string, paths map[string][]string) (map[string]map[string]string, error) { + asked = append(asked, paths) + out := map[string]map[string]string{} + for dir, ps := range paths { + out[dir] = map[string]string{} + for _, p := range ps { + out[dir][p] = objects[p] + } + } + return out, nil + } + return ix, mods, &asked +} + +// Design 47 H-A-202, H-A-172: the same-repository modules a Go build resolves from the tree, transitively, join the tree +// input of every form with their objects, so a change of one moves the fingerprint of a module whose program names no +// paths; the entry's paths stay the manifest's. +func TestADerivedDirectoryMovesTheFingerprintOfEveryForm(t *testing.T) { + objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} + tree := func(objects map[string]string) string { + ix, mods, asked := ptpIndexer(t, objects) + f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, _ := moduleOf(t, mods, "g") + if got := strings.Join(m.Paths["state"], ","); got != "state" { + t.Errorf("the entry's paths of state are %q, want the manifest's", got) + } + if len(*asked) != 1 || strings.Join((*asked)[0][""], ",") != "modules/b/builds,modules/c/client" { + t.Fatalf("asked the forge %v, want the derived directories from the root once", *asked) + } + if m.Inputs2 == nil || !strings.HasPrefix(m.Inputs2.Tree, "gtree\n") { + t.Fatalf("inputs = %+v (%s), want the directory's tree with the derived lines", m.Inputs2, m.None2) + } + return m.Inputs2.Tree + } + before := tree(objects) + if want := "gtree\nderived modules/b/builds " + objects["modules/b/builds"] + "\nderived modules/c/client " + objects["modules/c/client"]; before != want { + t.Errorf("the tree input is %q, want %q", before, want) + } + objects["modules/c/client"] = strings.Repeat("d", 40) + after := tree(objects) + fp := func(tree string) string { + v, _ := fingerprint.Of(fingerprint.Inputs{Module: "g", Tree: tree}) + return v + } + if fp(before) == fp(after) { + t.Errorf("a change of modules/c/client left the fingerprint %s", fp(after)) + } + // The named tree takes them too: a module whose every artifact names its paths. + named := `{"module":"g","version":"1.0","build":{"artifacts":[{"name":"state","kind":"package","language":"go","paths":["state"],"series":"0.1"}]}}` + ix, mods, _ := ptpIndexer(t, objects) + f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": named}, nil) + f.Paths = map[string]map[string]string{"modules/g": {"state": strings.Repeat("1", 40)}} + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "g"); m.Inputs2 == nil || !strings.HasPrefix(m.Inputs2.Tree, fingerprint.NamedTreePrefix) || + !strings.HasSuffix(m.Inputs2.Tree, "\nderived modules/c/client "+objects["modules/c/client"]) || strings.Join(m.Paths["state"], ",") != "state" { + t.Errorf("module = %+v, want the named tree with the derived lines and the manifest's paths", m) + } + // A forge that names no object of a derived directory gives the module none, naming it. + ix, mods, _ = ptpIndexer(t, map[string]string{"modules/b/builds": strings.Repeat("b", 40)}) + if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), + map[string]string{"modules/g": ptpProgram}, nil)); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "g"); m.Fp2 != "" || !strings.Contains(m.None2, "modules/c/client") { + t.Errorf("module = %+v, want none naming the directory the forge gave no object of", m) + } +}