mesh-build-go: a catalogue module's non-Go files enter its packages' digests; markers where tests read other modules (issue 1935)

modules/mesh-build-go/README.md

@@ -102,7 +102,7 @@ integration level, the postgres major and the nats-server version; and, when the
 workspace (`beside`), each one's repository and commit, which the answer echoes as `beside` (the checker keeps a verdict only
 for the siblings it was judged with). The text's form is `DigestVersion` 3. The Go version is not in it: the toolchain enters a
 fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and the standard library's packages are that
-version's. A package with no tests at the level has no digest and is named in `without_tests`. A package whose directory holds a file named `reads-outside` (any content; say what the tests read) is never kept (issue 1935): its tests read
+version's. A package with no tests at the level has no digest and is named in `without_tests`. A catalogue module's packages also hash the module's non-Go files (the nearest ancestor directory holding `module.json`: its manifest, README, `files/` and the rest, so `../../module.json` is covered; digest form 5). A package whose directory holds a file named `reads-outside` (any content; say what the tests read) is never kept (issue 1935): its tests read
 files outside it, which a digest from `go list` cannot name, so its digest is a fresh random one on every call, the ask finds no kept
 verdict for it and runs it, and the answer names it in `reads_outside` and the module's log says why. The checker still writes a
 verdict under that digest, which nothing ever reads again. The marker is the plain way to say it until each verdict keeps the files

modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go

@@ -116,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) {
 			t.Errorf("the integration digest listed %q", s.command)
 		}
 	}
-	if DigestVersion != "4" {
+	if DigestVersion != "5" {
 		t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion)
 	}
 }

modules/mesh-build-go/cmd/mesh-build-go/digest.go

@@ -25,7 +25,8 @@ import (
 // 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings'
 // commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement).
 // 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1).
-const DigestVersion = "4"
+// 5: a catalogue module's non-Go files enter the digest of its packages (issue 1935).
+const DigestVersion = "5"
 
 // besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file
 // depends on the siblings' commits.
@@ -331,6 +332,15 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
 			}
 		}
 	}
+	// A catalogue module's tests read its manifest, its README and its files from the package's directory (../../module.json):
+	// every non-Go file of the module's directory is an input (issue 1935). The module is the nearest ancestor with a module.json.
+	if modDir := enclosingModule(tree, root.Dir); modDir != "" {
+		more, err := moduleFileLines(tree, root, modDir)
+		if err != nil {
+			return "", err
+		}
+		lines = append(lines, more...)
+	}
 	data := filepath.Join(root.Dir, "testdata")
 	if _, err := os.Stat(data); err == nil {
 		err := filepath.WalkDir(data, func(path string, e fs.DirEntry, err error) error {
@@ -356,6 +366,41 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
 	return hex.EncodeToString(h.Sum(nil)), nil
 }
 
+// enclosingModule is the nearest ancestor of dir, inside tree, that holds a module.json, or "" when there is none.
+func enclosingModule(tree, dir string) string {
+	tree = filepath.Clean(tree)
+	for d := filepath.Clean(dir); d != tree && strings.HasPrefix(d, tree+string(filepath.Separator)); d = filepath.Dir(d) {
+		if st, err := os.Stat(filepath.Join(d, "module.json")); err == nil && !st.IsDir() {
+			return d
+		}
+	}
+	return ""
+}
+
+// moduleFileLines are the content lines of every file of a module's directory that is not Go source (Go source is the
+// packages' own, listed by go list), skipping .git and node_modules.
+func moduleFileLines(tree string, p *goPackage, modDir string) ([]string, error) {
+	var lines []string
+	err := filepath.WalkDir(modDir, func(path string, e fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if e.IsDir() {
+			if n := e.Name(); n == ".git" || n == "node_modules" {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if strings.HasSuffix(path, ".go") {
+			return nil
+		}
+		l, err := contentLine(tree, p, path)
+		lines = append(lines, l)
+		return err
+	})
+	return lines, err
+}
+
 func fileLine(tree string, p *goPackage, name string) (string, error) {
 	return contentLine(tree, p, filepath.Join(p.Dir, name))
 }

modules/mesh-build-go/cmd/mesh-build-go/digest_test.go

@@ -214,3 +214,46 @@ func TestAPackageWithTheReadsOutsideMarkerIsNeverKept(t *testing.T) {
 		t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a])
 	}
 }
+
+// Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the
+// digest of the module's package, and a change to another module's files does not.
+func TestACatalogueModulesFilesChangeTheDigestOfItsPackagesOnly(t *testing.T) {
+	t.Parallel()
+	m, tree := digestFixture(t)
+	for _, mod := range []string{"mod1", "mod2"} {
+		dir := filepath.Join(tree, mod, "cmd", "p")
+		if err := os.MkdirAll(filepath.Join(tree, mod, "files"), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		edit(t, tree, mod+"/module.json", `{"module":"`+mod+`"}`)
+		edit(t, tree, mod+"/README.md", "# "+mod+"\n")
+		edit(t, tree, mod+"/files/f.env", "A=1\n")
+		edit(t, tree, mod+"/cmd/p/p.go", "package p\n")
+		edit(t, tree, mod+"/cmd/p/p_test.go", "package p\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestReads(t *testing.T) {\n\tif _, err := os.ReadFile(\"../../module.json\"); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n")
+	}
+	p1, p2 := "fixture/digest/mod1/cmd/p", "fixture/digest/mod2/cmd/p"
+	before := digests(t, m, "unit")
+	if len(before.Packages[p1]) != 64 || len(before.Packages[p2]) != 64 {
+		t.Fatalf("no digest for the module packages: %+v", before.Packages)
+	}
+	if before.Packages[p1] == before.Packages[p2] {
+		t.Errorf("two modules, one digest")
+	}
+	for _, file := range []string{"mod1/module.json", "mod1/README.md", "mod1/files/f.env"} {
+		edit(t, tree, file, "changed "+file+"\n")
+		after := digests(t, m, "unit")
+		if after.Packages[p1] == before.Packages[p1] {
+			t.Errorf("a change to %s did not change the module's digest", file)
+		}
+		if after.Packages[p2] != before.Packages[p2] {
+			t.Errorf("a change to %s changed the other module's digest", file)
+		}
+		before = after
+	}
+	if got := digests(t, m, "unit"); got.Packages["fixture/digest/c"] == "" {
+		t.Errorf("a package outside any module lost its digest")
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/reads-outside

@@ -0,0 +1 @@
+These tests read other catalogue modules' files, outside this module (issue 1935): never kept.

modules/mesh-build-image/cmd/mesh-build-image/reads-outside

@@ -0,0 +1 @@
+These tests read other catalogue modules' files, outside this module (issue 1935): never kept.

modules/mesh-build-node/cmd/mesh-build-node/reads-outside

@@ -0,0 +1 @@
+These tests read other catalogue modules' files, outside this module (issue 1935): never kept.

modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside

@@ -0,0 +1 @@
+These tests read other catalogue modules' files, outside this module (issue 1935): never kept.

modules/minio/cmd/object-store/reads-outside

@@ -0,0 +1 @@
+These tests read other catalogue modules' files, outside this module (issue 1935): never kept.

modules/postgres/cmd/postgres-provider/reads-outside

@@ -0,0 +1 @@
+These tests read other catalogue modules' files, outside this module (issue 1935): never kept.