From dad8672b73ba57562c68d16b3e77bd7afbae04d1 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 15:21:36 +0100 Subject: [PATCH] Join 11: the fetch's proof loads both build tags and holds the tree's go.mod and go.sum to what they load Try 2 of milestone G failed in go test, verifying nats-server against the checksum database: restic's integration tests load it through testbus, neither its go.mod nor its go.sum named it, and the fetch's shut-off load ran untagged, so it never met it. The second failure of this kind (after the join's dropped zero-version requires), so the guard is held once, where every repository's check passes: each directory of the checked tree is loaded untagged and with the integration level's tags, and its own go.mod and go.sum must name every module the loads reach (a mesh module required, no sums; every other required at the version selected, with both go.sum lines). The fetch fails naming the module and the line. A sibling laid beside is loaded as before, unjudged. This guard lands with G, so G's own check still runs today's proof; no harm, the gaps it finds in the seven repositories are fixed in this join. Builds: design 47 G (join 11, the check's shut-off build of the group's modules) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 127 ++++++++++++++++-- .../cmd/mesh-build-go/fetch_test.go | 28 +++- .../cmd/mesh-build-go/workspace.go | 2 +- 3 files changed, 143 insertions(+), 14 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 1b8ad6fe7..b4d5ab854 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -9,6 +9,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -228,7 +229,7 @@ func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d s if failure == nil { // The proof that what was downloaded is all the shut-off steps load: else the fetch fails here, naming the // package, and not a vet, a test or a build later with less to say (issue 1966). - return taken, m.shutOffLoad(ctx, m.goEnvFor(dir), dir, d) + return taken, m.shutOffLoad(ctx, m.goEnvFor(dir), dir, d, true) } // A requirement at the zero version names no commit: it is taken from the asked versions or the trunk's latest. took, err := m.takeZeroPins(ctx, root, repository, dir, d, failure.Error(), asked, pins) @@ -251,29 +252,131 @@ func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d s // shutOffLoadFormat is what go list -e says of each package it loads: nothing, or the package and Go's words of its error. const shutOffLoadFormat = `{{with .Error}}{{$.ImportPath}}: {{.Err}}{{"\n"}}{{end}}` +// loadedModuleMark opens the line judgedLoadFormat says of the module a loaded package lies in. +const loadedModuleMark = "loaded-module " + +// judgedLoadFormat is shutOffLoadFormat with, for each package of a module other than the directory's own, a line +// naming that module, its version and its replacement: what the go.mod and go.sum of the tree are judged against. +const judgedLoadFormat = shutOffLoadFormat + `{{with .Module}}{{if not .Main}}` + loadedModuleMark + + `{{.Path}} {{.Version}}{{with .Replace}} {{.Path}} {{.Version}}{{end}}{{"\n"}}{{end}}{{end}}` + +// loadedModule is a module a package of the load lies in: its path and version, and the replacement Go read it from +// (a directory has no version). +type loadedModule struct{ path, version, replacePath, replaceVersion string } + // shutOffLoad loads the packages of dir, with their tests and every package they import, as a shut-off step does // (GOPROXY=off, the step's own environment env), and fails naming each package whose load would reach the network, in // Go's words. A package that fails to load for another cause (a compile error, an import of a package of the module // that does not exist) is the shut-off steps' to say, with their own verdict lines, and is not the fetch's. -func (m *Module) shutOffLoad(ctx context.Context, env []string, dir, d string) error { - r := m.Run(ctx, append(env, "GOPROXY=off"), "go", "-C", dir, "list", "-e", "-deps", "-test", "-f", shutOffLoadFormat, "./...") +// +// A judged directory (one of the checked tree, not a sibling laid beside) is loaded twice, untagged and with the +// integration level's tags, and its own go.mod and go.sum must name every module either load reaches (ownFilesCover): +// a module the shut-off steps meet that the tree's files do not name is looked up in the checksum database, which no +// shut-off step reaches (try 2 of design 47's milestone G: restic's tests load nats-server through testbus, which +// neither its go.mod nor its go.sum named, and its go test failed verifying it). The fetch fails naming the module +// and the line its go.mod or go.sum lacks, for every repository the check builds. +func (m *Module) shutOffLoad(ctx context.Context, env []string, dir, d string, judged bool) error { + tagSets := [][]string{nil} + format := shutOffLoadFormat + if judged { + tagSets, format = append(tagSets, integrationTags), judgedLoadFormat + } var lost []string - for _, l := range strings.Split(r.Stdout+"\n"+r.Stderr, "\n") { - if l = strings.TrimSpace(l); strings.Contains(l, "GOPROXY=off") && !slices.Contains(lost, l) { - lost = append(lost, l) + var loaded []loadedModule + for _, tags := range tagSets { + args := []string{"-C", dir, "list", "-e", "-deps", "-test"} + if len(tags) > 0 { + args = append(args, "-tags", strings.Join(tags, ",")) + } + r := m.Run(ctx, append(env, "GOPROXY=off"), "go", append(args, "-f", format, "./...")...) + for _, l := range strings.Split(r.Stdout+"\n"+r.Stderr, "\n") { + l = strings.TrimSpace(l) + if rest, ok := strings.CutPrefix(l, loadedModuleMark); ok { + f := append(strings.Fields(rest), "", "", "") + loaded = append(loaded, loadedModule{path: f[0], version: f[1], replacePath: f[2], replaceVersion: f[3]}) + } else if strings.Contains(l, "GOPROXY=off") && !slices.Contains(lost, l) { + lost = append(lost, l) + } + } + if len(lost) > 0 { + return fmt.Errorf("the packages of %s load shut off only with what the fetch did not download: %s", d, strings.Join(lost, "; ")) + } + // A load that could not run at all proves nothing: the fetch fails here, not a step later with less to say. + if r.Err != "" || r.Status != 0 { + say := strings.TrimSpace(r.Err + " " + r.Stderr) + return fmt.Errorf("the shut-off package load of %s could not run (exit %d): %s", d, r.Status, say) } } - if len(lost) > 0 { - return fmt.Errorf("the packages of %s load shut off only with what the fetch did not download: %s", d, strings.Join(lost, "; ")) + if !judged || len(loaded) == 0 { + return nil // a load that reaches no module beyond the directory's own has nothing to name } - // A load that could not run at all proves nothing: the fetch fails here, not a step later with less to say. - if r.Err != "" || r.Status != 0 { - say := strings.TrimSpace(r.Err + " " + r.Stderr) - return fmt.Errorf("the shut-off package load of %s could not run (exit %d): %s", d, r.Status, say) + if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil { + return nil // a directory that vendors is read from its vendor directory, never through the module cache + } + missing, err := ownFilesCover(dir, loaded) + if err != nil { + return fmt.Errorf("the go.mod and go.sum of %s: %w", d, err) + } + if len(missing) > 0 { + return fmt.Errorf("the go.mod and go.sum of %s do not name every module its packages load (untagged and %s), so a shut-off step would look it up: lacking %s", + d, strings.Join(integrationTags, ","), strings.Join(missing, "; ")) } return nil } +// ownFilesCover is what the go.mod and go.sum in dir lack for the modules its packages load, each as the line it lacks: +// a mesh module (of the forge) required, the line it lacks naming the zero version (ADR 0573 §2; one named at a commit +// is built against the asked version all the same, §4, and needs no lookup), every other required at the version the +// load selected, with its two go.sum lines (the module's and its go.mod's) for the version Go reads it at. A module +// replaced by a directory has no sums to lack. +func ownFilesCover(dir string, loaded []loadedModule) ([]string, error) { + gomod, err := os.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { + return nil, err + } + required, _ := goModRequirements(string(gomod)) + sums := map[string]bool{} + if raw, err := os.ReadFile(filepath.Join(dir, "go.sum")); err == nil { + for _, l := range strings.Split(string(raw), "\n") { + if f := strings.Fields(l); len(f) == 3 { + sums[f[0]+" "+f[1]] = true + } + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + var missing []string + lack := func(line string) { + if !slices.Contains(missing, line) { + missing = append(missing, line) + } + } + for _, l := range loaded { + if strings.HasPrefix(l.path, forgePrefix) { + if _, ok := required[l.path]; !ok { + lack("go.mod: require " + l.path + " " + zeroVersion) + } + continue + } + if required[l.path] != l.version { + lack("go.mod: require " + l.path + " " + l.version) + } + path, version := l.path, l.version + if l.replacePath != "" { + if l.replaceVersion == "" { + continue // a directory + } + path, version = l.replacePath, l.replaceVersion + } + for _, v := range []string{version, version + "/go.mod"} { + if !sums[path+" "+v] { + lack("go.sum: " + path + " " + v + " h1:") + } + } + } + return missing, nil +} + // verifyLocks hold one mutex per digest of a go.mod and go.sum, so two directories with the same pair verify once. var verifyLocks sync.Map diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index f98cdfee6..75c8823c9 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -215,6 +215,8 @@ func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) { const seatPkg = "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" for _, c := range []struct { name, ran, out string + tagged string // what the load with the integration level's tags says, when it differs + files map[string]string // the tree's go.mod and go.sum fatal bool want []string notWant string @@ -229,14 +231,38 @@ func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) { // 1966/AC182-1: a package load that could not run at all (a fatal exit, no GOPROXY=off line) fails the fetch too. {name: "the shut-off package load cannot run", ran: "tools list -e -deps -test", fatal: true, out: "go: go.mod file not found\n", want: []string{"tools", "go: go.mod file not found"}}, + // Try 2 of milestone G: restic's integration tests load nats-server through testbus, and neither its go.mod nor its + // go.sum named it, so its go test looked it up in the checksum database. The fetch fails naming the lines, for the + // tagged load too; a mesh module replaced by a directory at the zero version, and a module both files name, pass. + {name: "a module the integration tests load is neither required nor summed", ran: "tools list -e -deps -test", + out: loadedModuleMark + "git.novox.be/novox/mesh-sdk/go " + zeroVersion + " /tree/sdk \n" + + loadedModuleMark + "github.com/nats-io/nats.go v1.54.0\n", + tagged: loadedModuleMark + "github.com/nats-io/nats-server/v2 v2.11.17\n", + files: map[string]string{ + "tools/go.mod": "module fixture/tools\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go " + zeroVersion + "\n\tgithub.com/nats-io/nats.go v1.54.0\n)\n", + "tools/go.sum": "github.com/nats-io/nats.go v1.54.0 h1:a=\ngithub.com/nats-io/nats.go v1.54.0/go.mod h1:b=\n", + }, + want: []string{"tools", "go.mod: require github.com/nats-io/nats-server/v2 v2.11.17", + "go.sum: github.com/nats-io/nats-server/v2 v2.11.17 h1:", "go.sum: github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:"}, + notWant: "nats.go"}, + // Issue 1966's kind: a mesh module reached only through another, which go.mod does not require, is lacked at the zero + // version (ADR 0573 §2); one go.mod names at a commit (§4) is not. + {name: "a mesh module the load reaches is not required", ran: "tools list -e -deps -test", + out: loadedModuleMark + seatPkg + " " + zeroVersion + "\n" + + loadedModuleMark + "git.novox.be/novox/mesh-sdk/go v0.0.0-20261112000000-0123456789ab\n", + files: map[string]string{"tools/go.mod": "module fixture/tools\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-20261112000000-0123456789ab\n"}, + want: []string{"tools", "go.mod: require " + seatPkg + " " + zeroVersion}, notWant: "mesh-sdk"}, } { t.Run(c.name, func(t *testing.T) { - f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", goRule: func(line string) forgekey.Ran { + f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", files: c.files, goRule: func(line string) forgekey.Ran { if strings.Contains(line, "/"+c.ran) { if strings.Contains(line, " list ") { if c.fatal { return forgekey.Ran{Status: 1, Stderr: c.out} } + if strings.Contains(line, " -tags "+strings.Join(integrationTags, ",")) { + return forgekey.Ran{Stdout: c.tagged} + } return forgekey.Ran{Stdout: c.out} // go list -e says a package's error and exits 0 } return forgekey.Ran{Status: 1, Stdout: c.out} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go index b9184fdf8..2afd8fd14 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go @@ -176,7 +176,7 @@ func (m *Module) downloadBeside(ctx context.Context, root, tree string, dirs []s if res := m.Run(ctx, env, "go", "-C", r, "list", "-e", "-deps", "-test", "-f", shutOffLoadFormat, "./..."); res.Status != 0 || res.Err != "" { return fmt.Errorf("go list of the packages of %s beside: %s", filepath.Base(r), causeLine(res.Err+"\n"+res.Stdout+"\n"+res.Stderr)) } - if err := m.shutOffLoad(ctx, env, r, "beside "+filepath.Base(r)); err != nil { + if err := m.shutOffLoad(ctx, env, r, "beside "+filepath.Base(r), false); err != nil { return err } }