diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 1b8ad6fe7..b4d5ab854 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -9,6 +9,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -228,7 +229,7 @@ func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d s if failure == nil { // The proof that what was downloaded is all the shut-off steps load: else the fetch fails here, naming the // package, and not a vet, a test or a build later with less to say (issue 1966). - return taken, m.shutOffLoad(ctx, m.goEnvFor(dir), dir, d) + return taken, m.shutOffLoad(ctx, m.goEnvFor(dir), dir, d, true) } // A requirement at the zero version names no commit: it is taken from the asked versions or the trunk's latest. took, err := m.takeZeroPins(ctx, root, repository, dir, d, failure.Error(), asked, pins) @@ -251,29 +252,131 @@ func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d s // shutOffLoadFormat is what go list -e says of each package it loads: nothing, or the package and Go's words of its error. const shutOffLoadFormat = `{{with .Error}}{{$.ImportPath}}: {{.Err}}{{"\n"}}{{end}}` +// loadedModuleMark opens the line judgedLoadFormat says of the module a loaded package lies in. +const loadedModuleMark = "loaded-module " + +// judgedLoadFormat is shutOffLoadFormat with, for each package of a module other than the directory's own, a line +// naming that module, its version and its replacement: what the go.mod and go.sum of the tree are judged against. +const judgedLoadFormat = shutOffLoadFormat + `{{with .Module}}{{if not .Main}}` + loadedModuleMark + + `{{.Path}} {{.Version}}{{with .Replace}} {{.Path}} {{.Version}}{{end}}{{"\n"}}{{end}}{{end}}` + +// loadedModule is a module a package of the load lies in: its path and version, and the replacement Go read it from +// (a directory has no version). +type loadedModule struct{ path, version, replacePath, replaceVersion string } + // shutOffLoad loads the packages of dir, with their tests and every package they import, as a shut-off step does // (GOPROXY=off, the step's own environment env), and fails naming each package whose load would reach the network, in // Go's words. A package that fails to load for another cause (a compile error, an import of a package of the module // that does not exist) is the shut-off steps' to say, with their own verdict lines, and is not the fetch's. -func (m *Module) shutOffLoad(ctx context.Context, env []string, dir, d string) error { - r := m.Run(ctx, append(env, "GOPROXY=off"), "go", "-C", dir, "list", "-e", "-deps", "-test", "-f", shutOffLoadFormat, "./...") +// +// A judged directory (one of the checked tree, not a sibling laid beside) is loaded twice, untagged and with the +// integration level's tags, and its own go.mod and go.sum must name every module either load reaches (ownFilesCover): +// a module the shut-off steps meet that the tree's files do not name is looked up in the checksum database, which no +// shut-off step reaches (try 2 of design 47's milestone G: restic's tests load nats-server through testbus, which +// neither its go.mod nor its go.sum named, and its go test failed verifying it). The fetch fails naming the module +// and the line its go.mod or go.sum lacks, for every repository the check builds. +func (m *Module) shutOffLoad(ctx context.Context, env []string, dir, d string, judged bool) error { + tagSets := [][]string{nil} + format := shutOffLoadFormat + if judged { + tagSets, format = append(tagSets, integrationTags), judgedLoadFormat + } var lost []string - for _, l := range strings.Split(r.Stdout+"\n"+r.Stderr, "\n") { - if l = strings.TrimSpace(l); strings.Contains(l, "GOPROXY=off") && !slices.Contains(lost, l) { - lost = append(lost, l) + var loaded []loadedModule + for _, tags := range tagSets { + args := []string{"-C", dir, "list", "-e", "-deps", "-test"} + if len(tags) > 0 { + args = append(args, "-tags", strings.Join(tags, ",")) + } + r := m.Run(ctx, append(env, "GOPROXY=off"), "go", append(args, "-f", format, "./...")...) + for _, l := range strings.Split(r.Stdout+"\n"+r.Stderr, "\n") { + l = strings.TrimSpace(l) + if rest, ok := strings.CutPrefix(l, loadedModuleMark); ok { + f := append(strings.Fields(rest), "", "", "") + loaded = append(loaded, loadedModule{path: f[0], version: f[1], replacePath: f[2], replaceVersion: f[3]}) + } else if strings.Contains(l, "GOPROXY=off") && !slices.Contains(lost, l) { + lost = append(lost, l) + } + } + if len(lost) > 0 { + return fmt.Errorf("the packages of %s load shut off only with what the fetch did not download: %s", d, strings.Join(lost, "; ")) + } + // A load that could not run at all proves nothing: the fetch fails here, not a step later with less to say. + if r.Err != "" || r.Status != 0 { + say := strings.TrimSpace(r.Err + " " + r.Stderr) + return fmt.Errorf("the shut-off package load of %s could not run (exit %d): %s", d, r.Status, say) } } - if len(lost) > 0 { - return fmt.Errorf("the packages of %s load shut off only with what the fetch did not download: %s", d, strings.Join(lost, "; ")) + if !judged || len(loaded) == 0 { + return nil // a load that reaches no module beyond the directory's own has nothing to name } - // A load that could not run at all proves nothing: the fetch fails here, not a step later with less to say. - if r.Err != "" || r.Status != 0 { - say := strings.TrimSpace(r.Err + " " + r.Stderr) - return fmt.Errorf("the shut-off package load of %s could not run (exit %d): %s", d, r.Status, say) + if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil { + return nil // a directory that vendors is read from its vendor directory, never through the module cache + } + missing, err := ownFilesCover(dir, loaded) + if err != nil { + return fmt.Errorf("the go.mod and go.sum of %s: %w", d, err) + } + if len(missing) > 0 { + return fmt.Errorf("the go.mod and go.sum of %s do not name every module its packages load (untagged and %s), so a shut-off step would look it up: lacking %s", + d, strings.Join(integrationTags, ","), strings.Join(missing, "; ")) } return nil } +// ownFilesCover is what the go.mod and go.sum in dir lack for the modules its packages load, each as the line it lacks: +// a mesh module (of the forge) required, the line it lacks naming the zero version (ADR 0573 §2; one named at a commit +// is built against the asked version all the same, §4, and needs no lookup), every other required at the version the +// load selected, with its two go.sum lines (the module's and its go.mod's) for the version Go reads it at. A module +// replaced by a directory has no sums to lack. +func ownFilesCover(dir string, loaded []loadedModule) ([]string, error) { + gomod, err := os.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { + return nil, err + } + required, _ := goModRequirements(string(gomod)) + sums := map[string]bool{} + if raw, err := os.ReadFile(filepath.Join(dir, "go.sum")); err == nil { + for _, l := range strings.Split(string(raw), "\n") { + if f := strings.Fields(l); len(f) == 3 { + sums[f[0]+" "+f[1]] = true + } + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + var missing []string + lack := func(line string) { + if !slices.Contains(missing, line) { + missing = append(missing, line) + } + } + for _, l := range loaded { + if strings.HasPrefix(l.path, forgePrefix) { + if _, ok := required[l.path]; !ok { + lack("go.mod: require " + l.path + " " + zeroVersion) + } + continue + } + if required[l.path] != l.version { + lack("go.mod: require " + l.path + " " + l.version) + } + path, version := l.path, l.version + if l.replacePath != "" { + if l.replaceVersion == "" { + continue // a directory + } + path, version = l.replacePath, l.replaceVersion + } + for _, v := range []string{version, version + "/go.mod"} { + if !sums[path+" "+v] { + lack("go.sum: " + path + " " + v + " h1:") + } + } + } + return missing, nil +} + // verifyLocks hold one mutex per digest of a go.mod and go.sum, so two directories with the same pair verify once. var verifyLocks sync.Map diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index f98cdfee6..75c8823c9 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -215,6 +215,8 @@ func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) { const seatPkg = "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" for _, c := range []struct { name, ran, out string + tagged string // what the load with the integration level's tags says, when it differs + files map[string]string // the tree's go.mod and go.sum fatal bool want []string notWant string @@ -229,14 +231,38 @@ func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) { // 1966/AC182-1: a package load that could not run at all (a fatal exit, no GOPROXY=off line) fails the fetch too. {name: "the shut-off package load cannot run", ran: "tools list -e -deps -test", fatal: true, out: "go: go.mod file not found\n", want: []string{"tools", "go: go.mod file not found"}}, + // Try 2 of milestone G: restic's integration tests load nats-server through testbus, and neither its go.mod nor its + // go.sum named it, so its go test looked it up in the checksum database. The fetch fails naming the lines, for the + // tagged load too; a mesh module replaced by a directory at the zero version, and a module both files name, pass. + {name: "a module the integration tests load is neither required nor summed", ran: "tools list -e -deps -test", + out: loadedModuleMark + "git.novox.be/novox/mesh-sdk/go " + zeroVersion + " /tree/sdk \n" + + loadedModuleMark + "github.com/nats-io/nats.go v1.54.0\n", + tagged: loadedModuleMark + "github.com/nats-io/nats-server/v2 v2.11.17\n", + files: map[string]string{ + "tools/go.mod": "module fixture/tools\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go " + zeroVersion + "\n\tgithub.com/nats-io/nats.go v1.54.0\n)\n", + "tools/go.sum": "github.com/nats-io/nats.go v1.54.0 h1:a=\ngithub.com/nats-io/nats.go v1.54.0/go.mod h1:b=\n", + }, + want: []string{"tools", "go.mod: require github.com/nats-io/nats-server/v2 v2.11.17", + "go.sum: github.com/nats-io/nats-server/v2 v2.11.17 h1:", "go.sum: github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:"}, + notWant: "nats.go"}, + // Issue 1966's kind: a mesh module reached only through another, which go.mod does not require, is lacked at the zero + // version (ADR 0573 §2); one go.mod names at a commit (§4) is not. + {name: "a mesh module the load reaches is not required", ran: "tools list -e -deps -test", + out: loadedModuleMark + seatPkg + " " + zeroVersion + "\n" + + loadedModuleMark + "git.novox.be/novox/mesh-sdk/go v0.0.0-20261112000000-0123456789ab\n", + files: map[string]string{"tools/go.mod": "module fixture/tools\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-20261112000000-0123456789ab\n"}, + want: []string{"tools", "go.mod: require " + seatPkg + " " + zeroVersion}, notWant: "mesh-sdk"}, } { t.Run(c.name, func(t *testing.T) { - f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", goRule: func(line string) forgekey.Ran { + f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", files: c.files, goRule: func(line string) forgekey.Ran { if strings.Contains(line, "/"+c.ran) { if strings.Contains(line, " list ") { if c.fatal { return forgekey.Ran{Status: 1, Stderr: c.out} } + if strings.Contains(line, " -tags "+strings.Join(integrationTags, ",")) { + return forgekey.Ran{Stdout: c.tagged} + } return forgekey.Ran{Stdout: c.out} // go list -e says a package's error and exits 0 } return forgekey.Ran{Status: 1, Stdout: c.out} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go index b9184fdf8..2afd8fd14 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go @@ -176,7 +176,7 @@ func (m *Module) downloadBeside(ctx context.Context, root, tree string, dirs []s if res := m.Run(ctx, env, "go", "-C", r, "list", "-e", "-deps", "-test", "-f", shutOffLoadFormat, "./..."); res.Status != 0 || res.Err != "" { return fmt.Errorf("go list of the packages of %s beside: %s", filepath.Base(r), causeLine(res.Err+"\n"+res.Stdout+"\n"+res.Stderr)) } - if err := m.shutOffLoad(ctx, env, r, "beside "+filepath.Base(r)); err != nil { + if err := m.shutOffLoad(ctx, env, r, "beside "+filepath.Base(r), false); err != nil { return err } }