diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 8eb13073a..aa9024e38 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "sort" "strings" "sync" @@ -212,7 +213,9 @@ func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d s for { failure := m.downloadAndVerify(ctx, dir, d) if failure == nil { - return taken, nil + // The proof that what was downloaded is all the shut-off steps load: else the fetch fails here, naming the + // package, and not a vet, a test or a build later with less to say (issue 1966). + return taken, m.shutOffLoad(ctx, m.goEnvFor(dir), dir, d) } // A requirement at the zero version names no commit: it is taken from the asked versions or the trunk's latest. took, err := m.takeZeroPins(ctx, root, repository, dir, d, failure.Error(), asked, pins) @@ -232,6 +235,27 @@ func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d s } } +// shutOffLoadFormat is what go list -e says of each package it loads: nothing, or the package and Go's words of its error. +const shutOffLoadFormat = `{{with .Error}}{{$.ImportPath}}: {{.Err}}{{"\n"}}{{end}}` + +// shutOffLoad loads the packages of dir, with their tests and every package they import, as a shut-off step does +// (GOPROXY=off, the step's own environment env), and fails naming each package whose load would reach the network, in +// Go's words. A package that fails to load for another cause (a compile error, an import of a package of the module +// that does not exist) is the shut-off steps' to say, with their own verdict lines, and is not the fetch's. +func (m *Module) shutOffLoad(ctx context.Context, env []string, dir, d string) error { + r := m.Run(ctx, append(env, "GOPROXY=off"), "go", "-C", dir, "list", "-e", "-deps", "-test", "-f", shutOffLoadFormat, "./...") + var lost []string + for _, l := range strings.Split(r.Stdout+"\n"+r.Stderr, "\n") { + if l = strings.TrimSpace(l); strings.Contains(l, "GOPROXY=off") && !slices.Contains(lost, l) { + lost = append(lost, l) + } + } + if len(lost) > 0 { + return fmt.Errorf("the packages of %s load shut off only with what the fetch did not download: %s", d, strings.Join(lost, "; ")) + } + return nil +} + // verifyLocks hold one mutex per digest of a go.mod and go.sum, so two directories with the same pair verify once. var verifyLocks sync.Map diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index d76472a6f..7b91bd94a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" "path/filepath" + "slices" "strings" "sync" "testing" @@ -211,22 +212,54 @@ func TestNoCommandOfTheStepSeesAPasswordOrAToken(t *testing.T) { // 1485/AC3: go mod verify failing ends the step failed naming the directory and the first line of its output. func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) { - f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", goRule: func(line string) forgekey.Ran { - if strings.HasSuffix(line, "tools mod verify") || strings.Contains(line, "/tools mod verify") { - return forgekey.Ran{Status: 1, Stdout: "\nexample.org/x v1.0.0: dir has been modified\nsecond line\n"} - } - return forgekey.Ran{} - }} - m, states := fetchModule(t, f) - if _, err := fetchTool(t, m)(fetchArgs); err != nil { - t.Fatal(err) - } - st := states.waitEnd(t) - if st.State != sdkstep.StepFailed { - t.Fatalf("the step ended %s, want failed", st.State) - } - if !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, "example.org/x v1.0.0: dir has been modified") || strings.Contains(st.Reason, "second line") { - t.Errorf("reason = %q", st.Reason) + const seatPkg = "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + for _, c := range []struct { + name, ran, out string + want []string + notWant string + }{ + {name: "go mod verify fails", ran: "tools mod verify", out: "\nexample.org/x v1.0.0: dir has been modified\nsecond line\n", + want: []string{"tools", "example.org/x v1.0.0: dir has been modified"}, notWant: "second line"}, + // 1966/AC178-2: what the fetch downloaded misses a package the directory's code imports: the shut-off package load + // fails the fetch naming the package in Go's words, not a shut-off step later. + {name: "the shut-off package load would reach the network", ran: "tools list -e -deps -test", + out: "fixture/tools/cmd: \n" + seatPkg + ": module lookup disabled by GOPROXY=off\n", + want: []string{"tools", "shut off", seatPkg + ": module lookup disabled by GOPROXY=off"}, notWant: "fixture/tools/cmd"}, + } { + t.Run(c.name, func(t *testing.T) { + f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", goRule: func(line string) forgekey.Ran { + if strings.Contains(line, "/"+c.ran) { + if strings.Contains(line, " list ") { + return forgekey.Ran{Stdout: c.out} // go list -e says a package's error and exits 0 + } + return forgekey.Ran{Status: 1, Stdout: c.out} + } + return forgekey.Ran{} + }} + m, states := fetchModule(t, f) + if _, err := fetchTool(t, m)(fetchArgs); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != sdkstep.StepFailed { + t.Fatalf("the step ended %s, want failed", st.State) + } + for _, w := range c.want { + if !strings.Contains(st.Reason, w) { + t.Errorf("reason = %q, want it to hold %q", st.Reason, w) + } + } + if strings.Contains(st.Reason, c.notWant) { + t.Errorf("reason = %q holds %q", st.Reason, c.notWant) + } + if c.ran == "tools list -e -deps -test" { + for _, call := range f.calls { + if strings.Contains(call.line, " list -e -deps") && !slices.Contains(call.env, "GOPROXY=off") { + t.Errorf("the package load ran with the network: %s", call.env) + } + } + } + }) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go index 2fc825525..b9184fdf8 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go @@ -171,6 +171,14 @@ func (m *Module) downloadBeside(ctx context.Context, root, tree string, dirs []s if res := m.Run(ctx, env, "go", "-C", r, "mod", "download"); res.Status != 0 || res.Err != "" { return fmt.Errorf("go mod download of the graph of %s beside: %s", filepath.Base(r), causeLine(res.Err+"\n"+res.Stdout+"\n"+res.Stderr)) } + // What the packages' load needs beyond the module graph's download (the go.mod of a version a dependency requires, + // which a build of the composer reads: issue 1966), fetched with the network, then proven shut off. + if res := m.Run(ctx, env, "go", "-C", r, "list", "-e", "-deps", "-test", "-f", shutOffLoadFormat, "./..."); res.Status != 0 || res.Err != "" { + return fmt.Errorf("go list of the packages of %s beside: %s", filepath.Base(r), causeLine(res.Err+"\n"+res.Stdout+"\n"+res.Stderr)) + } + if err := m.shutOffLoad(ctx, env, r, "beside "+filepath.Base(r)); err != nil { + return err + } } return nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/workspace_test.go b/modules/mesh-build-go/cmd/mesh-build-go/workspace_test.go index 49990aeb5..1ad94163e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/workspace_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/workspace_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "testing" @@ -72,7 +73,7 @@ func TestTheFetchDownloadsTheGraphOfASiblingModuleLaidWithTheTreesItRequires(t * put(filepath.Join(root, "beside", "mesh-sdk", "go"), "module git.novox.be/novox/mesh-sdk/go\n\ngo 1.26.0\n") var said []string var lines, envs []string - fail := false + fail, lost := false, "" m := &Module{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { lines = append(lines, name+" "+strings.Join(args, " ")) @@ -80,14 +81,26 @@ func TestTheFetchDownloadsTheGraphOfASiblingModuleLaidWithTheTreesItRequires(t * if fail { return forgekey.Ran{Status: 1, Stderr: "dial tcp: network is unreachable"} } + if slices.Contains(env, "GOPROXY=off") { + return forgekey.Ran{Stdout: lost} + } return forgekey.Ran{} } if err := m.downloadBeside(context.Background(), root, tree, []string{"modules/x"}); err != nil { t.Fatal(err) } - if len(lines) != 1 || lines[0] != "go -C "+host+" mod download" || !strings.Contains(envs[0], "GOWORK="+filepath.Join(root, "beside-0.work")) { - t.Fatalf("commands %q env %q, want one go mod download of the node-engine's module over its workspace", lines, envs) + // The graph's download, the packages' load with the network, then the same load shut off (issue 1966). + load := "go -C " + host + " list -e -deps -test -f " + shutOffLoadFormat + " ./..." + if len(lines) != 3 || lines[0] != "go -C "+host+" mod download" || lines[1] != load || lines[2] != load || + !strings.Contains(envs[0], "GOWORK="+filepath.Join(root, "beside-0.work")) || strings.HasSuffix(envs[1], "GOPROXY=off") || !strings.HasSuffix(envs[2], "GOPROXY=off") { + t.Fatalf("commands %q env %q, want go mod download of the node-engine's module over its workspace, then its packages loaded with the network and shut off", lines, envs) } + // 1966/AC178-3 (its unit half): the shut-off load misses a go.mod the composer's build reads: the fetch fails naming it. + lines, lost = nil, "github.com/nats-io/nkeys: github.com/nats-io/nkeys@v0.4.16 requires golang.org/x/crypto@v0.52.0: module lookup disabled by GOPROXY=off\n" + if err := m.downloadBeside(context.Background(), root, tree, []string{"modules/x"}); err == nil || !strings.Contains(err.Error(), "beside mesh-host") || !strings.Contains(err.Error(), "golang.org/x/crypto@v0.52.0: module lookup disabled") { + t.Errorf("a shut-off load that would reach the network = %v, want the fetch failed naming it", err) + } + lost = "" work, _ := os.ReadFile(filepath.Join(root, "beside-0.work")) for _, want := range []string{filepath.Join(tree, "modules", "x"), filepath.Join(root, "beside", "mesh-sdk", "go"), host} { if !strings.Contains(string(work), want) {