From 81287cb11dc5a273276edcbf33ff3cde4693a96d Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 7 Nov 2026 08:32:14 +0100 Subject: [PATCH] The check's fetch downloads the graph of a sibling Go module laid with the trees it requires; the compose test fetches nothing (issue 1921, milestone 47/G, the group check of c5d32074c945) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../mesh-build-go/compose_integration_test.go | 11 +- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 3 + .../cmd/mesh-build-go/workspace.go | 138 ++++++++++++++++++ .../cmd/mesh-build-go/workspace_test.go | 131 ++++++++--------- 4 files changed, 200 insertions(+), 83 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/workspace.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/compose_integration_test.go index e69064ac5..3310fdea5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/compose_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose_integration_test.go @@ -5,7 +5,6 @@ package main import ( "context" "encoding/json" - "fmt" "io/fs" "os" "os/exec" @@ -191,14 +190,8 @@ func TestAComposeAskIsBuiltOnceAndAnsweredOnTheBus(t *testing.T) { return nil, err } workFile = filepath.Join(m.Work, ws, "go.work") - // What the real fetch does before any build (fetchInto, downloadAndVerify): every module the graph - // names is downloaded into the cache, so the build with GOPROXY=off needs none, however much of the - // graph this machine's cache (filled by this module's own build) lacks. Go here is the same go - // command in the same environment, with the workspace in place of the tree's own go.mod. - r := m.Run(ctx, m.goEnv(), "go", "-C", tree, "mod", "download") - if r.Status != 0 || r.Err != "" { - return nil, fmt.Errorf("NOT CHECKED: the composer's modules cannot be downloaded as the fetch step downloads them: %s", causeLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr)) - } + // Nothing is downloaded here: this level has no network. The check's own fetch downloaded this workspace's + // graph (downloadBeside), and a module it lacks fails the build naming it. } return map[string]any{"workspace": ws}, nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 98c1200e5..59a06ade8 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -189,6 +189,9 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str return nil, err } } + if err := m.downloadBeside(ctx, root, tree, dirs); err != nil { + return nil, err + } for _, t := range taken { res.Packages = append(res.Packages, t...) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go new file mode 100644 index 000000000..e38bb1fd1 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// zeroVersion is the version a mesh package is required at in every go.mod of the mesh (ADR 0573 §2, §4). +const zeroVersion = "v0.0.0-00010101000000-000000000000" + +// workspaceOf is the go.work of the module in first and of every module it requires at the zero version, transitively, +// each found among the modules in candidates by its module path: each used, and each also replaced at the zero version by +// its own directory. Go (from 1.27) looks the zero version of a required module up even where the module is a member of +// the workspace, once a build loads the module graph (any import outside the workspace); a versioned replace is the one +// form it takes without a lookup, and an unversioned one is refused for a member of the workspace (issue 1921: the +// composer's build fetched mesh-contracts from the network). Only what first needs is laid (issue 1921, the group check +// of 8a80e8e86858): a module laid beside that requires one no tree holds (mesh-grants requiring a catalogue module, the +// catalogue being the repository under check) made Go fetch that module's zero version and fail, though the composer +// needs neither. A requirement at the zero version no candidate holds is an error naming it and who requires it. +func workspaceOf(first string, candidates []string) (string, error) { + paths := map[string]string{} + for _, d := range candidates { + path, _, err := goModOf(d) + if err != nil { + return "", err + } + if _, seen := paths[path]; !seen { // the same module laid twice: the first is kept + paths[path] = d + } + } + var use, replace strings.Builder + laid := map[string]bool{} + queue := []string{first} + for len(queue) > 0 { + d := queue[0] + queue = queue[1:] + path, needs, err := goModOf(d) + if err != nil { + return "", err + } + if laid[path] { + continue + } + laid[path] = true + fmt.Fprintf(&use, "\t%s\n", d) + fmt.Fprintf(&replace, "replace %s %s => %s\n", path, zeroVersion, d) + for _, need := range needs { + if laid[need] { + continue + } + dir, ok := paths[need] + if !ok { + return "", fmt.Errorf("%s (%s) requires %s at the zero version, and no tree laid beside holds it", path, d, need) + } + queue = append(queue, dir) + } + } + return "go 1.26.0\n\nuse (\n" + use.String() + ")\n\n" + replace.String(), nil +} + +// goModOf is the module path of the go.mod in d and the modules it requires at the zero version. +func goModOf(d string) (path string, needs []string, err error) { + raw, err := os.ReadFile(filepath.Join(d, "go.mod")) + if err != nil { + return "", nil, err + } + for _, line := range strings.Split(string(raw), "\n") { + f := strings.Fields(strings.TrimPrefix(strings.TrimSpace(line), "require ")) + switch { + case len(f) == 2 && f[0] == "module" && path == "": + path = strings.Trim(f[1], `"`) + case len(f) >= 2 && f[1] == zeroVersion: + needs = append(needs, strings.Trim(f[0], `"`)) + } + } + if path == "" { + return "", nil, fmt.Errorf("%s/go.mod names no module", d) + } + return path, needs, nil +} + +// downloadBeside downloads, where the network is, the Go modules of the graph that the tests of a workspace's siblings +// build: a sibling that is a Go module at its root (the node-engine's repository, whose composer a catalogue test builds) +// is laid with what it requires at the zero version, found in the fetched tree and the other siblings (workspaceOf), as +// a go.work, and its graph is downloaded into the module cache. The integration level runs shut off from the network and +// builds with GOPROXY=off, so a module this fetch did not download is one it cannot get. A sibling whose requirement no +// tree holds is said and left out: the test that builds it fails naming that module. A download that fails fails the fetch. +func (m *Module) downloadBeside(ctx context.Context, root, tree string, dirs []string) error { + beside := filepath.Join(root, "beside") + entries, err := os.ReadDir(beside) + if err != nil { + return nil // no siblings placed + } + var candidates, roots []string + for _, d := range dirs { + candidates = append(candidates, filepath.Join(tree, filepath.FromSlash(d))) + } + for _, e := range entries { + dir := filepath.Join(beside, e.Name()) + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + roots = append(roots, dir) + } + filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if d.IsDir() && (d.Name() == ".git" || d.Name() == "testdata" || d.Name() == "node_modules") { + return filepath.SkipDir + } + if !d.IsDir() && d.Name() == "go.mod" { + candidates = append(candidates, filepath.Dir(p)) + } + return nil + }) + } + for i, r := range roots { + work, err := workspaceOf(r, candidates) + if err != nil { + if m.Say != nil { + m.Say("beside %s: its modules are not downloaded: %v", filepath.Base(r), err) + } + continue + } + file := filepath.Join(root, fmt.Sprintf("beside-%d.work", i)) + if err := os.WriteFile(file, []byte(work), 0o600); err != nil { + return err + } + env := append(m.goEnv(), "GOWORK="+file, "GOFLAGS=-mod=readonly") // a workspace takes no -mod=mod + if res := m.Run(ctx, env, "go", "-C", r, "mod", "download"); res.Status != 0 || res.Err != "" { + return fmt.Errorf("go mod download of the graph of %s beside: %s", filepath.Base(r), causeLine(res.Err+"\n"+res.Stdout+"\n"+res.Stderr)) + } + } + return nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/workspace_test.go b/modules/mesh-build-go/cmd/mesh-build-go/workspace_test.go index 17d4c3b72..49990aeb5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/workspace_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/workspace_test.go @@ -1,88 +1,17 @@ package main import ( + "context" "fmt" "os" "os/exec" "path/filepath" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" ) -// zeroVersion is the version a mesh package is required at in every go.mod of the mesh (ADR 0573 §2, §4). -const zeroVersion = "v0.0.0-00010101000000-000000000000" - -// workspaceOf is the go.work of the module in first and of every module it requires at the zero version, transitively, -// each found among the modules in candidates by its module path: each used, and each also replaced at the zero version by -// its own directory. Go (from 1.27) looks the zero version of a required module up even where the module is a member of -// the workspace, once a build loads the module graph (any import outside the workspace); a versioned replace is the one -// form it takes without a lookup, and an unversioned one is refused for a member of the workspace (issue 1921: the -// composer's build fetched mesh-contracts from the network). Only what first needs is laid (issue 1921, the group check -// of 8a80e8e86858): a module laid beside that requires one no tree holds (mesh-grants requiring a catalogue module, the -// catalogue being the repository under check) made Go fetch that module's zero version and fail, though the composer -// needs neither. A requirement at the zero version no candidate holds is an error naming it and who requires it. -func workspaceOf(first string, candidates []string) (string, error) { - paths := map[string]string{} - for _, d := range candidates { - path, _, err := goModOf(d) - if err != nil { - return "", err - } - if _, seen := paths[path]; !seen { // the same module laid twice: the first is kept - paths[path] = d - } - } - var use, replace strings.Builder - laid := map[string]bool{} - queue := []string{first} - for len(queue) > 0 { - d := queue[0] - queue = queue[1:] - path, needs, err := goModOf(d) - if err != nil { - return "", err - } - if laid[path] { - continue - } - laid[path] = true - fmt.Fprintf(&use, "\t%s\n", d) - fmt.Fprintf(&replace, "replace %s %s => %s\n", path, zeroVersion, d) - for _, need := range needs { - if laid[need] { - continue - } - dir, ok := paths[need] - if !ok { - return "", fmt.Errorf("%s (%s) requires %s at the zero version, and no tree laid beside holds it", path, d, need) - } - queue = append(queue, dir) - } - } - return "go 1.26.0\n\nuse (\n" + use.String() + ")\n\n" + replace.String(), nil -} - -// goModOf is the module path of the go.mod in d and the modules it requires at the zero version. -func goModOf(d string) (path string, needs []string, err error) { - raw, err := os.ReadFile(filepath.Join(d, "go.mod")) - if err != nil { - return "", nil, err - } - for _, line := range strings.Split(string(raw), "\n") { - f := strings.Fields(strings.TrimPrefix(strings.TrimSpace(line), "require ")) - switch { - case len(f) == 2 && f[0] == "module" && path == "": - path = strings.Trim(f[1], `"`) - case len(f) >= 2 && f[1] == zeroVersion: - needs = append(needs, strings.Trim(f[0], `"`)) - } - } - if path == "" { - return "", nil, fmt.Errorf("%s/go.mod names no module", d) - } - return path, needs, nil -} - // Issue 1921 (the fourth group try): a workspace whose members require each other at the zero version loads its module // graph with no lookup, so with nothing fetched (GOPROXY=off), as a composer built beside the group's heads does. func TestAWorkspaceOfTheGroupResolvesTheZeroVersionWithNoLookup(t *testing.T) { @@ -122,3 +51,57 @@ func TestAWorkspaceOfTheGroupResolvesTheZeroVersionWithNoLookup(t *testing.T) { t.Errorf("a requirement no tree holds: %v, want an error naming it", err) } } + +// Issue 1921 (the group check of c5d32074c945): the integration level has no network, so the fetch, which has, downloads +// the graph the composer test builds: a sibling that is a Go module at its root, laid with what it requires at the zero +// version, found among the fetched tree and the other siblings. A failed download fails the fetch, naming the sibling. +func TestTheFetchDownloadsTheGraphOfASiblingModuleLaidWithTheTreesItRequires(t *testing.T) { + root := t.TempDir() + put := func(dir, body string) string { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return dir + } + tree := filepath.Join(root, "tree") + put(filepath.Join(tree, "modules", "x"), "module git.novox.be/novox/mesh-catalog/modules/x\n\ngo 1.26.0\n") + host := put(filepath.Join(root, "beside", "mesh-host"), "module github.com/novox/mesh-host\n\ngo 1.26.0\n\nrequire (\n\tgit.novox.be/novox/mesh-catalog/modules/x "+zeroVersion+"\n\tgit.novox.be/novox/mesh-sdk/go "+zeroVersion+"\n)\n") + put(filepath.Join(root, "beside", "mesh-sdk", "go"), "module git.novox.be/novox/mesh-sdk/go\n\ngo 1.26.0\n") + var said []string + var lines, envs []string + fail := false + m := &Module{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + lines = append(lines, name+" "+strings.Join(args, " ")) + envs = append(envs, strings.Join(env, "\n")) + if fail { + return forgekey.Ran{Status: 1, Stderr: "dial tcp: network is unreachable"} + } + return forgekey.Ran{} + } + if err := m.downloadBeside(context.Background(), root, tree, []string{"modules/x"}); err != nil { + t.Fatal(err) + } + if len(lines) != 1 || lines[0] != "go -C "+host+" mod download" || !strings.Contains(envs[0], "GOWORK="+filepath.Join(root, "beside-0.work")) { + t.Fatalf("commands %q env %q, want one go mod download of the node-engine's module over its workspace", lines, envs) + } + work, _ := os.ReadFile(filepath.Join(root, "beside-0.work")) + for _, want := range []string{filepath.Join(tree, "modules", "x"), filepath.Join(root, "beside", "mesh-sdk", "go"), host} { + if !strings.Contains(string(work), want) { + t.Errorf("the workspace lays no %s:\n%s", want, work) + } + } + fail = true + if err := m.downloadBeside(context.Background(), root, tree, []string{"modules/x"}); err == nil || !strings.Contains(err.Error(), "mesh-host") || !strings.Contains(err.Error(), "network is unreachable") { + t.Errorf("a failed download = %v, want it named", err) + } + // A requirement no tree holds is said and the sibling left out; the test that builds it fails naming the module. + put(filepath.Join(root, "beside", "mesh-host"), "module github.com/novox/mesh-host\n\ngo 1.26.0\n\nrequire git.novox.be/novox/nobody/go "+zeroVersion+"\n") + lines, said = nil, nil + if err := m.downloadBeside(context.Background(), root, tree, nil); err != nil || len(lines) != 0 || len(said) != 1 || !strings.Contains(said[0], "nobody/go") { + t.Errorf("an unheld requirement: err %v, commands %q, said %q", err, lines, said) + } +}