From 77b30bd2966c3077441944a3a72b99bdcee4746a Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 7 Nov 2026 11:36:08 +0100 Subject: [PATCH 1/2] The check's fetch downloads the graph of a sibling Go module laid with the trees it requires (issue 1921, milestone 47/G trunk step) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-build-go/beside_digest_test.go | 17 +- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 3 + .../cmd/mesh-build-go/workspace.go | 145 ++++++++++++++++++ 3 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/workspace.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go index f59242f7c..c5a7bccf7 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go @@ -83,7 +83,8 @@ func TestASiblingsCommitChangesTheDigestOfAPackageWhoseTestsReachAReaderOfTheSib // The fetch writes the siblings it placed, which the digest reads. func TestTheFetchRecordsTheSiblingsItPlaced(t *testing.T) { - f := &fakeStep{gomods: "go.mod\n"} + // Every checkout writes this go.mod: the placed sibling is a Go module at its root, so the fetch downloads its graph. + f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "module git.novox.be/novox/mesh-sdk/go\n\ngo 1.26.0\n"}} m, states := fetchModule(t, f) args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{ map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}} @@ -101,6 +102,20 @@ func TestTheFetchRecordsTheSiblingsItPlaced(t *testing.T) { if err != nil || json.Unmarshal(raw, &placed) != nil || len(placed) != 1 || placed[0].Repository != "novox/mesh-sdk" || placed[0].Commit != commitSHA { t.Errorf("the record is %s (%v)", raw, err) } + // The placed sibling's graph is downloaded once, over the workspace the fetch laid for it (downloadBeside). + want := "go -C " + filepath.Join(m.Work, res.Workspace, "beside", "mesh-sdk") + " mod download" + n := 0 + for _, c := range f.calls { + if c.line == want { + n++ + if !strings.Contains(strings.Join(c.env, "\n"), "GOWORK="+filepath.Join(m.Work, res.Workspace, "beside-0.work")) { + t.Errorf("%q ran without the sibling's workspace: %q", want, c.env) + } + } + } + if n != 1 { + t.Errorf("%q ran %d times, want once:\n%s", want, n, strings.Join(f.lines(), "\n")) + } } // The checker's agreement: the integration level runs and digests under the tags integration and isolation, so the diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index a1988224a..e8707c2f5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -189,6 +189,9 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str return nil, err } } + if err := m.downloadBeside(ctx, root, tree, dirs); err != nil { + return nil, err + } for _, t := range taken { res.Packages = append(res.Packages, t...) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go new file mode 100644 index 000000000..5b46f23f4 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go @@ -0,0 +1,145 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// zeroVersion is the version a mesh package is required at in every go.mod of the mesh (ADR 0573 §2, §4). +const zeroVersion = "v0.0.0-00010101000000-000000000000" + +// workspaceOf is the go.work of the module in first and of every module it requires at the zero version, transitively, +// each found among the modules in candidates by its module path: each used, and each also replaced at the zero version by +// its own directory. Go (from 1.27) looks the zero version of a required module up even where the module is a member of +// the workspace, once a build loads the module graph (any import outside the workspace); a versioned replace is the one +// form it takes without a lookup, and an unversioned one is refused for a member of the workspace (issue 1921: the +// composer's build fetched mesh-contracts from the network). Only what first needs is laid (issue 1921, the group check +// of 8a80e8e86858): a module laid beside that requires one no tree holds (mesh-grants requiring a catalogue module, the +// catalogue being the repository under check) made Go fetch that module's zero version and fail, though the composer +// needs neither. A requirement at the zero version no candidate holds is an error naming it and who requires it. +func workspaceOf(first string, candidates []string) (string, error) { + paths := map[string]string{} + for _, d := range candidates { + path, _, err := goModOf(d) + if err != nil { + return "", err + } + if _, seen := paths[path]; !seen { // the same module laid twice: the first is kept + paths[path] = d + } + } + var use, replace strings.Builder + laid := map[string]bool{} + queue := []string{first} + for len(queue) > 0 { + d := queue[0] + queue = queue[1:] + path, needs, err := goModOf(d) + if err != nil { + return "", err + } + if laid[path] { + continue + } + laid[path] = true + fmt.Fprintf(&use, "\t%s\n", d) + fmt.Fprintf(&replace, "replace %s %s => %s\n", path, zeroVersion, d) + for _, need := range needs { + if laid[need] { + continue + } + dir, ok := paths[need] + if !ok { + return "", fmt.Errorf("%s (%s) requires %s at the zero version, and no tree laid beside holds it", path, d, need) + } + queue = append(queue, dir) + } + } + return "go 1.26.0\n\nuse (\n" + use.String() + ")\n\n" + replace.String(), nil +} + +// goModOf is the module path of the go.mod in d and the modules it requires at the zero version. +func goModOf(d string) (path string, needs []string, err error) { + raw, err := os.ReadFile(filepath.Join(d, "go.mod")) + if err != nil { + return "", nil, err + } + for _, line := range strings.Split(string(raw), "\n") { + f := strings.Fields(strings.TrimPrefix(strings.TrimSpace(line), "require ")) + switch { + case len(f) == 2 && f[0] == "module" && path == "": + path = strings.Trim(f[1], `"`) + case len(f) >= 2 && f[1] == zeroVersion: + needs = append(needs, strings.Trim(f[0], `"`)) + } + } + if path == "" { + return "", nil, fmt.Errorf("%s/go.mod names no module", d) + } + return path, needs, nil +} + +// downloadBeside downloads, where the network is, the Go modules of the graph that the tests of a workspace's siblings +// build: a sibling that is a Go module at its root (the node-engine's repository, whose composer a catalogue test builds) +// is laid with what it requires at the zero version, found in the fetched tree and the other siblings (workspaceOf), as +// a go.work, and its graph is downloaded into the module cache. The integration level runs shut off from the network and +// builds with GOPROXY=off, so a module this fetch did not download is one it cannot get. A sibling whose requirement no +// tree holds is said and left out: the test that builds it fails naming that module. A download that fails fails the fetch. +func (m *Module) downloadBeside(ctx context.Context, root, tree string, dirs []string) error { + beside := filepath.Join(root, "beside") + entries, err := os.ReadDir(beside) + if errors.Is(err, fs.ErrNotExist) { + return nil // no siblings placed + } + if err != nil { + return err + } + var candidates, roots []string + for _, d := range dirs { + candidates = append(candidates, filepath.Join(tree, filepath.FromSlash(d))) + } + for _, e := range entries { + dir := filepath.Join(beside, e.Name()) + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + roots = append(roots, dir) + } + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && (d.Name() == ".git" || d.Name() == "testdata" || d.Name() == "node_modules") { + return filepath.SkipDir + } + if !d.IsDir() && d.Name() == "go.mod" { + candidates = append(candidates, filepath.Dir(p)) + } + return nil + }) + if err != nil { + return err + } + } + for i, r := range roots { + work, err := workspaceOf(r, candidates) + if err != nil { + if m.Say != nil { + m.Say("beside %s: its modules are not downloaded: %v", filepath.Base(r), err) + } + continue + } + file := filepath.Join(root, fmt.Sprintf("beside-%d.work", i)) + if err := os.WriteFile(file, []byte(work), 0o600); err != nil { + return err + } + env := append(m.goEnv(), "GOWORK="+file, "GOFLAGS=-mod=readonly") // a workspace takes no -mod=mod + if res := m.Run(ctx, env, "go", "-C", r, "mod", "download"); res.Status != 0 || res.Err != "" { + return fmt.Errorf("go mod download of the graph of %s beside: %s", filepath.Base(r), causeLine(res.Err+"\n"+res.Stdout+"\n"+res.Stderr)) + } + } + return nil +} From 36c19694c9df894fee4231f669e924ef5dd988f1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 7 Nov 2026 11:49:42 +0100 Subject: [PATCH 2/2] The fetch walks the siblings for Go modules only when one is a Go module at its root; the beside workspace's go directive is the highest among its modules (issue 1921) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-build-go/beside_digest_test.go | 20 ++++++- .../cmd/mesh-build-go/workspace.go | 55 +++++++++++++++---- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go index c5a7bccf7..293cb0054 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "os" "path/filepath" @@ -84,7 +85,7 @@ func TestASiblingsCommitChangesTheDigestOfAPackageWhoseTestsReachAReaderOfTheSib // The fetch writes the siblings it placed, which the digest reads. func TestTheFetchRecordsTheSiblingsItPlaced(t *testing.T) { // Every checkout writes this go.mod: the placed sibling is a Go module at its root, so the fetch downloads its graph. - f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "module git.novox.be/novox/mesh-sdk/go\n\ngo 1.26.0\n"}} + f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "module git.novox.be/novox/mesh-sdk/go\n\ngo 1.27.1\n"}} m, states := fetchModule(t, f) args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{ map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}} @@ -113,6 +114,23 @@ func TestTheFetchRecordsTheSiblingsItPlaced(t *testing.T) { } } } + // The go.work's go directive is the highest of the laid modules. + if work, _ := os.ReadFile(filepath.Join(m.Work, res.Workspace, "beside-0.work")); !strings.HasPrefix(string(work), "go 1.27.1\n") { + t.Errorf("the workspace file begins %q, want go 1.27.1", work) + } + // No sibling a Go module at its root: nothing is walked, so a directory the walk cannot read does not fail the fetch. + plain := t.TempDir() + locked := filepath.Join(plain, "beside", "docs", "private") + if err := os.MkdirAll(locked, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(locked, 0o700) }) + if err := m.downloadBeside(context.Background(), plain, filepath.Join(plain, "tree"), nil); err != nil { + t.Errorf("a fetch with no Go-module sibling walked the siblings: %v", err) + } if n != 1 { t.Errorf("%q ran %d times, want once:\n%s", want, n, strings.Join(f.lines(), "\n")) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go index 5b46f23f4..2fc825525 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" "path/filepath" + "strconv" "strings" ) @@ -25,7 +26,7 @@ const zeroVersion = "v0.0.0-00010101000000-000000000000" func workspaceOf(first string, candidates []string) (string, error) { paths := map[string]string{} for _, d := range candidates { - path, _, err := goModOf(d) + path, _, _, err := goModOf(d) if err != nil { return "", err } @@ -35,11 +36,12 @@ func workspaceOf(first string, candidates []string) (string, error) { } var use, replace strings.Builder laid := map[string]bool{} + highest := "1.26.0" // the go directive is the highest among the laid modules, never lower than this queue := []string{first} for len(queue) > 0 { d := queue[0] queue = queue[1:] - path, needs, err := goModOf(d) + path, needs, goV, err := goModOf(d) if err != nil { return "", err } @@ -47,6 +49,9 @@ func workspaceOf(first string, candidates []string) (string, error) { continue } laid[path] = true + if goNewer(goV, highest) { + highest = goV + } fmt.Fprintf(&use, "\t%s\n", d) fmt.Fprintf(&replace, "replace %s %s => %s\n", path, zeroVersion, d) for _, need := range needs { @@ -60,28 +65,48 @@ func workspaceOf(first string, candidates []string) (string, error) { queue = append(queue, dir) } } - return "go 1.26.0\n\nuse (\n" + use.String() + ")\n\n" + replace.String(), nil + return "go " + highest + "\n\nuse (\n" + use.String() + ")\n\n" + replace.String(), nil } -// goModOf is the module path of the go.mod in d and the modules it requires at the zero version. -func goModOf(d string) (path string, needs []string, err error) { +// goNewer reports whether the go directive a is higher than b (dotted numbers; a missing part is 0). +func goNewer(a, b string) bool { + x, y := strings.Split(a, "."), strings.Split(b, ".") + for i := 0; i < len(x) || i < len(y); i++ { + var p, q int + if i < len(x) { + p, _ = strconv.Atoi(x[i]) + } + if i < len(y) { + q, _ = strconv.Atoi(y[i]) + } + if p != q { + return p > q + } + } + return false +} + +// goModOf is the module path of the go.mod in d, the modules it requires at the zero version and its go directive. +func goModOf(d string) (path string, needs []string, goV string, err error) { raw, err := os.ReadFile(filepath.Join(d, "go.mod")) if err != nil { - return "", nil, err + return "", nil, "", err } for _, line := range strings.Split(string(raw), "\n") { f := strings.Fields(strings.TrimPrefix(strings.TrimSpace(line), "require ")) switch { case len(f) == 2 && f[0] == "module" && path == "": path = strings.Trim(f[1], `"`) + case len(f) == 2 && f[0] == "go" && goV == "": + goV = f[1] case len(f) >= 2 && f[1] == zeroVersion: needs = append(needs, strings.Trim(f[0], `"`)) } } if path == "" { - return "", nil, fmt.Errorf("%s/go.mod names no module", d) + return "", nil, "", fmt.Errorf("%s/go.mod names no module", d) } - return path, needs, nil + return path, needs, goV, nil } // downloadBeside downloads, where the network is, the Go modules of the graph that the tests of a workspace's siblings @@ -99,15 +124,21 @@ func (m *Module) downloadBeside(ctx context.Context, root, tree string, dirs []s if err != nil { return err } - var candidates, roots []string - for _, d := range dirs { - candidates = append(candidates, filepath.Join(tree, filepath.FromSlash(d))) - } + var candidates, roots, all []string for _, e := range entries { dir := filepath.Join(beside, e.Name()) + all = append(all, dir) if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { roots = append(roots, dir) } + } + if len(roots) == 0 { + return nil // no sibling is a Go module at its root: nothing is walked, nothing downloaded + } + for _, d := range dirs { + candidates = append(candidates, filepath.Join(tree, filepath.FromSlash(d))) + } + for _, dir := range all { err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { if err != nil { return err