From 3f60d1ff3177e076d8d58774ab163f58d4d7b6cf Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 00:50:56 +0000 Subject: [PATCH 1/4] Build Go modules with Go's own release of the recipe's version, put in the module cache by the probe, so the distribution's build no longer makes the guard refuse (issue 1560, ADR 0476) --- modules/mesh-build-go/README.md | 12 ++-- .../mesh-build-go/cmd/mesh-build-go/build.go | 4 +- .../cmd/mesh-build-go/build_test.go | 2 +- .../cmd/mesh-build-go/digest_test.go | 10 ++- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 4 +- .../cmd/mesh-build-go/fetch_test.go | 4 +- .../mesh-build-go/cmd/mesh-build-go/main.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/probe.go | 65 ++++++++++++++----- .../cmd/mesh-build-go/test_test.go | 2 +- .../mesh-build-go/testdata/captured/README | 4 +- 10 files changed, 76 insertions(+), 33 deletions(-) diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index f523a1e5..0d3526dd 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -14,9 +14,11 @@ the nats-server library's version, read from the program's build information, so ## The guard and the health -At start, every ten minutes, before every step and at every health look, the module reads the installed Go -(`go version`) and writes it with the recipe version to `.go` of its `toolchains` state. A machine whose Go -is not the recipe's refuses every step, the step ending `refused` on its key in `steps` with both versions named, +At start, every ten minutes, before every step and at every health look, the module reads the version of Go's own +release of the recipe's version (`go version` run with `GOTOOLCHAIN=`, ADR 0476) and writes it with the recipe version to `.go` of its `toolchains` state. The probe runs with network and +the module's Go module cache (`GOMODCACHE`): while the release is missing from it, the installed Go downloads it there, +verified against Go's checksum database; no step fetches it. A machine where the release cannot be had, or that does not +report the recipe's version, refuses every step, the step ending `refused` on its key in `steps` with both versions named, and `build_go_health` (the module's `runtime-health`) answers unhealthy with the same text, so the controller raises the machine's condition. Moving to a new Go is a change of `RecipeGo`. @@ -31,7 +33,7 @@ step answers at once with its id and ends on `.` of `steps` (ADR 0469 `build_go_fetch {repository, commit, ref?}` fetches the commit over SSH as the module's account (`mesh-forge` is the forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor, -with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an +with `GOTOOLCHAIN=`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day are removed. @@ -86,7 +88,7 @@ is listed and tested from its vendor directory (`-mod=vendor`). agent's recipe (mesh-controller's `toolchain.go`, the Go entry, copied in `testdata/captured/toolchain-go-entry.txt`, and `builder.go`): `CGO_ENABLED=0`, `GOFLAGS=-trimpath -buildvcs=false` (with `-mod=mod`, or `-mod=vendor` for a directory that vendors), `go build -ldflags "-s -w -X main.builtFor=" -o .mesh-build// ./` in the -module's directory, by the installed Go (`GOTOOLCHAIN=local`) from the module cache the fetch filled and no network +module's directory, by the recipe's Go release (`GOTOOLCHAIN=`, found in the module cache) from the module cache the fetch filled and no network (`GOPROXY=off`). The binary is named as the build agent names it. The output directory is packed with mesh-sdk's `Pack`, so the digest is the build agent's for the same tree. The step ends with the artifact, the binary, the digest, the size and the path of the archive, which stays in the workspace's `artifacts` directory: publishing it is the builder's, in a later diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build.go b/modules/mesh-build-go/cmd/mesh-build-go/build.go index b18915bf..d64769c1 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build.go @@ -145,7 +145,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { return BuildResult{Artifact: art.Name, Binary: binary, Digest: digest, Size: len(archive), Path: path}, nil } -// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed toolchain +// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the recipe's Go release // alone, the modules the fetch put in the module's cache and no way to fetch another (GOPROXY=off), so a build never // reaches the network. A directory that vendors is built from its vendor directory. func (m *Module) buildEnv(dir string) []string { @@ -156,7 +156,7 @@ func (m *Module) buildEnv(dir string) []string { return []string{ "CGO_ENABLED=0", "GOFLAGS=" + recipeFlags + " " + mod, - "GOTOOLCHAIN=local", + "GOTOOLCHAIN=" + RecipeGo, "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPROXY=off", diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build_test.go b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go index 181f7d87..d5598bd5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go @@ -120,7 +120,7 @@ func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) { } } env := strings.Join(c.env, "\n") - for _, want := range []string{compile[1], "GOFLAGS=" + strings.TrimPrefix(compile[2], "GOFLAGS=") + " -mod=mod", "GOTOOLCHAIN=local", "GOPROXY=off", + for _, want := range []string{compile[1], "GOFLAGS=" + strings.TrimPrefix(compile[2], "GOFLAGS=") + " -mod=mod", "GOTOOLCHAIN=go1.27.1", "GOPROXY=off", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild")} { if !strings.Contains(env, want) { t.Errorf("the environment lacks %q:\n%s", want, env) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index 60a4efe4..da9d83bf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -37,7 +37,15 @@ func digestFixture(t *testing.T) (*Module, string) { case "postgres": return forgekey.Ran{Stdout: "postgres (PostgreSQL) " + strconv.Itoa(m.Recipe.PostgresMajor) + ".1\n"} } - return real(ctx, env, name, args...) + // The unit suite has no network: the release (ADR 0476) is not fetched here, the installed Go runs the fixture. + local := []string{} + for _, e := range env { + if e == "GOTOOLCHAIN="+RecipeGo { + e = "GOTOOLCHAIN=local" + } + local = append(local, e) + } + return real(ctx, local, name, args...) } return m, tree } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index bc09c9b3..2318a224 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -112,13 +112,13 @@ func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) { return dirs, nil } -// goEnv is what a go command of a step is given beyond the base environment: the installed toolchain and no other, +// goEnv is what a go command of a step is given beyond the base environment: the recipe's Go release (ADR 0476: the probe put it in the module's cache; a step never fetches it), // modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by // git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory // that vendors, see goEnvFor. func (m *Module) goEnv() []string { return []string{ - "GOTOOLCHAIN=local", + "GOTOOLCHAIN=" + RecipeGo, "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index d223cdaa..03de3b9c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -93,7 +93,7 @@ func fetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) { var fetchArgs = map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA} // 1485/AC1: git is asked through FetchCommit with the ssh remote, and go mod download and verify run in each go.mod -// directory (not in one that vendors) with GOTOOLCHAIN=local, GOFLAGS=-mod=mod and the module's cache paths. +// directory (not in one that vendors) with GOTOOLCHAIN=go1.27.1, GOFLAGS=-mod=mod and the module's cache paths. func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) { f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}} m, states := fetchModule(t, f) @@ -128,7 +128,7 @@ func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) { t.Errorf("a module that vendors was downloaded: %s", c.line) } env := strings.Join(c.env, " ") - for _, want := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be"} { + for _, want := range []string{"GOTOOLCHAIN=go1.27.1", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be"} { if !strings.Contains(env, want) { t.Errorf("%s: environment lacks %s: %s", c.line, want, env) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 2764e81a..5778776b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -33,7 +33,6 @@ func main() { m := &Module{ Node: strings.TrimSpace(os.Getenv("MESH_NODE")), Recipe: Current(), - Probe: ProbeGo, Toolchains: stdio.State(delivery.ToolchainsState).Put, Emit: func(t string, b any) error { return stdio.Emit(t, b) }, Steps: stdio.State(delivery.StepsState).Put, @@ -46,6 +45,7 @@ func main() { m.Home = u.HomeDir } m.Run = StepRunner(m.Home) + m.Probe = ProbeGo(m.Home, m.Cache) // Outside a launch by the runtime — a build, a check — there is no node and no state to write. if m.Node != "" { go m.Start(ctx) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/probe.go b/modules/mesh-build-go/cmd/mesh-build-go/probe.go index 838ece4d..bfc54209 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/probe.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/probe.go @@ -3,28 +3,61 @@ package main import ( "context" "fmt" - "os" - "os/exec" + "path/filepath" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" ) -// ProbeTimeout bounds `go version`. -const ProbeTimeout = 15 * time.Second +// ProbeTimeout bounds `go version` when the release is in the cache; FetchTimeout bounds it while the release is +// missing and the installed Go downloads it (about seventy megabytes). +const ( + ProbeTimeout = 15 * time.Second + FetchTimeout = 5 * time.Minute +) -// ProbeGo reads the installed Go version: the third word `go version` prints ("go version go1.27.1 linux/amd64"), -// as the recipe's RecipeGo is written. GOTOOLCHAIN=local keeps an older Go from fetching another toolchain: what is -// installed is what is read. -func ProbeGo(ctx context.Context) (string, error) { - ctx, cancel := context.WithTimeout(ctx, ProbeTimeout) - defer cancel() - cmd := exec.CommandContext(ctx, "go", "version") - cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local") - out, err := cmd.Output() - if err != nil { - return "", err +// ProbeGo is the probe of the guard (novox/hq ADR 0476 §2, §3; issue 1560): it runs `go version` with +// GOTOOLCHAIN=RecipeGo, so it reads the version of Go's own release of the recipe's version, which is what the +// module builds with, and the third word of its answer is that version. It runs with network and the module's own +// Go module cache (cache/gomod): while the release is missing from it, the installed Go downloads it there, verified +// against Go's checksum database. No step fetches it; a step runs behind this probe and refuses when it is missing. +// The environment is built from scratch like a step's (StepRunner), so the module's bus credentials are not in it. +func ProbeGo(home, cache string) func(ctx context.Context) (string, error) { + run := forgekey.ExecRunnerClean(0) + return func(ctx context.Context) (string, error) { return probeGo(ctx, run, home, cache) } +} + +// probeEnv is what the probe gives `go version` beyond the base environment: the recipe's release as the toolchain, +// the module's cache for it to land in (and the build cache the go command wants), and Go's own proxy and checksum +// database, which this module's steps never leave at their default except here and in fetch. +func probeEnv(cache string) []string { + return []string{ + "GOTOOLCHAIN=" + RecipeGo, + "GOMODCACHE=" + filepath.Join(cache, "gomod"), + "GOCACHE=" + filepath.Join(cache, "gobuild"), } - return parseGoVersion(string(out)) +} + +// releaseCached says whether the release is in the module cache: golang.org/toolchain@v0.0.1-.-. +func releaseCached(cache string) bool { + dirs, _ := filepath.Glob(filepath.Join(cache, "gomod", "golang.org", "toolchain@v0.0.1-"+RecipeGo+".*")) + return len(dirs) > 0 +} + +func probeGo(ctx context.Context, run forgekey.Runner, home, cache string) (string, error) { + bound := ProbeTimeout + if !releaseCached(cache) { + bound = FetchTimeout + } + ctx, cancel := context.WithTimeout(ctx, bound) + defer cancel() + r := run(ctx, append(baseEnv(home), probeEnv(cache)...), "go", "version") + if r.Status != 0 || r.Err != "" { + return "", fmt.Errorf("`go version` with GOTOOLCHAIN=%s failed (the recipe builds with %s): %s", RecipeGo, RecipeGo, + firstLine(r.Err+"\n"+r.Stderr)) + } + return parseGoVersion(r.Stdout) } func parseGoVersion(out string) (string, error) { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go index c13e8085..4fdbda2c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go @@ -175,7 +175,7 @@ func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t * if line != want { t.Errorf("command = %q, want %q", line, want) } - for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod")} { + for _, w := range []string{"GOTOOLCHAIN=go1.27.1", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod")} { if !strings.Contains(env, w) { t.Errorf("environment %s lacks %s", env, w) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README index 5d207ada..7afe1a95 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README @@ -1,8 +1,8 @@ -Each file is the standard output of `GOTOOLCHAIN=local GOFLAGS=-mod=mod go test -count=1 -json ./...` (go1.27.1) +Each file is the standard output of `GOTOOLCHAIN=go1.27.1 GOFLAGS=-mod=mod go test -count=1 -json ./...` (go1.27.1) run in the fixture module of the same name under ../repos, captured on 2026-10-20 for issue 1290. The unit tests feed them to the parser through a fake runner, so no test here runs `go test`. -golist-external.json is `GOTOOLCHAIN=local GOFLAGS=-mod=mod go list -json=ImportPath,Standard,DepOnly,Module +golist-external.json is `GOTOOLCHAIN=go1.27.1 GOFLAGS=-mod=mod go list -json=ImportPath,Standard,DepOnly,Module git.novox.be/novox/mesh-sdk/go/buildstep` run in this module (go1.27.1, 2026-10-20 for issue 1291): a package of a module outside the tree as go list describes it. From c5f8f81a07a6c6c8477ece841ed76123cd2628ff Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 00:51:40 +0000 Subject: [PATCH 2/4] Test that every step and the probe run Go's own release and the probe fills the module cache (issue 1560, ADR 0476) --- .../cmd/mesh-build-go/toolchain_env_test.go | 64 +++++++++ .../toolchain_integration_test.go | 52 ++++++++ .../cmd/mesh-build-go/toolchain_probe_test.go | 124 ++++++++++++++++++ 3 files changed, 240 insertions(+) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/toolchain_env_test.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/toolchain_integration_test.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/toolchain_probe_test.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/toolchain_env_test.go b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_env_test.go new file mode 100644 index 00000000..d1eefa4c --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_env_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1560 (ADR 0476 §1): every environment a step gives its go commands names the recipe's release as the toolchain +// and never GOTOOLCHAIN=local, which would run the distribution's build. +func TestEveryStepEnvironmentCarriesTheRecipesRelease(t *testing.T) { + m := &Module{Cache: t.TempDir()} + vend := t.TempDir() + os.MkdirAll(filepath.Join(vend, "vendor"), 0o755) + os.WriteFile(filepath.Join(vend, "vendor", "modules.txt"), nil, 0o644) + plain := t.TempDir() + envs := map[string][]string{ + "goEnv": m.goEnv(), "goEnvFor(plain)": m.goEnvFor(plain), "goEnvFor(vendor)": m.goEnvFor(vend), + "buildEnv(plain)": m.buildEnv(plain), "buildEnv(vendor)": m.buildEnv(vend), + } + for name, env := range envs { + joined := "\n" + strings.Join(env, "\n") + "\n" + if !strings.Contains(joined, "\nGOTOOLCHAIN="+RecipeGo+"\n") { + t.Errorf("%s lacks GOTOOLCHAIN=%s:\n%s", name, RecipeGo, joined) + } + if strings.Contains(joined, "GOTOOLCHAIN=local") { + t.Errorf("%s still says GOTOOLCHAIN=local:\n%s", name, joined) + } + if !strings.Contains(joined, "\nGOMODCACHE="+filepath.Join(m.Cache, "gomod")+"\n") { + t.Errorf("%s lacks the module cache:\n%s", name, joined) + } + } +} + +// 1560 (ADR 0476 §3): the fetch step downloads the change's locked dependencies and nothing else: no `go version` +// and no toolchain download among its commands, and each go command carries the recipe's release. +func TestTheFetchStepsCommandsFetchNoToolchain(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + if _, err := fetchTool(t, m)(fetchArgs); err != nil { + t.Fatal(err) + } + if st := states.waitEnd(t); st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + goCalls := 0 + for _, c := range f.calls { + if strings.Contains(c.line, "version") || strings.Contains(c.line, "toolchain") { + t.Errorf("the fetch step ran %q", c.line) + } + if strings.HasPrefix(c.line, "go ") { + goCalls++ + if env := strings.Join(c.env, " "); !strings.Contains(env, "GOTOOLCHAIN="+RecipeGo) || strings.Contains(env, "GOTOOLCHAIN=local") { + t.Errorf("%s: environment %s", c.line, env) + } + } + } + if goCalls == 0 { + t.Error("no go command ran") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/toolchain_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_integration_test.go new file mode 100644 index 00000000..710e95ee --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_integration_test.go @@ -0,0 +1,52 @@ +//go:build integration + +package main + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +// 1560 (ADR 0476 §3): the probe, with a fresh module cache and network, lets the installed Go put +// golang.org/toolchain@v0.0.1-.- into the cache and reads the release's version; then a step +// with the cache read-only and no network (GOPROXY=off) reports the recipe's version from it. +func TestTheProbePutsTheReleaseIntoTheCacheAndAStepReadsItOffline(t *testing.T) { + home, cache := t.TempDir(), t.TempDir() + t.Cleanup(func() { // the Go module cache is read-only by Go's own doing + filepath.WalkDir(cache, func(p string, d fs.DirEntry, err error) error { + if err == nil && d.IsDir() { + os.Chmod(p, 0o755) + } + return nil + }) + }) + v, err := ProbeGo(home, cache)(context.Background()) + if err != nil || v != RecipeGo { + t.Fatalf("probe = %q, %v; want %s", v, err, RecipeGo) + } + dirs, _ := filepath.Glob(filepath.Join(cache, "gomod", "golang.org", "toolchain@v0.0.1-"+RecipeGo+".*")) + if len(dirs) != 1 { + t.Fatalf("the cache holds %v, want golang.org/toolchain@v0.0.1-%s.*", dirs, RecipeGo) + } + filepath.WalkDir(cache, func(p string, d fs.DirEntry, err error) error { + if err == nil { + if info, e := d.Info(); e == nil { + os.Chmod(p, info.Mode()&^0o222) + } + } + return nil + }) + r := StepRunner(home)(context.Background(), []string{ + "GOTOOLCHAIN=" + RecipeGo, "GOMODCACHE=" + filepath.Join(cache, "gomod"), "GOCACHE=" + t.TempDir(), "GOPROXY=off", "GOFLAGS=-mod=mod", + }, "go", "version") + if r.Status != 0 || r.Err != "" { + t.Fatalf("go version offline: %s %s", r.Err, r.Stderr) + } + if got, err := parseGoVersion(r.Stdout); err != nil || got != RecipeGo || !strings.Contains(r.Stdout, RecipeGo) { + t.Errorf("the step reports %q, %v; want %s", r.Stdout, err, RecipeGo) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/toolchain_probe_test.go b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_probe_test.go new file mode 100644 index 00000000..ac21510a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_probe_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// probeFake is a runner that records the command and answers by rule. +type probeFake struct { + env []string + line string + ran forgekey.Ran +} + +func (p *probeFake) run(_ context.Context, env []string, name string, args ...string) forgekey.Ran { + p.env, p.line = env, name+" "+strings.Join(args, " ") + return p.ran +} + +// 1560 (ADR 0476 §2, §3): the probe runs `go version` with the recipe's release as the toolchain and the module's +// cache as GOMODCACHE, in an environment built from scratch (nothing of the process's, no GOTOOLCHAIN=local). +func TestTheProbeRunsGoVersionWithTheRecipesReleaseAndTheModuleCache(t *testing.T) { + p := &probeFake{ran: forgekey.Ran{Stdout: "go version go1.27.1 linux/amd64\n"}} + cache := t.TempDir() + v, err := probeGo(context.Background(), p.run, "/home/x", cache) + if err != nil || v != "go1.27.1" { + t.Fatalf("= %q, %v", v, err) + } + env := "\n" + strings.Join(p.env, "\n") + "\n" + for _, want := range []string{"\nGOTOOLCHAIN=" + RecipeGo + "\n", "\nGOMODCACHE=" + filepath.Join(cache, "gomod") + "\n", "\nHOME=/home/x\n"} { + if !strings.Contains(env, want) { + t.Errorf("the probe's environment lacks %q:\n%s", want, env) + } + } + if strings.Contains(env, "GOTOOLCHAIN=local") || p.line != "go version" { + t.Errorf("command %q, environment:\n%s", p.line, env) + } +} + +// 1560: the release's own output parses to its version; the distribution's build parses to a different one. +func TestParseGoVersionOfTheReleaseAndOfTheDistributionsBuild(t *testing.T) { + if v, err := parseGoVersion("go version go1.27.1 linux/amd64\n"); err != nil || v != RecipeGo { + t.Errorf("release: %q, %v", v, err) + } + if v, _ := parseGoVersion("go version go1.27.1-X:nodwarf5 linux/amd64\n"); v == RecipeGo { + t.Errorf("the distribution's build parsed as the release: %q", v) + } +} + +// 1560 (ADR 0476 §2): the toolchains key carries the version the probe read from the release. +func TestTheToolchainsKeyIsTheReleasesVersion(t *testing.T) { + states := newStates() + m := moduleWith(states, "") + p := &probeFake{ran: forgekey.Ran{Stdout: "go version go1.27.1 linux/amd64\n"}} + m.Probe = func(ctx context.Context) (string, error) { return probeGo(ctx, p.run, "/home/x", t.TempDir()) } + if err := m.Check(context.Background()); err != nil { + t.Fatal(err) + } + tc, ok := states.toolchain("ace.go") + if !ok || tc.Version != RecipeGo { + t.Fatalf("ace.go = %+v, %v; want version %s", tc, ok, RecipeGo) + } +} + +// 1560 (ADR 0476 §2): a probe that fails (the release cannot be had) or reads another version makes the module +// refuse, and its health is unhealthy naming the recipe's version and what the probe got. +func TestAFailedFetchOrAnotherVersionMakesHealthUnhealthyNamingBoth(t *testing.T) { + cases := []struct { + name string + ran forgekey.Ran + want []string + }{ + {"fetch fails", forgekey.Ran{Status: 1, Stderr: "go: download go1.27.1 for linux/amd64: dial tcp: no route to host\n"}, + []string{"go1.27.1", "no route to host"}}, + {"other version", forgekey.Ran{Stdout: "go version go1.27.1-X:nodwarf5 linux/amd64\n"}, + []string{"go1.27.1", "go1.27.1-X:nodwarf5"}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + m := moduleWith(newStates(), "") + p := &probeFake{ran: c.ran} + m.Probe = func(ctx context.Context) (string, error) { return probeGo(ctx, p.run, "/home/x", t.TempDir()) } + a := m.Health(context.Background()) + if a.Healthy || a.State != health.StateUnhealthy { + t.Fatalf("health = %+v, want unhealthy", a) + } + for _, w := range c.want { + if !strings.Contains(a.Why, w) { + t.Errorf("health does not name %q: %q", w, a.Why) + } + } + if !strings.Contains(a.Why, "recipe") && !strings.Contains(a.Why, RecipeGo) { + t.Errorf("health does not name the recipe's version: %q", a.Why) + } + }) + } +} + +// 1560: the probe is bound long enough to download while the release is missing, and short once it is cached. +func TestTheProbeIsBoundLongOnlyWhileTheReleaseIsMissing(t *testing.T) { + var got time.Duration + run := func(ctx context.Context, _ []string, _ string, _ ...string) forgekey.Ran { + d, _ := ctx.Deadline() + got = time.Until(d) + return forgekey.Ran{Stdout: "go version go1.27.1 linux/amd64\n"} + } + cache := t.TempDir() + probeGo(context.Background(), run, "/h", cache) + if got < time.Minute { + t.Errorf("missing release: bound %v", got) + } + os.MkdirAll(filepath.Join(cache, "gomod", "golang.org", "toolchain@v0.0.1-"+RecipeGo+".linux-amd64"), 0o755) + probeGo(context.Background(), run, "/h", cache) + if got > ProbeTimeout { + t.Errorf("cached release: bound %v", got) + } +} From 4605fcf46de7c4f9d201a1ba7f1ec4b32325f0b8 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 00:56:03 +0000 Subject: [PATCH 3/4] Fill the shared module cache with the release by the probe before the real-go integration tests, and skip loudly when it cannot (issue 1560, ADR 0476) --- .../test_real_integration_test.go | 56 +++++++++++++++++-- .../toolchain_integration_test.go | 7 ++- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go index 60c2aac7..9a91ef8f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go @@ -7,11 +7,14 @@ import ( "bytes" "compress/gzip" "context" + "fmt" "io" + "io/fs" "os" "os/exec" "path/filepath" "strings" + "sync" "testing" "time" ) @@ -21,7 +24,52 @@ func TestMain(m *testing.M) { if len(os.Args) > 1 && os.Args[1] == busArg { os.Exit(runBus(os.Args[2:])) } - os.Exit(m.Run()) + code := m.Run() + releaseCleanup() + os.Exit(code) +} + +// The module cache the real-go tests share, filled once by the probe as the module's start does (ADR 0476 §3): the +// steps carry GOTOOLCHAIN= and fetch no toolchain, so a test that runs the real go needs the release in its +// cache first. +var ( + releaseOnce sync.Once + releaseCache string + releaseErr error +) + +// needRelease answers the cache with the recipe's release in it, filled by ProbeGo, and skips loudly when the probe +// cannot have it (no network to Go's download service where the check runs); the build node has it (issues 1486, 1560). +func needRelease(t *testing.T, home string) string { + t.Helper() + releaseOnce.Do(func() { + releaseCache, releaseErr = os.MkdirTemp("", "release-cache") + if releaseErr != nil { + return + } + var v string + if v, releaseErr = ProbeGo(home, releaseCache)(context.Background()); releaseErr == nil && v != RecipeGo { + releaseErr = fmt.Errorf("the probe read %s", v) + } + }) + if releaseErr != nil { + t.Skipf("NOT RUN: ADR 0476: the probe could not put %s into the module cache where this check runs (%v); proven live on a build node", RecipeGo, releaseErr) + } + return releaseCache +} + +// releaseCleanup removes the shared cache; Go writes its module cache read-only. +func releaseCleanup() { + if releaseCache == "" { + return + } + filepath.WalkDir(releaseCache, func(p string, d fs.DirEntry, err error) error { + if err == nil && d.IsDir() { + os.Chmod(p, 0o755) + } + return nil + }) + os.RemoveAll(releaseCache) } // needTools skips, loudly, a test whose tools are not installed where the check runs (the mesh's Go toolchain image @@ -68,7 +116,7 @@ func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) { for name, w := range want { tree := repository(t, name) home := t.TempDir() - m := &Module{Home: home, Work: t.TempDir(), Cache: t.TempDir()} + m := &Module{Home: home, Work: t.TempDir(), Cache: needRelease(t, home)} m.Run = StepRunner(home) res, err := m.runLevel(context.Background(), tree, levelOpts{level: LevelUnit, bounds: Bounds{Run: UnitBound}}) if err != nil { @@ -111,7 +159,7 @@ func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing. t.Fatal(err) } home := t.TempDir() - m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()} + m := &Module{Recipe: Current(), Home: home, Work: work, Cache: needRelease(t, home)} m.Run = StepRunner(home) res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil) if err != nil { @@ -149,7 +197,7 @@ func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) { t.Fatal(err) } home := t.TempDir() - m := &Module{Home: home, Work: work, Cache: t.TempDir()} + m := &Module{Home: home, Work: work, Cache: needRelease(t, home)} m.Run = StepRunner(home) args := map[string]any{"workspace": "ws1", "module": ".", "artifact": "tools"} first, err := m.build(context.Background(), args) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/toolchain_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_integration_test.go index 710e95ee..0dee6370 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/toolchain_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_integration_test.go @@ -25,8 +25,11 @@ func TestTheProbePutsTheReleaseIntoTheCacheAndAStepReadsItOffline(t *testing.T) }) }) v, err := ProbeGo(home, cache)(context.Background()) - if err != nil || v != RecipeGo { - t.Fatalf("probe = %q, %v; want %s", v, err, RecipeGo) + if err != nil { + t.Skipf("NOT RUN: ADR 0476: the probe could not put %s into the module cache where this check runs (%v); proven live on a build node", RecipeGo, err) + } + if v != RecipeGo { + t.Fatalf("probe = %q; want %s", v, RecipeGo) } dirs, _ := filepath.Glob(filepath.Join(cache, "gomod", "golang.org", "toolchain@v0.0.1-"+RecipeGo+".*")) if len(dirs) != 1 { From 3c4deb8bee8bb06b2d18144de6759fc3f021d6c7 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 00:59:12 +0000 Subject: [PATCH 4/4] Name the cause of a failed toolchain or module download, not Go's progress line before it (issue 1560, review round 1) --- modules/mesh-build-go/cmd/mesh-build-go/build.go | 2 +- modules/mesh-build-go/cmd/mesh-build-go/digest.go | 2 +- modules/mesh-build-go/cmd/mesh-build-go/fetch.go | 14 +++++++++++++- modules/mesh-build-go/cmd/mesh-build-go/probe.go | 2 +- .../cmd/mesh-build-go/toolchain_probe_test.go | 13 ++++++++++++- 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build.go b/modules/mesh-build-go/cmd/mesh-build-go/build.go index d64769c1..1adb0b5c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build.go @@ -128,7 +128,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { } cmd := []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(art.From, "./")} if r := m.Run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" { - return nil, fmt.Errorf("go build of %s: %s", art.Name, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + return nil, fmt.Errorf("go build of %s: %s", art.Name, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) } archive, digest, err := buildstep.Pack(out) if err != nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index 0adff4f4..2158df98 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -148,7 +148,7 @@ func (m *Module) listPackages(ctx context.Context, tree, dir string, tags []stri args = append(args, "./...") r := m.Run(ctx, m.goEnvFor(abs), "go", args...) if r.Status != 0 || r.Err != "" { - return nil, fmt.Errorf("go list in %s: %s", dir, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + return nil, fmt.Errorf("go list in %s: %s", dir, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) } return parsePackages(r.Stdout) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 2318a224..59d18db8 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -89,7 +89,7 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { for _, verb := range []string{"download", "verify"} { r := m.Run(ctx, m.goEnv(), "go", "-C", filepath.Join(tree, d), "mod", verb) if r.Status != 0 || r.Err != "" { - return nil, fmt.Errorf("go mod %s in %s: %s", verb, d, firstLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr)) + return nil, fmt.Errorf("go mod %s in %s: %s", verb, d, causeLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr)) } } } @@ -164,6 +164,18 @@ func (m *Module) clock() time.Time { } func (m *Module) now() time.Time { return m.clock() } +// causeLine is the first line of a go command's output that says something other than Go's own progress: `go: +// downloading ...` comes first when Go fetches a toolchain or a module, and the cause of a failure on a later line +// (issue 1560: a refusal read only "go: downloading go1.27.1 (linux/amd64)"). +func causeLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "go: downloading ") { + return l + } + } + return firstLine(s) +} + func firstLine(s string) string { for _, l := range strings.Split(s, "\n") { if l = strings.TrimSpace(l); l != "" { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/probe.go b/modules/mesh-build-go/cmd/mesh-build-go/probe.go index bfc54209..1d96435b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/probe.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/probe.go @@ -55,7 +55,7 @@ func probeGo(ctx context.Context, run forgekey.Runner, home, cache string) (stri r := run(ctx, append(baseEnv(home), probeEnv(cache)...), "go", "version") if r.Status != 0 || r.Err != "" { return "", fmt.Errorf("`go version` with GOTOOLCHAIN=%s failed (the recipe builds with %s): %s", RecipeGo, RecipeGo, - firstLine(r.Err+"\n"+r.Stderr)) + causeLine(r.Err+"\n"+r.Stderr)) } return parseGoVersion(r.Stdout) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/toolchain_probe_test.go b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_probe_test.go index ac21510a..f2b0a4d5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/toolchain_probe_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/toolchain_probe_test.go @@ -77,7 +77,7 @@ func TestAFailedFetchOrAnotherVersionMakesHealthUnhealthyNamingBoth(t *testing.T ran forgekey.Ran want []string }{ - {"fetch fails", forgekey.Ran{Status: 1, Stderr: "go: download go1.27.1 for linux/amd64: dial tcp: no route to host\n"}, + {"fetch fails", forgekey.Ran{Status: 1, Stderr: "go: downloading go1.27.1 (linux/amd64)\ngo: download go1.27.1 for linux/amd64: dial tcp: no route to host\n"}, []string{"go1.27.1", "no route to host"}}, {"other version", forgekey.Ran{Stdout: "go version go1.27.1-X:nodwarf5 linux/amd64\n"}, []string{"go1.27.1", "go1.27.1-X:nodwarf5"}}, @@ -122,3 +122,14 @@ func TestTheProbeIsBoundLongOnlyWhileTheReleaseIsMissing(t *testing.T) { t.Errorf("cached release: bound %v", got) } } + +// 1560 (review): Go's progress line is not the cause: the refusal names the line after it. +func TestCauseLineSkipsGosDownloadingLine(t *testing.T) { + got := causeLine("go: downloading go1.27.1 (linux/amd64)\ngo: download go1.27.1: checksum mismatch\n") + if got != "go: download go1.27.1: checksum mismatch" { + t.Errorf("causeLine = %q", got) + } + if got := causeLine("go: downloading go1.27.1 (linux/amd64)\n"); got != "go: downloading go1.27.1 (linux/amd64)" { + t.Errorf("only a progress line: %q", got) + } +}