Build Go modules with Go's own release of the recipe's version, put in the module cache by the probe, so the distribution's build no longer makes the guard refuse (issue 1560, ADR 0476)

2026-10-21 00:50:56 +00:00
parent 76bb22c362
commit 3f60d1ff31
10 changed files with 76 additions and 33 deletions
@@ -14,9 +14,11 @@ the nats-server library's version, read from the program's build information, so

 ## The guard and the health

-At start, every ten minutes, before every step and at every health look, the module reads the installed Go
-(`go version`) and writes it with the recipe version to `<node>.go` of its `toolchains` state. A machine whose Go
-is not the recipe's refuses every step, the step ending `refused` on its key in `steps` with both versions named,
+At start, every ten minutes, before every step and at every health look, the module reads the version of Go's own
+release of the recipe's version (`go version` run with `GOTOOLCHAIN=<RecipeGo>`, ADR 0476) and writes it with the recipe version to `<node>.go` of its `toolchains` state. The probe runs with network and
+the module's Go module cache (`GOMODCACHE`): while the release is missing from it, the installed Go downloads it there,
+verified against Go's checksum database; no step fetches it. A machine where the release cannot be had, or that does not
+report the recipe's version, refuses every step, the step ending `refused` on its key in `steps` with both versions named,
 and `build_go_health` (the module's `runtime-health`) answers unhealthy with the same text, so the controller raises
 the machine's condition. Moving to a new Go is a change of `RecipeGo`.

@@ -31,7 +33,7 @@ step answers at once with its id and ends on `<node>.<id>` of `steps` (ADR 0469
 `build_go_fetch {repository, commit, ref?}` fetches the commit over SSH as the module's account (`mesh-forge` is the
 forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the
 work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor,
-with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an
+with `GOTOOLCHAIN=<RecipeGo>`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an
 environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day
 are removed.

@@ -86,7 +88,7 @@ is listed and tested from its vendor directory (`-mod=vendor`).
 agent's recipe (mesh-controller's `toolchain.go`, the Go entry, copied in `testdata/captured/toolchain-go-entry.txt`, and
 `builder.go`): `CGO_ENABLED=0`, `GOFLAGS=-trimpath -buildvcs=false` (with `-mod=mod`, or `-mod=vendor` for a directory
 that vendors), `go build -ldflags "-s -w -X main.builtFor=<system>" -o .mesh-build/<artifact>/<binary> ./<from>` in the
-module's directory, by the installed Go (`GOTOOLCHAIN=local`) from the module cache the fetch filled and no network
+module's directory, by the recipe's Go release (`GOTOOLCHAIN=<RecipeGo>`, found in the module cache) from the module cache the fetch filled and no network
 (`GOPROXY=off`). The binary is named as the build agent names it. The output directory is packed with mesh-sdk's `Pack`,
 so the digest is the build agent's for the same tree. The step ends with the artifact, the binary, the digest, the size and
 the path of the archive, which stays in the workspace's `artifacts` directory: publishing it is the builder's, in a later
@@ -145,7 +145,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
 	return BuildResult{Artifact: art.Name, Binary: binary, Digest: digest, Size: len(archive), Path: path}, nil
 }

-// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed toolchain
+// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the recipe's Go release
 // alone, the modules the fetch put in the module's cache and no way to fetch another (GOPROXY=off), so a build never
 // reaches the network. A directory that vendors is built from its vendor directory.
 func (m *Module) buildEnv(dir string) []string {
@@ -156,7 +156,7 @@ func (m *Module) buildEnv(dir string) []string {
 	return []string{
 		"CGO_ENABLED=0",
 		"GOFLAGS=" + recipeFlags + " " + mod,
-		"GOTOOLCHAIN=local",
+		"GOTOOLCHAIN=" + RecipeGo,
 		"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
 		"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
 		"GOPROXY=off",
@@ -120,7 +120,7 @@ func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) {
 		}
 	}
 	env := strings.Join(c.env, "\n")
-	for _, want := range []string{compile[1], "GOFLAGS=" + strings.TrimPrefix(compile[2], "GOFLAGS=") + " -mod=mod", "GOTOOLCHAIN=local", "GOPROXY=off",
+	for _, want := range []string{compile[1], "GOFLAGS=" + strings.TrimPrefix(compile[2], "GOFLAGS=") + " -mod=mod", "GOTOOLCHAIN=go1.27.1", "GOPROXY=off",
 		"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild")} {
 		if !strings.Contains(env, want) {
 			t.Errorf("the environment lacks %q:\n%s", want, env)
@@ -37,7 +37,15 @@ func digestFixture(t *testing.T) (*Module, string) {
 		case "postgres":
 			return forgekey.Ran{Stdout: "postgres (PostgreSQL) " + strconv.Itoa(m.Recipe.PostgresMajor) + ".1\n"}
 		}
-		return real(ctx, env, name, args...)
+		// The unit suite has no network: the release (ADR 0476) is not fetched here, the installed Go runs the fixture.
+		local := []string{}
+		for _, e := range env {
+			if e == "GOTOOLCHAIN="+RecipeGo {
+				e = "GOTOOLCHAIN=local"
+			}
+			local = append(local, e)
+		}
+		return real(ctx, local, name, args...)
 	}
 	return m, tree
 }
@@ -112,13 +112,13 @@ func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
 	return dirs, nil
 }

-// goEnv is what a go command of a step is given beyond the base environment: the installed toolchain and no other,
+// goEnv is what a go command of a step is given beyond the base environment: the recipe's Go release (ADR 0476: the probe put it in the module's cache; a step never fetches it),
 // modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by
 // git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
 // that vendors, see goEnvFor.
 func (m *Module) goEnv() []string {
 	return []string{
-		"GOTOOLCHAIN=local",
+		"GOTOOLCHAIN=" + RecipeGo,
 		"GOFLAGS=-mod=mod",
 		"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
 		"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
@@ -93,7 +93,7 @@ func fetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
 var fetchArgs = map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}

 // 1485/AC1: git is asked through FetchCommit with the ssh remote, and go mod download and verify run in each go.mod
-// directory (not in one that vendors) with GOTOOLCHAIN=local, GOFLAGS=-mod=mod and the module's cache paths.
+// directory (not in one that vendors) with GOTOOLCHAIN=go1.27.1, GOFLAGS=-mod=mod and the module's cache paths.
 func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) {
 	f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}}
 	m, states := fetchModule(t, f)
@@ -128,7 +128,7 @@ func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) {
 			t.Errorf("a module that vendors was downloaded: %s", c.line)
 		}
 		env := strings.Join(c.env, " ")
-		for _, want := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be"} {
+		for _, want := range []string{"GOTOOLCHAIN=go1.27.1", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be"} {
 			if !strings.Contains(env, want) {
 				t.Errorf("%s: environment lacks %s: %s", c.line, want, env)
 			}
@@ -33,7 +33,6 @@ func main() {
 	m := &Module{
 		Node:       strings.TrimSpace(os.Getenv("MESH_NODE")),
 		Recipe:     Current(),
-		Probe:      ProbeGo,
 		Toolchains: stdio.State(delivery.ToolchainsState).Put,
 		Emit:       func(t string, b any) error { return stdio.Emit(t, b) },
 		Steps:      stdio.State(delivery.StepsState).Put,
@@ -46,6 +45,7 @@ func main() {
 		m.Home = u.HomeDir
 	}
 	m.Run = StepRunner(m.Home)
+	m.Probe = ProbeGo(m.Home, m.Cache)
 	// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
 	if m.Node != "" {
 		go m.Start(ctx)
@@ -3,28 +3,61 @@ package main
 import (
 	"context"
 	"fmt"
-	"os"
-	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
 )

-// ProbeTimeout bounds `go version`.
-const ProbeTimeout = 15 * time.Second
+// ProbeTimeout bounds `go version` when the release is in the cache; FetchTimeout bounds it while the release is
+// missing and the installed Go downloads it (about seventy megabytes).
+const (
+	ProbeTimeout = 15 * time.Second
+	FetchTimeout = 5 * time.Minute
+)

-// ProbeGo reads the installed Go version: the third word `go version` prints ("go version go1.27.1 linux/amd64"),
-// as the recipe's RecipeGo is written. GOTOOLCHAIN=local keeps an older Go from fetching another toolchain: what is
-// installed is what is read.
-func ProbeGo(ctx context.Context) (string, error) {
-	ctx, cancel := context.WithTimeout(ctx, ProbeTimeout)
-	defer cancel()
-	cmd := exec.CommandContext(ctx, "go", "version")
-	cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local")
-	out, err := cmd.Output()
-	if err != nil {
-		return "", err
+// ProbeGo is the probe of the guard (novox/hq ADR 0476 §2, §3; issue 1560): it runs `go version` with
+// GOTOOLCHAIN=RecipeGo, so it reads the version of Go's own release of the recipe's version, which is what the
+// module builds with, and the third word of its answer is that version. It runs with network and the module's own
+// Go module cache (cache/gomod): while the release is missing from it, the installed Go downloads it there, verified
+// against Go's checksum database. No step fetches it; a step runs behind this probe and refuses when it is missing.
+// The environment is built from scratch like a step's (StepRunner), so the module's bus credentials are not in it.
+func ProbeGo(home, cache string) func(ctx context.Context) (string, error) {
+	run := forgekey.ExecRunnerClean(0)
+	return func(ctx context.Context) (string, error) { return probeGo(ctx, run, home, cache) }
+}
+
+// probeEnv is what the probe gives `go version` beyond the base environment: the recipe's release as the toolchain,
+// the module's cache for it to land in (and the build cache the go command wants), and Go's own proxy and checksum
+// database, which this module's steps never leave at their default except here and in fetch.
+func probeEnv(cache string) []string {
+	return []string{
+		"GOTOOLCHAIN=" + RecipeGo,
+		"GOMODCACHE=" + filepath.Join(cache, "gomod"),
+		"GOCACHE=" + filepath.Join(cache, "gobuild"),
 	}
-	return parseGoVersion(string(out))
+}
+
+// releaseCached says whether the release is in the module cache: golang.org/toolchain@v0.0.1-<version>.<os>-<arch>.
+func releaseCached(cache string) bool {
+	dirs, _ := filepath.Glob(filepath.Join(cache, "gomod", "golang.org", "toolchain@v0.0.1-"+RecipeGo+".*"))
+	return len(dirs) > 0
+}
+
+func probeGo(ctx context.Context, run forgekey.Runner, home, cache string) (string, error) {
+	bound := ProbeTimeout
+	if !releaseCached(cache) {
+		bound = FetchTimeout
+	}
+	ctx, cancel := context.WithTimeout(ctx, bound)
+	defer cancel()
+	r := run(ctx, append(baseEnv(home), probeEnv(cache)...), "go", "version")
+	if r.Status != 0 || r.Err != "" {
+		return "", fmt.Errorf("`go version` with GOTOOLCHAIN=%s failed (the recipe builds with %s): %s", RecipeGo, RecipeGo,
+			firstLine(r.Err+"\n"+r.Stderr))
+	}
+	return parseGoVersion(r.Stdout)
 }

 func parseGoVersion(out string) (string, error) {
@@ -175,7 +175,7 @@ func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *
 	if line != want {
 		t.Errorf("command = %q, want %q", line, want)
 	}
-	for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod")} {
+	for _, w := range []string{"GOTOOLCHAIN=go1.27.1", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod")} {
 		if !strings.Contains(env, w) {
 			t.Errorf("environment %s lacks %s", env, w)
 		}
@@ -1,8 +1,8 @@
-Each file is the standard output of `GOTOOLCHAIN=local GOFLAGS=-mod=mod go test -count=1 -json ./...` (go1.27.1)
+Each file is the standard output of `GOTOOLCHAIN=go1.27.1 GOFLAGS=-mod=mod go test -count=1 -json ./...` (go1.27.1)
 run in the fixture module of the same name under ../repos, captured on 2026-10-20 for issue 1290. The unit
 tests feed them to the parser through a fake runner, so no test here runs `go test`.

-golist-external.json is `GOTOOLCHAIN=local GOFLAGS=-mod=mod go list -json=ImportPath,Standard,DepOnly,Module
+golist-external.json is `GOTOOLCHAIN=go1.27.1 GOFLAGS=-mod=mod go list -json=ImportPath,Standard,DepOnly,Module
 git.novox.be/novox/mesh-sdk/go/buildstep` run in this module (go1.27.1, 2026-10-20 for issue 1291): a package of a module
 outside the tree as go list describes it.