Run mesh-lab's replays where no pull request reaches

The replays hold the container runtime's socket, yet took their HOME, Go
environment, build cache and module cache from the workspace a pull request's
own script writes, so a planted -toolexec hook ran as root on the build node
(hq issue 709). They now run from a copy of mesh-lab made before anything of
the change runs, with a fresh home and the judge cache's caches, as a judge is
built (ADR 0358 §6).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/builder/check.go

@@ -329,6 +329,13 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	if err := forgetLeftCredentials(workspace); err != nil {
 		return CheckVerdict{}, err
 	}
+	// **mesh-lab's replays run where no pull request reaches** (novox/hq issue 709, ADR 0358 §6): they hold the
+	// container runtime's socket, so their sources, home, Go cache and module cache are none of the workspace's.
+	replay, err := replayDirs(spec.JudgeCache, workspace, root, spec.ID)
+	if err != nil {
+		return CheckVerdict{}, err
+	}
+	defer os.RemoveAll(replay.base)
 	// cloneAt clones a repository and checks out ref — or, when the repository has no branch ref and orElse is said,
 	// orElse (novox/hq ADR 0358 §3); readAt keeps which, by directory.
 	readAt := map[string]string{}
@@ -447,6 +454,12 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 		if err := cloneAt(b.Repository, b.Ref, b.Else, dir); err != nil {
 			return CheckVerdict{}, fmt.Errorf("beside it, %w", err)
 		}
+		if dir == "mesh-lab" && replay.lab != "" {
+			// The copy the replays run from, made before anything of the change runs, outside every check container.
+			if err := copyCheckout(ctx, run, filepath.Join(root, dir), replay.lab); err != nil {
+				return CheckVerdict{}, fmt.Errorf("mesh-lab could not be copied for its replays: %w", err)
+			}
+		}
 		if spec.Release != "" {
 			// The branch each was read at is named with its commit (ADR 0358 §4).
 			if judged.BesideAt == nil {
@@ -617,6 +630,21 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 		return append([]string{args[0], "--volume", judgeHome + ":" + judgeHome, "--volume",
 			judgeGoCache + ":" + judgeGoCache}, args[1:]...)
 	}
+	replayLab := filepath.Join(root, "mesh-lab")
+	if replay.lab != "" {
+		replayLab = replay.lab
+	}
+	inReplays := func(env []string, command ...string) []string {
+		args := in(spec.Toolchain, filepath.Join(replayLab, "replays"), append(ReplayEnv(replay.home, replay.goCache,
+			replay.modCache), env...), command...)
+		// The socket goes to the replays alone, never to the change's own code.
+		mounts := []string{"--volume", "/var/run/docker.sock:/var/run/docker.sock"}
+		if spec.JudgeCache != "" {
+			mounts = append(mounts, "--volume", replay.base+":"+replay.base, "--volume", replay.goCache+":"+replay.goCache,
+				"--volume", replay.modCache+":"+replay.modCache)
+		}
+		return append(append([]string{args[0]}, mounts...), args[1:]...)
+	}
 	var out tail
 	// running runs one container of the check, its output into the report, in a process group of its own.
 	running := func(args []string) error {
@@ -670,7 +698,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 			say("check", "the gate: %d module(s) of the graph touched — %s", len(modules), strings.Join(modules, ", "))
 			fmt.Fprintf(&out, "--- the gate: %s\n", strings.Join(modules, ", "))
 			v.Gate.Verdict, v.Gate.Summary = gateLayer(ctx, spec, tree, root, gate, verdictFile, env, inToolchain,
-				running, &out, bus, workspace, name, say)
+				running, &out, bus, name, replayLab, inReplays, say)
 		}
 		if timedOut() {
 			v.Gate.Verdict, v.Gate.Summary = "error", fmt.Sprintf("the check ran past %s and was ended", CheckTimeout)
@@ -947,8 +975,9 @@ const SomeManifestsEnv = "MESH_MODULE_CHECK_SOME=1"
 // gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the
 // change, and the replays of what the mesh runs. It answers the gate's verdict and summary.
 func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string,
-	inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus, workspace,
+	inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus,
-	name string, say func(step, format string, args ...any)) (string, string) {
+	name, replayLab string, inReplays func([]string, ...string) []string,
+	say func(step, format string, args ...any)) (string, string) {
 	// 1. The manifests the change touches, as the judge reads them: a manifest it cannot read, or one with a
 	// problem the change brings, fails here. **What was already so on the base branch is said and fails
 	// nothing** — the gate's own rule: a module whose manifest the running controller already finds fault
@@ -1040,7 +1069,7 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
 	// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
 	// so given the container runtime the resolver replay raises containers with — against the bus of the
 	// release the mesh runs and the change's own catalogue when the change is to the catalogue.
-	if lab := filepath.Join(root, "mesh-lab", "replays"); ctx.Err() == nil {
+	if lab := filepath.Join(replayLab, "replays"); ctx.Err() == nil {
 		if _, err := os.Stat(lab); err == nil {
 			catalogue := filepath.Join(root, "mesh-catalog")
 			if name == "mesh-catalog" {
@@ -1048,11 +1077,10 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
 			}
 			say("check", "the replays of what the mesh runs, from mesh-lab")
 			fmt.Fprintln(out, "--- the replays (mesh-lab replays/)")
-			args := inToolchain(lab, []string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue,
+			// In the replays' own environment (inReplays): reviewed code, the socket, and nothing the workspace holds
-				"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")},
+			// but the catalogue they read.
+			args := inReplays([]string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue},
 				"go", "test", "-count=1", "./...")
-			// The socket goes to the replays alone, never to the change's own code above.
-			args = append([]string{args[0], "--volume", "/var/run/docker.sock:/var/run/docker.sock"}, args[1:]...)
 			if err := running(args); err != nil {
 				if ctx.Err() != nil {
 					return "error", "the check was ended during the replays"
@@ -1225,6 +1253,76 @@ func judgeBuildDirs(cache, workspace, root, id string) (string, string, error) {
 	return home, goCache, nil
 }
 
+// replayPlace is where mesh-lab's replays run from and with (novox/hq issue 709): base, the directory made for this
+// check and removed after it; lab, the copy of mesh-lab they run (empty: the checkout beside the check); home, their
+// HOME and configuration; and the Go build and module caches.
+type replayPlace struct {
+	base, lab, home, goCache, modCache string
+}
+
+// replayDirs makes the replays' place (issue 709, ADR 0358 §6). With a judge cache: a directory under its homes/ for
+// this check, holding a copy of mesh-lab made before anything of the change runs and a fresh home, and the judge
+// cache's Go build and module caches — none of which a check's container mounts. The replays' build is as trusted as a
+// judge's: it compiles mesh-lab's trunk, reviewed code, and nothing of the pull request. Without a judge cache: a home
+// and caches under the check's root, which every check empties before it clones, and the replays run from the
+// checkout beside the check, as before.
+func replayDirs(cache, workspace, root, id string) (replayPlace, error) {
+	if cache == "" {
+		p := replayPlace{base: filepath.Join(root, "replay-home"), goCache: filepath.Join(root, "judge-go-cache"),
+			modCache: filepath.Join(root, "replay-go-modules")}
+		p.home = p.base
+		for _, d := range []string{p.home, p.goCache, p.modCache} {
+			if err := os.MkdirAll(d, 0o700); err != nil {
+				return replayPlace{}, err
+			}
+		}
+		return p, nil
+	}
+	if withinDir(workspace, cache) {
+		return replayPlace{}, fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers "+
+			"mount: no replay is run where a pull request could reach it", cache, workspace)
+	}
+	if !safeName.MatchString(id) {
+		id = "check"
+	}
+	base := filepath.Join(cache, "homes", id+"-replays")
+	p := replayPlace{base: base, lab: filepath.Join(base, "mesh-lab"), home: filepath.Join(base, "home"),
+		goCache: filepath.Join(cache, "go-cache"), modCache: filepath.Join(cache, "go-modules")}
+	if err := os.RemoveAll(base); err != nil {
+		return replayPlace{}, err
+	}
+	for _, d := range []string{p.home, p.goCache, p.modCache} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			return replayPlace{}, err
+		}
+	}
+	return p, nil
+}
+
+// ReplayEnv is the environment mesh-lab's replays run in (novox/hq issue 709): home as HOME and configuration, the Go
+// build and module caches given, no global or system git configuration, no Go environment file, and no version
+// stamped from git — as a judge is built (JudgeBuildEnv), with the modules the replays fetch.
+func ReplayEnv(home, goCache, modCache string) []string {
+	return []string{"HOME=" + home, "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"),
+		"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", "GOENV=off",
+		"GOCACHE=" + goCache, "GOMODCACHE=" + modCache, "GOPATH=" + filepath.Join(home, "go"),
+		"GOFLAGS=-buildvcs=false"}
+}
+
+// copyCheckout copies a checkout at the commit it stands at into dir, over git's own transport — never by linking
+// objects — so what is written into the source afterwards does not reach the copy.
+func copyCheckout(ctx context.Context, run Runner, from, dir string) error {
+	commit := fullHeadAt(ctx, run, from)
+	if commit == "" {
+		return fmt.Errorf("%s names no commit", from)
+	}
+	if _, err := run(ctx, filepath.Dir(dir), "git", "clone", "--quiet", "--no-local", "--no-checkout", from, dir); err != nil {
+		return err
+	}
+	_, err := run(ctx, dir, "git", "checkout", "--quiet", "--detach", commit)
+	return err
+}
+
 // keptJudges is where a judge cache keeps the release-branch judges, empty when there is no judge cache.
 func keptJudges(cache string) string {
 	if cache == "" {

internal/builder/release_branch_test.go

@@ -147,6 +147,9 @@ run)
   shift
   if [ "$1" = go ]; then echo "GOCACHE=$GOCACHE GOENV=$GOENV $*" >> "` + built + `"; fi
   if [ "$1" = go ] && [ "$2" = build ] && [ -n "$JUDGE_HOMES" ]; then echo "$HOME" >> "$JUDGE_HOMES"; fi
+  if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS" ]; then
+    echo "HOME=$HOME GOCACHE=$GOCACHE GOMODCACHE=$GOMODCACHE GOENV=$GOENV GOFLAGS=$GOFLAGS DIR=$workdir" >> "$REPLAYS"
+  fi
   cd "$workdir" && exec "$@" ;;
 esac
 exit 1
@@ -362,7 +365,7 @@ func TestWithoutTheNodeEnginesReleaseBranchTheJudgeKeepsItsVendoredValidator(t *
 	}
 }
 
-// **Clause 5** (ADR 0358 §5): a release branch's own pull request into its trunk is checked as any pull request into a
+// **Clause 7** (ADR 0358 §7): a release branch's own pull request into its trunk is checked as any pull request into a
 // trunk — by the controller beside it as the mesh runs it, the repositories at what a trunk check reads — and its
 // verdict names no release branch.
 func TestAReleaseBranchsPullRequestIntoItsTrunkIsJudgedByTheControllerTheMeshRuns(t *testing.T) {
@@ -652,3 +655,72 @@ func TestWithoutAJudgeCacheAJudgeIsBuiltColdUnderTheChecksRoot(t *testing.T) {
 		t.Fatalf("a judge cache inside the workspace was taken: %v", err)
 	}
 }
+
+// aLabWithReplays is mesh-lab with replays: a Go module of its own whose one test passes, needing nothing fetched.
+func aLabWithReplays(t *testing.T) (string, string) {
+	t.Helper()
+	lab, labMain, _ := aRepositoryWithARelease(t, map[string]string{
+		"replays/go.mod":         "module example.org/replays\n\ngo 1.22\n",
+		"replays/replay_test.go": "package replays\n\nimport \"testing\"\n\nfunc TestAReplay(t *testing.T) {}\n",
+	}, nil)
+	return lab, labMain
+}
+
+// **mesh-lab's replays run where no pull request reaches** (novox/hq issue 709, ADR 0358 §6): they hold the container
+// runtime's socket, so a Go environment file with a -toolexec hook and a Go cache and module cache planted in the
+// workspace by a pull request's script are neither run nor read — the replays run from a copy of mesh-lab outside
+// the workspace with a fresh home and the judge cache's caches, all removed or kept outside every check container.
+func TestTheReplaysRunWhereNoPullRequestReaches(t *testing.T) {
+	aRuntimeThatBuildsTheJudge(t)
+	replays := filepath.Join(t.TempDir(), "replays")
+	t.Setenv("REPLAYS", replays)
+	registry := aFactsRegistry(t)
+	workspace, ran := aPlantedWorkspace(t)
+	for _, poisoned := range []string{filepath.Join(workspace, "go-cache", "README"),
+		filepath.Join(workspace, "go-modules", "cache", "download", "planted")} {
+		if err := os.MkdirAll(filepath.Dir(poisoned), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(poisoned, []byte("planted by a pull request"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cache := t.TempDir()
+	controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), nil)
+	lab, labMain := aLabWithReplays(t)
+	catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
+		CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"})
+	v, err := Check(t.Context(), Command, CheckSpec{ID: "check-709", Repository: catalogue, Ref: head, Owner: "novox",
+		Repo: "mesh-catalog", Number: 1, Base: "main", JudgeCache: cache, Paths: []string{"modules/gitea/index.ts"},
+		Modules: []string{"gitea"}, Toolchain: "golang", Beside: map[string]Beside{
+			"mesh-controller": {Repository: controller, Ref: controllerMain}, "mesh-lab": {Repository: lab, Ref: labMain}}},
+		workspace, registry, GitCredential{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v.Gate == nil || v.Gate.Verdict != "pass" || !strings.Contains(v.Report, "--- the replays") {
+		t.Fatalf("the replays did not run and pass: %+v\n%s", v.Gate, v.Report)
+	}
+	body, _ := os.ReadFile(replays)
+	said := strings.TrimSpace(string(body))
+	if said == "" || strings.Count(said, "\n") != 0 {
+		t.Fatalf("the replays ran %q, not once", said)
+	}
+	for _, want := range []string{"GOCACHE=" + filepath.Join(cache, "go-cache") + " ",
+		"GOMODCACHE=" + filepath.Join(cache, "go-modules") + " ", "GOENV=off ", "GOFLAGS=-buildvcs=false ",
+		"HOME=" + filepath.Join(cache, "homes", "check-709-replays", "home") + " ",
+		"DIR=" + filepath.Join(cache, "homes", "check-709-replays", "mesh-lab", "replays")} {
+		if !strings.Contains(said, want) {
+			t.Fatalf("the replays did not run with %q: %s", want, said)
+		}
+	}
+	if strings.Contains(said, workspace) {
+		t.Fatalf("the replays ran with something of the workspace: %s", said)
+	}
+	if body, _ := os.ReadFile(ran); strings.Contains(string(body), "replays") || strings.Contains(string(body), "mesh-lab") {
+		t.Fatalf("a hook planted in the workspace ran during the replays:\n%s", body)
+	}
+	if left, _ := os.ReadDir(filepath.Join(cache, "homes")); len(left) != 0 {
+		t.Fatalf("the replays left %d entr(ies) under the judge cache's homes", len(left))
+	}
+}