Land feature 57's controller work on its release branch

The controller plans config, templates and moved providers' provisions, hands bus accounts to the vault and judges the new manifest words, on the feature's SDK and node-engine validator (feature 57, config and provisioning).
2026-10-15 21:27:33 +00:00
parent fcdecb4361
commit 97a2d841c7
349 changed files with 42440 additions and 786 deletions
@@ -13,7 +13,9 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
+	"sync"
 	"time"

 	"github.com/novox/mesh-controller/internal/artifacts"
@@ -87,6 +89,19 @@ type CheckSpec struct {
 	// Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator"
 	// the running one with the change's validator.
 	Judge string
+	// Release is the release branch the pull request merges into (novox/hq ADR 0358), empty for any other base. The
+	// gate is then judged by the controller beside it as mesh-controller — mesh-controller's release branch, never
+	// the controller the mesh runs nor its main — with mesh-host's validator from beside it when mesh-host was read at
+	// the release branch too; a change to the controller still judges itself and a change to the node-engine brings
+	// its own validator. The verdict names the branch, the judge's commit and the branch each beside was read at.
+	Release string
+	// JudgeCache is the judge cache (novox/hq ADR 0358 §6): a directory of the build agent's own, outside the
+	// workspace, that only judge builds mount — never a check's containers, so no pull request's script can write it.
+	// It holds the Go build cache every judge is built with (judges stay warm), the release-branch judges kept by the
+	// commits they are built from (judges/, one per release-branch head), and each judge build's fresh home (homes/,
+	// removed after the build). Empty: each judge is built cold under the check's own root, which every check empties
+	// before it clones, and nothing is kept.
+	JudgeCache string
 	// Toolchain is the image a check's Go runs in: the mesh's own Go toolchain, as it holds it.
 	// Toolchains is every toolchain the mesh holds, by language, for a script that declares another.
 	Toolchain  string
@@ -153,10 +168,12 @@ func ToolchainsOf(held map[string]string) map[string]string {
 	return out
 }

-// Beside is one repository cloned next to the one checked.
+// Beside is one repository cloned next to the one checked: at Ref, or at Else when Else is said and the repository
+// has no branch Ref (novox/hq ADR 0358 §3).
 type Beside struct {
 	Repository string
 	Ref        string
+	Else       string
 }

 // CheckVerdict is what came of one check. Verdict and Summary are the gate's; Gate and Repo each layer.
@@ -179,6 +196,16 @@ type CheckVerdict struct {
 	Members []JudgedHead
 	// Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not.
 	Built []CheckBuilt
+	// Release is the release branch it was judged for (novox/hq ADR 0358 §4), JudgedBy the commit of the controller
+	// that judged its gate (twelve characters), and BesideAt the branch each repository beside it was read at, by
+	// directory. Empty for a check of any other base.
+	Release  string
+	JudgedBy string
+	BesideAt map[string]string
+	// Validator says which node-engine validator the judge of a release-branch check was built with (ADR 0358 §2):
+	// mesh-host's of the release branch with its commit, the one the release branch's controller vendors, the
+	// change's own, or the one the change's own controller vendors. Empty for a check of any other base.
+	Validator string
 }

 // Layer is one layer of a check, judged.
@@ -274,7 +301,17 @@ func ScriptToolchain(script []byte) string {
 // Check runs one merge check. An error is that it could not run; the verdict is then "error".
 func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential,
 	log Log) (CheckVerdict, error) {
-	// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462).
+	// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462). A line
+	// at a time: its builds for the merge say theirs while its tests run (novox/hq issue 716).
+	if log != nil {
+		var saying sync.Mutex
+		inner := log
+		log = func(step, message string) {
+			saying.Lock()
+			defer saying.Unlock()
+			inner(step, message)
+		}
+	}
 	redact := redactorFor(forge)
 	say := logging(func(step, message string) {
 		if log != nil {
@@ -303,13 +340,32 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	if err := forgetLeftCredentials(workspace); err != nil {
 		return CheckVerdict{}, err
 	}
-	clone := func(repository, ref, dir string) error {
+	// **mesh-lab's replays run where no pull request reaches** (novox/hq issue 709, ADR 0358 §6): they hold the
+	// container runtime's socket, so their sources, home, Go cache and module cache are none of the workspace's.
+	replay, err := replayDirs(spec.JudgeCache, workspace, root, spec.ID)
+	if err != nil {
+		return CheckVerdict{}, err
+	}
+	if replay.base != "" {
+		defer os.RemoveAll(replay.base)
+	}
+	// cloneAt clones a repository and checks out ref — or, when the repository has no branch ref and orElse is said,
+	// orElse (novox/hq ADR 0358 §3); readAt keeps which, by directory.
+	readAt := map[string]string{}
+	cloneAt := func(repository, ref, orElse, dir string) error {
 		if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil {
 			return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error()))
 		}
 		if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil {
 			return err
 		}
+		if orElse != "" && ref != "" {
+			if _, err := run(ctx, filepath.Join(root, dir), "git", "rev-parse", "--verify", "--quiet",
+				"refs/remotes/origin/"+ref); err != nil {
+				ref = orElse
+			}
+		}
+		readAt[dir] = ref
 		if ref != "" {
 			if _, err := run(ctx, filepath.Join(root, dir), "git", "checkout", "--quiet", ref); err != nil {
 				return fmt.Errorf("%s has no %s: %w", redact.redact(repository), ref, err)
@@ -317,6 +373,9 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 		}
 		return nil
 	}
+	clone := func(repository, ref, dir string) error {
+		return cloneAt(repository, ref, "", dir)
+	}
 	name := spec.Repo
 	if name == "" || !safeName.MatchString(name) {
 		name = "checked"
@@ -352,8 +411,28 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	}
 	named := func(v CheckVerdict) CheckVerdict {
 		v.Head, v.Trunk, v.Merge, v.Beside, v.Members = judged.Head, judged.Trunk, judged.Merge, judged.Beside, judged.Members
+		v.Release, v.JudgedBy, v.BesideAt, v.Validator = spec.Release, judged.JudgedBy, judged.BesideAt, judged.Validator
 		return v
 	}
+	// **What the merge would move is built from a checkout nothing of the check touches** (novox/hq issue 703): the
+	// gate and the repository's own check run in containers that mount the workspace read-write, so they can change
+	// its tree, its objects and its caches. The builds are made from a copy of the merge's objects, cloned before
+	// anything of the change runs, outside the workspace — so what is kept under a key is what the key describes. Each
+	// build's working copy is cloned from it into the workspace only after the check has ended (forTheMerge): the
+	// host's container engine sees no other directory of the holder's (novox/hq issue 713).
+	buildingTree := ""
+	if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
+		at := judged.Merge
+		if at == "" {
+			at = spec.Ref
+		}
+		var building string
+		building, buildingTree, err = checkoutForTheBuilds(ctx, run, tree, at, workspace)
+		if err != nil {
+			return CheckVerdict{}, err
+		}
+		defer os.RemoveAll(building)
+	}
 	// **The base branch's check judges the change** (novox/hq issue 310), never the one the change brings.
 	judging, err := TheCheckThatJudges(ctx, tree, spec.Base)
 	if err != nil {
@@ -376,9 +455,35 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 		if dir == name || !safeName.MatchString(dir) {
 			continue
 		}
-		if err := clone(b.Repository, b.Ref, dir); err != nil {
+		if dir == "mesh-lab" {
+			// **mesh-lab is never read at a release branch** (novox/hq ADR 0358): its replays run with the container
+			// runtime's socket, so a branch of a release's name — which anyone may create — is never what runs there,
+			// whether it is asked with a fallback or without.
+			if b.Else != "" || (spec.Release != "" && b.Ref == spec.Release) {
+				b.Ref = b.Else
+				if b.Ref == "" {
+					b.Ref = "main"
+				}
+			}
+			b.Else = ""
+		}
+		if err := cloneAt(b.Repository, b.Ref, b.Else, dir); err != nil {
 			return CheckVerdict{}, fmt.Errorf("beside it, %w", err)
 		}
+		if dir == "mesh-lab" {
+			// The copy the replays run from, made before anything of the change runs, outside every check container
+			// (novox/hq issue 709) — and whether it holds replays, read now, so nothing written later decides it.
+			if err := replay.copyLab(ctx, run, filepath.Join(root, dir)); err != nil {
+				return CheckVerdict{}, fmt.Errorf("mesh-lab could not be copied for its replays: %w", err)
+			}
+		}
+		if spec.Release != "" {
+			// The branch each was read at is named with its commit (ADR 0358 §4).
+			if judged.BesideAt == nil {
+				judged.BesideAt = map[string]string{}
+			}
+			judged.BesideAt[dir] = readAt[dir]
+		}
 		// The commit each one read is named in the verdict (ADR 0323 §3): its move is issue 568's question.
 		if at := headAt(ctx, run, filepath.Join(root, dir)); at != "" {
 			if judged.Beside == nil {
@@ -386,7 +491,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 			}
 			judged.Beside[dir] = at
 		}
-		say("check", "beside it %s at %s", dir, short(b.Ref))
+		say("check", "beside it %s at %s", dir, refWords(readAt[dir]))
 	}

 	// A delivery group's other heads (novox/hq ADR 0239), each at its head, for the gate to compose with this.
@@ -438,6 +543,25 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	// Every clone is made: the credential is gone before anything of the check runs (novox/hq issue 462).
 	forget()

+	// **What the merge moves is built beside the tests** (novox/hq issue 716) when the build agent has a judge cache to
+	// build it in, where no container of the check reaches; without one, after the check has judged.
+	logged := func(step, message string) {
+		if log != nil {
+			log(step, redact.redact(message))
+		}
+	}
+	var beside *besideTheTests
+	if buildingTree != "" {
+		place, err := buildsBesideTheTests(spec.JudgeCache, workspace, spec.ID)
+		if err != nil {
+			return CheckVerdict{}, err
+		}
+		if place != "" {
+			beside = startBesideTheTests(ctx, run, spec, place, buildingTree, judged, forge, logged, say)
+			defer beside.abandon()
+		}
+	}
+
 	// The facts, and the versions they say the mesh runs.
 	if registry == "" {
 		return CheckVerdict{}, errors.New("no artifact store to read the facts snapshot from")
@@ -500,21 +624,56 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 		user = fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid())
 	}
 	in := func(image, dir string, env []string, command ...string) []string {
+		home := workspace
+		for _, e := range env {
+			if strings.HasPrefix(e, "HOME=") {
+				home = strings.TrimPrefix(e, "HOME=")
+			}
+		}
 		// As the builder itself: what a check writes into the workspace is the builder's to remove.
 		args := []string{"run", "--rm", "--network", "host", "--volume", workspace + ":" + workspace, "--workdir", dir,
-			"--user", user, "--env", "HOME=" + workspace,
+			"--user", user, "--env", "HOME=" + home,
 			// The check's own checkouts, whoever cloned them: git in a container of another user than the
 			// one that cloned refuses a repository it does not own ("dubious ownership"), and Go's build
 			// stamps the version from git — a judge that would not build for want of it.
 			"--env", "GIT_CONFIG_COUNT=1", "--env", "GIT_CONFIG_KEY_0=safe.directory", "--env", "GIT_CONFIG_VALUE_0=*"}
 		for _, e := range env {
-			args = append(args, "--env", e)
+			if !strings.HasPrefix(e, "HOME=") {
+				args = append(args, "--env", e)
+			}
 		}
 		return append(append(args, image), command...)
 	}
 	inToolchain := func(dir string, env []string, command ...string) []string {
 		return in(spec.Toolchain, dir, env, command...)
 	}
+	// **Every judge is built where no pull request can reach** (novox/hq issue 482, ADR 0358 review): the workspace's
+	// Go cache, HOME and git configuration are written by pull requests' own scripts, and git runs during a Go build
+	// to stamp its version — a planted hook would rewrite the judge's sources before they compile. So a judge is built
+	// with a HOME and configuration made fresh for this check and a Go cache only judge builds mount (judgeBuildDirs),
+	// no global or system git configuration, no Go environment file, and no version stamping from git.
+	judgeHome, judgeGoCache, err := judgeBuildDirs(spec.JudgeCache, workspace, root, spec.ID)
+	if err != nil {
+		return CheckVerdict{}, err
+	}
+	defer os.RemoveAll(judgeHome)
+	inJudgeBuild := func(dir string, _ []string, command ...string) []string {
+		args := in(spec.Toolchain, dir, JudgeBuildEnv(judgeHome, judgeGoCache), command...)
+		if spec.JudgeCache == "" {
+			return args // under the check's root, in the workspace mounted already
+		}
+		// The judge cache's parts this build needs, mounted into judge builds alone (ADR 0358 §6).
+		return append([]string{args[0], "--volume", judgeHome + ":" + judgeHome, "--volume",
+			judgeGoCache + ":" + judgeGoCache}, args[1:]...)
+	}
+	inReplays := func(env []string, command ...string) []string {
+		args := in(spec.Toolchain, filepath.Join(replay.lab, "replays"), append(ReplayEnv(replay.home, replay.goCache,
+			replay.modCache), env...), command...)
+		// The socket goes to the replays alone, never to the change's own code.
+		mounts := []string{"--volume", "/var/run/docker.sock:/var/run/docker.sock", "--volume", replay.base + ":" +
+			replay.base, "--volume", replay.goCache + ":" + replay.goCache, "--volume", replay.modCache + ":" + replay.modCache}
+		return append(append([]string{args[0]}, mounts...), args[1:]...)
+	}
 	var out tail
 	// running runs one container of the check, its output into the report, in a process group of its own.
 	running := func(args []string) error {
@@ -526,10 +685,23 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry

 	// The judge. Its failing to build is the change's fault when the change is the judge or its validator,
 	// and the check's when it is the controller the mesh runs.
-	gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, inToolchain, say)
+	gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, judged.BesideAt, inJudgeBuild, say)
 	if err != nil {
 		return CheckVerdict{}, err
 	}
+	if spec.Release != "" {
+		// **A release-branch judgement says so** (novox/hq ADR 0358 §4): the branch, the controller's commit that
+		// judged, and each repository read beside it at its branch and commit.
+		judgedBy := filepath.Join(root, "mesh-controller")
+		if spec.Judge == "self" {
+			judgedBy = tree
+		}
+		judged.JudgedBy = headAt(ctx, run, judgedBy)
+		judged.Validator = releaseValidator(spec, judged.BesideAt, judged.Beside)
+		said := releaseWords(spec.Release, judged.JudgedBy, judged.Validator, judged.BesideAt, judged.Beside)
+		fmt.Fprintf(&out, "--- %s\n", said)
+		say("check", "%s", said)
+	}
 	verdictFile := filepath.Join(root, "verdict.json")
 	env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL,
 		EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo,
@@ -555,7 +727,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 			say("check", "the gate: %d module(s) of the graph touched — %s", len(modules), strings.Join(modules, ", "))
 			fmt.Fprintf(&out, "--- the gate: %s\n", strings.Join(modules, ", "))
 			v.Gate.Verdict, v.Gate.Summary = gateLayer(ctx, spec, tree, root, gate, verdictFile, env, inToolchain,
-				running, &out, bus, workspace, name, say)
+				running, &out, bus, name, replay.hasReplays, replay.refused(), inReplays, say)
 		}
 		if timedOut() {
 			v.Gate.Verdict, v.Gate.Summary = "error", fmt.Sprintf("the check ran past %s and was ended", CheckTimeout)
@@ -595,6 +767,11 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 		}
 	}

+	if spec.Release != "" {
+		// The statuses name the release branch and the judge first, so the forge's short description keeps them.
+		mark := releaseMark(spec.Release, judged.JudgedBy)
+		v.Gate.Summary, v.Repo.Summary = mark+v.Gate.Summary, mark+v.Repo.Summary
+	}
 	// What the check printed travels in the verdict to the forge and the controller: redacted as the log is.
 	v.Gate.Summary, v.Repo.Summary, v.Repo.Failed = redact.redact(v.Gate.Summary), redact.redact(v.Repo.Summary),
 		redact.redact(v.Repo.Failed)
@@ -605,24 +782,68 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry

 	// **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it
 	// judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again.
-	if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
-		if why := skipsTheBuilds(v); why != "" {
-			say("check", "%s", why)
-		} else {
-			began := time.Now()
-			v.Built = buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, tree, judged.Merge, judged, forge,
-				func(step, message string) {
-					if log != nil {
-						log(step, redact.redact(message))
-					}
-				}, say)
-			say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(v.Built), len(v.Built),
-				time.Since(began).Round(time.Second))
-		}
+	switch {
+	case beside != nil:
+		v.Built = beside.settle(ctx, v)
+	case len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0:
+		v.Built = forTheMerge(ctx, run, spec, v, workspace, buildingTree, judged, forge, logged, say)
 	}
 	return named(v), nil
 }

+// forTheMerge builds what the merge moves once the check has judged it, from merged — the copy of the merge
+// checkoutForTheBuilds made outside the workspace — **in the builder's own workspace** (novox/hq issue 713). A build's
+// containers are run by the host's container engine, which mounts a path as the host names it: the build seat's
+// holder runs in a container that shares only its workspace with the host, at the same path, so a working copy in
+// its own temporary directory reaches the toolchain as an empty directory ("go.mod file not found"), every check
+// build failed, and every walk built again. The copy stays outside the workspace, so nothing the tests ran reaches it;
+// the working copy cloned from it is made only now, after every container of the check has ended.
+func forTheMerge(ctx context.Context, run Runner, spec CheckSpec, verdict CheckVerdict, workspace, merged string,
+	judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
+	if why := skipsTheBuilds(verdict); why != "" {
+		say("check", "%s", why)
+		return nil
+	}
+	began := time.Now()
+	built := buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, merged, judged.Merge, judged, forge, log, say)
+	say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(built), len(built),
+		time.Since(began).Round(time.Second))
+	return built
+}
+
+// checkoutForTheBuilds is a clone of the commit a check builds for its merge, made before anything of the check runs,
+// outside the workspace its containers mount (novox/hq issue 703): the directory to remove, and the clone in it.
+// Cloned over git's own transport, never by linking the objects: what the check's containers later write into the
+// workspace's objects cannot reach it.
+func checkoutForTheBuilds(ctx context.Context, run Runner, tree, at, workspace string) (string, string, error) {
+	dir, err := os.MkdirTemp("", "mesh-check-builds-")
+	if err != nil {
+		return "", "", err
+	}
+	if withinDir(workspace, dir) {
+		os.RemoveAll(dir)
+		return "", "", fmt.Errorf("the temporary directory %s is inside the workspace %s, which a check's containers "+
+			"mount: the merge is not built from where its tests could change it", dir, workspace)
+	}
+	commit, err := run(ctx, tree, "git", "rev-parse", "--verify", at+"^{commit}")
+	if err != nil {
+		os.RemoveAll(dir)
+		return "", "", fmt.Errorf("the commit to build for the merge, %s: %w", at, err)
+	}
+	commit = strings.TrimSpace(commit)
+	if _, err := run(ctx, tree, "git", "update-ref", "refs/heads/mesh-check-builds", commit); err != nil {
+		os.RemoveAll(dir)
+		return "", "", err
+	}
+	cloned := filepath.Join(dir, "merged")
+	if _, err := run(ctx, dir, "git", "clone", "--quiet", "--no-local", "--no-checkout", "--branch",
+		"mesh-check-builds", tree, cloned); err != nil {
+		os.RemoveAll(dir)
+		return "", "", fmt.Errorf("cannot copy the merge to build it: %w", err)
+	}
+	return dir, cloned, nil
+}
+
 // builtCount is how many of a check's builds for its merge were built.
 func builtCount(built []CheckBuilt) int {
 	n := 0
@@ -794,8 +1015,9 @@ const SomeManifestsEnv = "MESH_MODULE_CHECK_SOME=1"
 // gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the
 // change, and the replays of what the mesh runs. It answers the gate's verdict and summary.
 func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string,
-	inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus, workspace,
-	name string, say func(step, format string, args ...any)) (string, string) {
+	inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus,
+	name string, hasReplays bool, replaysRefused string, inReplays func([]string, ...string) []string,
+	say func(step, format string, args ...any)) (string, string) {
 	// 1. The manifests the change touches, as the judge reads them: a manifest it cannot read, or one with a
 	// problem the change brings, fails here. **What was already so on the base branch is said and fails
 	// nothing** — the gate's own rule: a module whose manifest the running controller already finds fault
@@ -887,24 +1109,37 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
 	// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
 	// so given the container runtime the resolver replay raises containers with — against the bus of the
 	// release the mesh runs and the change's own catalogue when the change is to the catalogue.
-	if lab := filepath.Join(root, "mesh-lab", "replays"); ctx.Err() == nil {
-		if _, err := os.Stat(lab); err == nil {
+	if ctx.Err() == nil && hasReplays && replaysRefused != "" {
+		// **Never from the workspace** (novox/hq issue 709): without the judge cache there is no place outside it.
+		say("check", "%s", replaysRefused)
+		fmt.Fprintf(out, "--- the replays: %s\n", replaysRefused)
+		return "error", replaysRefused
+	}
+	if ctx.Err() == nil {
+		if hasReplays {
 			catalogue := filepath.Join(root, "mesh-catalog")
 			if name == "mesh-catalog" {
 				catalogue = tree
 			}
 			say("check", "the replays of what the mesh runs, from mesh-lab")
 			fmt.Fprintln(out, "--- the replays (mesh-lab replays/)")
-			args := inToolchain(lab, []string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue,
-				"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")},
+			// In the replays' own environment (inReplays): reviewed code, the socket, and nothing the workspace holds
+			// but the catalogue they read.
+			args := inReplays([]string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue},
 				"go", "test", "-count=1", "./...")
-			// The socket goes to the replays alone, never to the change's own code above.
-			args = append([]string{args[0], "--volume", "/var/run/docker.sock:/var/run/docker.sock"}, args[1:]...)
 			if err := running(args); err != nil {
 				if ctx.Err() != nil {
 					return "error", "the check was ended during the replays"
 				}
-				return "fail", "a replay of a core incident fails with this change: " + lastLine(out.String())
+				if replaysNotStarted(err) {
+					// The container runtime could not start them (a mount, an image): the check's fault, never the change's.
+					said := "the replays could not be started, so the change is not judged by them: " + lastLine(out.String())
+					say("check", "%s", said)
+					return "error", said
+				}
+				said := "a replay of a core incident fails with this change: " + lastLine(out.String())
+				say("check", "%s", said)
+				return "fail", said
 			}
 		}
 	}
@@ -919,21 +1154,32 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
 // the mesh runs. It answers the judge's path; or, when the change makes its own judge unbuildable, why —
 // the change's fault, a failing gate; or an error when the check cannot build a judge at all. A check
 // whose gate does not run builds a judge only to hand its scripts one, and goes on without when it cannot.
-func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string,
+//
+// **A pull request into a release branch is judged by that branch's controller** (novox/hq ADR 0358 §2): the one
+// beside it as mesh-controller, read at the release branch — with mesh-host's validator from beside it when mesh-host
+// has that branch too, or with the change's own validator for a change to the node-engine — never the controller the
+// mesh runs. besideAt is the branch each repository beside it was read at.
+func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string, besideAt map[string]string,
 	inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) {
 	gated := spec.Gated()
 	switch spec.Judge {
 	case "self":
 		bin := filepath.Join(root, "bin", "judge-of-itself")
-		if _, err := labelled(ctx, root, "docker", inToolchain(tree,
-			[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
+		if _, err := labelled(ctx, root, "docker", inToolchain(tree, nil,
 			"go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil {
 			return "", "the change's controller, which judges itself, does not build: " + firstLine(err.Error()), nil
 		}
 		say("check", "judged by the change's own controller")
 		return bin, "", nil
 	}
-	bin, judgeTree, err := judge(ctx, labelled, run, root, inToolchain, say)
+	var bin, judgeTree string
+	var err error
+	if spec.Release != "" {
+		withHost := spec.Judge != "validator" && besideAt["mesh-host"] == spec.Release
+		bin, judgeTree, err = releaseJudge(ctx, labelled, run, spec, root, withHost, inToolchain, say)
+	} else {
+		bin, judgeTree, err = judge(ctx, labelled, run, root, inToolchain, say)
+	}
 	if err != nil {
 		if !gated {
 			say("check", "no judge for its script: %v", err)
@@ -952,15 +1198,346 @@ func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, t
 		}
 	}
 	withValidator := filepath.Join(root, "bin", "judge-with-this-validator")
-	if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree),
-		[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
+	if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree), nil,
 		"go", "build", "-o", withValidator, "./cmd/mesh-controller")...); err != nil {
+		if spec.Release != "" {
+			return "", spec.Release + "'s controller does not build with this change's validator: " +
+				firstLine(err.Error()), nil
+		}
 		return "", "the controller the mesh runs does not build with this change's validator: " + firstLine(err.Error()), nil
 	}
+	if spec.Release != "" {
+		say("check", "judged by %s's controller, with this change's validator", spec.Release)
+		return withValidator, "", nil
+	}
 	say("check", "judged by the controller the mesh runs, with this change's validator")
 	return withValidator, "", nil
 }

+// releaseJudge builds the controller that judges a pull request into a release branch (novox/hq ADR 0358 §2): the
+// one beside the check as mesh-controller, read at that branch — with mesh-host's validator from beside it in place
+// of the one it vendors when withHost. **Built once per release-branch head**: kept in the spec's judge cache by the
+// commits it is built from and taken from there while they stand. It answers the binary and the directory it was
+// built from.
+func releaseJudge(ctx context.Context, run, plain Runner, spec CheckSpec, root string, withHost bool,
+	inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) {
+	const dir = "mesh-controller"
+	if _, err := os.Stat(filepath.Join(root, dir)); err != nil {
+		return "", "", fmt.Errorf("no controller of %s beside the check to judge it with", spec.Release)
+	}
+	key := fullHeadAt(ctx, plain, filepath.Join(root, dir))
+	if key == "" {
+		return "", "", fmt.Errorf("the controller of %s beside the check names no commit", spec.Release)
+	}
+	with := ""
+	if withHost {
+		host := fullHeadAt(ctx, plain, filepath.Join(root, "mesh-host"))
+		if host == "" {
+			return "", "", fmt.Errorf("mesh-host of %s beside the check names no commit", spec.Release)
+		}
+		key += "-with-mesh-host-" + host
+		with = ", with mesh-host's validator of " + spec.Release + " at " + twelve(host)
+	}
+	bin := filepath.Join(root, "bin", "mesh-controller")
+	if took := judgeKept(keptJudges(spec.JudgeCache), key, bin); took {
+		say("check", "judged by %s's controller at %s%s, built for an earlier check of that head", spec.Release,
+			twelve(key), with)
+		return bin, dir, nil
+	}
+	if withHost {
+		vendored := filepath.Join(root, dir, "vendor", "github.com", "novox", "mesh-host")
+		for _, pkg := range []string{"validate", filepath.Join("internal", "declaration")} {
+			if err := replaceGoFiles(filepath.Join(root, "mesh-host", pkg), filepath.Join(vendored, pkg)); err != nil {
+				return "", "", fmt.Errorf("mesh-host's validator of %s could not be put in its controller: %w", spec.Release, err)
+			}
+		}
+	}
+	// Built where no pull request reaches (JudgeBuildEnv): a judge kept for this head is the head's, nothing else.
+	if _, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), nil,
+		"go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil {
+		return "", "", fmt.Errorf("the controller of %s does not build%s: %w", spec.Release, with, err)
+	}
+	// Kept before anything of the change runs: nothing a pull request's script writes can reach the cache.
+	keepJudge(keptJudges(spec.JudgeCache), key, bin, say)
+	say("check", "judged by %s's controller at %s%s", spec.Release, twelve(key), with)
+	return bin, dir, nil
+}
+
+// JudgeBuildEnv is the environment every judge is built in (novox/hq ADR 0358 §6, issue 482): home, made fresh for the
+// build, as HOME and configuration; goCache, the judge cache's Go cache that no check's container mounts; no global
+// or system git configuration; no Go environment file; and no version stamped from git, so no git runs at all.
+func JudgeBuildEnv(home, goCache string) []string {
+	return []string{"HOME=" + home, "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"),
+		"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", "GOENV=off",
+		"GOCACHE=" + goCache, "GOPATH=" + filepath.Join(home, "go"),
+		"CGO_ENABLED=0", "GOFLAGS=-mod=vendor -buildvcs=false", "GOPROXY=off"}
+}
+
+// judgeBuildDirs are the home and the Go cache a check's judge builds use (ADR 0358 §6). With a judge cache: a home
+// made fresh under its homes/ for this check, and its go-cache/, kept warm between checks — both on the host path the
+// build agent declares, so the container runtime mounts what the agent sees, and nothing is left in a temporary
+// directory of the host. Without one: both under the check's root, which every check empties before it clones. A
+// judge cache inside the workspace is refused: a check's containers mount the workspace.
+func judgeBuildDirs(cache, workspace, root, id string) (string, string, error) {
+	if cache == "" {
+		home, goCache := filepath.Join(root, "judge-home"), filepath.Join(root, "judge-go-cache")
+		for _, d := range []string{home, goCache} {
+			if err := os.MkdirAll(d, 0o700); err != nil {
+				return "", "", err
+			}
+		}
+		return home, goCache, nil
+	}
+	if withinDir(workspace, cache) {
+		return "", "", fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers mount: "+
+			"no judge is built where a pull request could reach it", cache, workspace)
+	}
+	if !safeName.MatchString(id) {
+		return "", "", fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
+	}
+	home, goCache := filepath.Join(cache, "homes", id), filepath.Join(cache, "go-cache")
+	if err := os.RemoveAll(home); err != nil {
+		return "", "", err
+	}
+	for _, d := range []string{home, goCache} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			return "", "", err
+		}
+	}
+	return home, goCache, nil
+}
+
+// replayPlace is where mesh-lab's replays run from and with (novox/hq issue 709), never anything a pull request can
+// write: base, the directory made for this check under the judge cache's homes/ (the same path on the host and in the
+// build agent), removed after it; lab, the copy of mesh-lab they run; home, their HOME and configuration; and the judge
+// cache's Go build and module caches. Empty without a judge cache: the replays are then refused. hasReplays says whether
+// mesh-lab held replays when it was cloned, before anything of the change ran.
+type replayPlace struct {
+	base, lab, home, goCache, modCache string
+	hasReplays                         bool
+}
+
+// replaysNeedTheJudgeCache is the gate's error when mesh-lab holds replays and the build agent has no judge cache.
+const replaysNeedTheJudgeCache = "the replays need the build agent's judge cache (mesh-catalog#297); refused rather " +
+	"than run from the workspace"
+
+// refused is why the replays cannot run here, empty when they can.
+func (p replayPlace) refused() string {
+	if p.base == "" {
+		return replaysNeedTheJudgeCache
+	}
+	return ""
+}
+
+// replayDirs makes the replays' place (issue 709, ADR 0358 §6). The replays' build is as trusted as a judge's: it
+// compiles mesh-lab's trunk, reviewed code, and nothing of the pull request, which hands them only its catalogue, as
+// data. Never the workspace: the gate runs before the replays, a pull request's own code among its containers. Without
+// a judge cache there is no place: the build agent declares no other host path, and the replays are refused.
+func replayDirs(cache, workspace, root, id string) (replayPlace, error) {
+	if cache == "" {
+		return replayPlace{}, nil
+	}
+	if withinDir(workspace, cache) {
+		return replayPlace{}, fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers "+
+			"mount: no replay is run where a pull request could reach it", cache, workspace)
+	}
+	if !safeName.MatchString(id) {
+		return replayPlace{}, fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
+	}
+	base := filepath.Join(cache, "homes", id+"-replays")
+	p := replayPlace{base: base, lab: filepath.Join(base, "mesh-lab"), home: filepath.Join(base, "home"),
+		goCache: filepath.Join(cache, "go-cache"), modCache: filepath.Join(cache, "go-modules")}
+	if err := os.RemoveAll(base); err != nil {
+		return replayPlace{}, err
+	}
+	for _, d := range []string{p.home, p.goCache, p.modCache} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			return replayPlace{}, err
+		}
+	}
+	return p, nil
+}
+
+// copyLab reads whether mesh-lab's checkout, freshly cloned and before anything of the change runs, holds replays, and
+// copies it over git's own transport to where the replays run it — when there is such a place.
+func (p *replayPlace) copyLab(ctx context.Context, run Runner, from string) error {
+	_, err := os.Stat(filepath.Join(from, "replays"))
+	p.hasReplays = err == nil
+	if p.lab == "" {
+		return nil
+	}
+	return copyCheckout(ctx, run, from, p.lab)
+}
+
+// replaysNotStarted says the container runtime could not start the replays' container — its own exit status 125, or
+// 126 and 127 for a command it could not run — rather than that a replay failed.
+func replaysNotStarted(err error) bool {
+	var exit *exec.ExitError
+	if !errors.As(err, &exit) {
+		return true
+	}
+	code := exit.ExitCode()
+	return code == 125 || code == 126 || code == 127
+}
+
+// ReplayEnv is the environment mesh-lab's replays run in (novox/hq issue 709): home as HOME and configuration, the Go
+// build and module caches given, no global or system git configuration, no Go environment file, and no version
+// stamped from git — as a judge is built (JudgeBuildEnv), with the modules the replays fetch.
+func ReplayEnv(home, goCache, modCache string) []string {
+	return []string{"HOME=" + home, "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"),
+		"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", "GOENV=off",
+		"GOCACHE=" + goCache, "GOMODCACHE=" + modCache, "GOPATH=" + filepath.Join(home, "go"),
+		"GOFLAGS=-buildvcs=false", "GOWORK=off"}
+}
+
+// copyCheckout copies a checkout at the commit it stands at into dir, over git's own transport — never by linking
+// objects — so what is written into the source afterwards does not reach the copy.
+func copyCheckout(ctx context.Context, run Runner, from, dir string) error {
+	commit := fullHeadAt(ctx, run, from)
+	if commit == "" {
+		return fmt.Errorf("%s names no commit", from)
+	}
+	if _, err := run(ctx, filepath.Dir(dir), "git", "clone", "--quiet", "--no-local", "--no-checkout", from, dir); err != nil {
+		return err
+	}
+	_, err := run(ctx, dir, "git", "checkout", "--quiet", "--detach", commit)
+	return err
+}
+
+// keptJudges is where a judge cache keeps the release-branch judges, empty when there is no judge cache.
+func keptJudges(cache string) string {
+	if cache == "" {
+		return ""
+	}
+	return filepath.Join(cache, "judges")
+}
+
+// judgesKept is how many release-branch judges a judge cache keeps: a head moved past is not built from again.
+const judgesKept = 4
+
+// judgeKept puts the judge kept under key in the cache at bin, and says whether there was one.
+func judgeKept(cache, key, bin string) bool {
+	if cache == "" {
+		return false
+	}
+	body, err := os.ReadFile(filepath.Join(cache, key, "mesh-controller"))
+	if err != nil {
+		return false
+	}
+	if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil {
+		return false
+	}
+	if err := os.WriteFile(bin, body, 0o755); err != nil {
+		return false
+	}
+	now := time.Now()
+	_ = os.Chtimes(filepath.Join(cache, key), now, now)
+	return true
+}
+
+// keepJudge keeps a judge just built under key, and lets go of all but the judgesKept most recently used. A judge
+// that cannot be kept is said and built again next time: the cache only saves time.
+func keepJudge(cache, key, bin string, say func(step, format string, args ...any)) {
+	if cache == "" {
+		return
+	}
+	body, err := os.ReadFile(bin)
+	if err == nil {
+		err = os.MkdirAll(filepath.Join(cache, key), 0o755)
+	}
+	if err == nil {
+		kept := filepath.Join(cache, key, "mesh-controller")
+		if err = os.WriteFile(kept+".part", body, 0o755); err == nil {
+			err = os.Rename(kept+".part", kept)
+		}
+	}
+	if err != nil {
+		say("check", "the judge could not be kept for the next check, which builds it again: %v", err)
+		return
+	}
+	entries, err := os.ReadDir(cache)
+	if err != nil || len(entries) <= judgesKept {
+		return
+	}
+	type used struct {
+		name string
+		at   time.Time
+	}
+	var all []used
+	for _, e := range entries {
+		if info, err := e.Info(); err == nil && e.IsDir() {
+			all = append(all, used{e.Name(), info.ModTime()})
+		}
+	}
+	sort.Slice(all, func(i, j int) bool { return all[i].at.After(all[j].at) })
+	for _, u := range all[min(judgesKept, len(all)):] {
+		if u.name != key {
+			_ = os.RemoveAll(filepath.Join(cache, u.name))
+		}
+	}
+}
+
+// fullHeadAt is the whole commit a checkout stands at, empty when it names none.
+func fullHeadAt(ctx context.Context, run Runner, tree string) string {
+	out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", "HEAD^{commit}")
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(out)
+}
+
+// releaseValidator says which node-engine validator a release-branch check's judge is built with (novox/hq ADR 0358
+// §2), by the rule judgeFor builds it by.
+func releaseValidator(spec CheckSpec, at, commits map[string]string) string {
+	switch {
+	case spec.Judge == "self":
+		return "the validator the change's own controller vendors"
+	case spec.Judge == "validator":
+		return "the change's own validator"
+	case at["mesh-host"] == spec.Release:
+		return fmt.Sprintf("mesh-host's validator of %s at %s", spec.Release, commits["mesh-host"])
+	}
+	return fmt.Sprintf("the validator %s's controller vendors (mesh-host has no branch %s)", spec.Release, spec.Release)
+}
+
+// releaseWords is how a release-branch check says what judged it and what it read (novox/hq ADR 0358 §4).
+func releaseWords(release, judgedBy, validator string, at, commits map[string]string) string {
+	var read []string
+	for _, dir := range sortedKeys(at) {
+		read = append(read, fmt.Sprintf("%s %s at %s", dir, refWords(at[dir]), commits[dir]))
+	}
+	said := fmt.Sprintf("judged for the release branch %s by its controller at %s, with %s", release, judgedBy, validator)
+	if len(read) > 0 {
+		said += "; read beside it: " + strings.Join(read, ", ")
+	}
+	return said
+}
+
+// releaseMark opens each layer's summary of a release-branch check, so a status says it first.
+func releaseMark(release, judgedBy string) string {
+	return fmt.Sprintf("[release %s, judged by its controller %s] ", release, judgedBy)
+}
+
+// refWords is a ref as a line says it: a commit by its first eight characters, a branch whole.
+func refWords(ref string) string {
+	if commitLike.MatchString(ref) {
+		return short(ref)
+	}
+	return ref
+}
+
+// commitLike is a ref that names a commit.
+var commitLike = regexp.MustCompile(`^[0-9a-f]{7,40}$`)
+
+func sortedKeys(m map[string]string) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
 // replaceGoFiles puts a package's Go files — never its tests — in place of another copy's.
 func replaceGoFiles(from, into string) error {
 	old, err := filepath.Glob(filepath.Join(into, "*.go"))
@@ -1195,8 +1772,7 @@ func judge(ctx context.Context, run, plain Runner, root string, inToolchain func
 	say func(step, format string, args ...any)) (string, string, error) {
 	bin := filepath.Join(root, "bin", "mesh-controller")
 	build := func(dir string) error {
-		_, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir),
-			[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
+		_, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), nil,
 			"go", "build", "-o", bin, "./cmd/mesh-controller")...)
 		return err
 	}