mesh-controller/internal/builder/check.go

package builder

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/artifacts"
	"github.com/novox/mesh-controller/internal/facts"
)

// A pull request's merge check, run on the build seat (novox/hq to-be 45 §9, ADR 0237).
//
// **The build machine already has what a check needs**: the repositories, a container runtime, the
// artifact store where the controller keeps the facts snapshot, and a Go toolchain. So a check is one
// more kind of work on the build seat's queue rather than a CI the mesh would have to run beside itself.
//
// **Two layers, each its own status on the pull request** (ADR 0237 as amended, 2026-10-06):
//
//   - **the gate** (`mesh/merge-gate`) runs when the change touches a module of the mesh's graph — the
//     controller, which holds the graph, says which (Modules) and which directories it adds a module in
//     (New). The touched manifests through `module check`; every machine of the facts snapshot composed
//     with the change and validated by the node-engine's own validator; then mesh-lab's replays. The
//     judge is the controller the mesh runs — or, for a change to the controller, the change's own
//     controller, and for a change to the node-engine, the running controller with the change's validator
//     in place of the one it vendors. The graph decides whether this runs, never the repository.
//   - **the repository's own check** (`mesh/repo-check`): its merge-check.sh, its unit tests and code
//     quality, run when present in the toolchain it declares (`# mesh-check-toolchain: go|typescript`
//     among its first lines; go when it declares none). A repository that reaches the build seat with
//     none is said as a warning: it is the mesh's, and nothing of its own is tested before it merges.
//     **A repository in two languages declares the other part too** (`# mesh-check-also: <toolchain>
//     <script>`, one line each, among the same first lines): each named script is run from the root in
//     its own toolchain's container, after merge-check.sh, and the layer passes only when every part
//     does (novox/hq issue 302 — the lab's Go replays went unchecked because its script runs in the
//     TypeScript toolchain, which holds no Go compiler). **The script, its parts and its toolchain are
//     the base branch's when it holds one** (novox/hq issue 310, check_base.go): a pull request cannot
//     skip its own tests by deleting or weakening them, and one that alters them is said so.
//
// One check:
//
//  1. clones the repository at the pull request's head and **makes the merge of that head into the trunk's
//     head**, the check's trunk commit, in its own clone (novox/hq ADR 0323, issue 341; check_merge.go): what
//     is judged is what merging makes, never pushed. Beside it the repositories its check reads — the
//     controller the mesh runs, the catalogue, the host, the lab — each at the ref asked;
//  2. reads the facts snapshot the controller keeps, and with it **the versions the mesh runs**: the
//     store a check's tests stand on is the store's own release, and the bus the bus's;
//  3. raises a throwaway PostgreSQL and a throwaway bus of those versions, labelled with the ask so a
//     kill or a crash leaves nothing behind;
//  4. runs the gate, then the repository's script — **each in a toolchain container of its own**, never
//     in the build machine: a pull request is code nobody has approved yet, and the build machine holds
//     the container runtime's socket; the check's container holds none, and reaches only the throwaway
//     store and bus on loopback. Only the replays — mesh-lab's main, reviewed code — get the socket;
//  5. answers each layer pass, warning or fail from what ran, and error — never pass — when it could
//     not run.

// CheckSpec is one check, as the controller asks it.
type CheckSpec struct {
	ID         string
	Repository string
	Ref        string
	Owner      string
	Repo       string
	Number     int
	Paths      []string
	// Beside are the repositories cloned next to it, by the directory they are found under.
	Beside map[string]Beside
	// Modules are the modules of the mesh's graph the change touches, New the directories it adds a
	// module in, and Manifests the manifests among them in the change's tree: the gate runs when Modules
	// or New is not empty.
	Modules   []string
	New       []string
	Manifests []string
	// Base is the branch the pull request merges into: what the gate compares the change against, and the trunk
	// whose head the change is merged into before it is judged (novox/hq ADR 0323). Empty judges the head alone.
	Base string
	// Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator"
	// the running one with the change's validator.
	Judge string
	// Release is the release branch the pull request merges into (novox/hq ADR 0358), empty for any other base. The
	// gate is then judged by the controller beside it as mesh-controller — mesh-controller's release branch, never
	// the controller the mesh runs nor its main — with mesh-host's validator from beside it when mesh-host was read at
	// the release branch too; a change to the controller still judges itself and a change to the node-engine brings
	// its own validator. The verdict names the branch, the judge's commit and the branch each beside was read at.
	Release string
	// JudgeCache is a directory outside the workspace where a release branch's judge is kept by the commits it is
	// built from, so it is built once per release-branch head (ADR 0358 §2). Outside: the check's containers mount
	// the workspace and a pull request's own script must not plant a judge for the next check. Empty keeps none.
	JudgeCache string
	// Toolchain is the image a check's Go runs in: the mesh's own Go toolchain, as it holds it.
	// Toolchains is every toolchain the mesh holds, by language, for a script that declares another.
	Toolchain  string
	Toolchains map[string]string
	// User is who a check's containers run as, uid:gid: the builder's own when empty — on the build seat,
	// the user its service runs as. A check run by hand (`mesh-controller check-here`) says the seat's.
	User string
	// Group are a delivery group's other heads (novox/hq ADR 0239), each cloned beside this one at its head
	// and composed with it by the gate as one future state. The repository's own check is not run for a
	// group: each member's pull request runs its own.
	Group []GroupHead
	// Builds are the modules the delivery plan moves, in the walk's order, and BuildWith what they are built with
	// (novox/hq ADR 0350 §1): built from the merge the check judged into the scratch namespace once it has judged,
	// for the walk to take. None for a group's composed check, whose members' own checks build.
	Builds    []CheckBuildSpec
	BuildWith *CheckBuilding
	// Tests and StoreOnly narrow what the repository's own script runs, for a branch tried before its pull
	// request (novox/hq issue 686): handed to the script as EnvTests and EnvStoreOnly, which it may read.
	Tests     string
	StoreOnly bool
}

// selection is what a check hands the repository's script to narrow what it runs (novox/hq issue 686): nothing
// for a pull request's check, which always runs whole.
func selection(spec CheckSpec) []string {
	var env []string
	if spec.Tests != "" {
		env = append(env, EnvTests+"="+spec.Tests)
	}
	if spec.StoreOnly {
		env = append(env, EnvStoreOnly+"=1")
	}
	return env
}

// GroupHead is one other head of a delivery group's composed check.
type GroupHead struct {
	Owner      string
	Repo       string
	Repository string
	Ref        string
	Paths      []string
	// Base is the branch it merges into: the head is merged into that trunk's head before it is composed
	// (novox/hq ADR 0323 §6). Empty composes the head alone, as a controller from before it asks.
	Base string
}

// Gated is whether the change touches the mesh's graph, and so whether the gate runs.
func (s CheckSpec) Gated() bool { return len(s.Modules)+len(s.New) > 0 }

// ToolchainOf is the Go toolchain image among what the mesh holds, empty when it holds none.
func ToolchainOf(held map[string]string) string {
	return ToolchainsOf(held)["go"]
}

// ToolchainsOf is every toolchain image the mesh holds, by language.
func ToolchainsOf(held map[string]string) map[string]string {
	out := map[string]string{}
	for _, chain := range toolchains {
		if image := held[chain.Base+"/"+chain.Artifact]; image != "" {
			out[chain.Language] = image
		}
	}
	return out
}

// Beside is one repository cloned next to the one checked: at Ref, or at Else when Else is said and the repository
// has no branch Ref (novox/hq ADR 0358 §3).
type Beside struct {
	Repository string
	Ref        string
	Else       string
}

// CheckVerdict is what came of one check. Verdict and Summary are the gate's; Gate and Repo each layer.
type CheckVerdict struct {
	Verdict string
	Summary string
	Report  string
	Took    time.Duration
	Gate    *Layer
	Repo    *Layer
	// Head and Trunk are what was judged (novox/hq ADR 0323 §3): the pull request's head merged into the trunk
	// commit, each as twelve characters. Trunk is empty for a check asked with no trunk, which judged the head
	// alone. Merge is the merge commit made for the check, whole: only checked, never pushed or registered.
	Head, Trunk, Merge string
	// Beside are the commits of the repositories the check read beside its own, by the directory each was
	// cloned under, as twelve characters: named, not judged against (ADR 0323 §3, issue 568).
	Beside map[string]string
	// Members are a delivery group's other heads as the composed check judged them, each with the trunk
	// commit it was merged into (ADR 0323 §6).
	Members []JudgedHead
	// Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not.
	Built []CheckBuilt
	// Release is the release branch it was judged for (novox/hq ADR 0358 §4), JudgedBy the commit of the controller
	// that judged its gate (twelve characters), and BesideAt the branch each repository beside it was read at, by
	// directory. Empty for a check of any other base.
	Release  string
	JudgedBy string
	BesideAt map[string]string
	// Validator says which node-engine validator the judge of a release-branch check was built with (ADR 0358 §2):
	// mesh-host's of the release branch with its commit, the one the release branch's controller vendors, the
	// change's own, or the one the change's own controller vendors. Empty for a check of any other base.
	Validator string
}

// Layer is one layer of a check, judged.
type Layer struct {
	Verdict string
	Summary string
	Modules []string
	// Failed is what the repository's own check printed that names what failed, picked from the whole of
	// its output and said at the end of the verdict's report (novox/hq issue 460). Empty when it passed.
	Failed string
}

// CheckScript is what a repository declares its own merge check as: run from its root, with the
// environment below.
const CheckScript = "merge-check.sh"

// Where a check finds what the builder raised and read for it.
const (
	EnvFacts      = "MESH_FACTS"
	EnvGate       = "MESH_GATE"
	EnvGateStore  = "MESH_GATE_POSTGRES"
	EnvTestStore  = "MESH_TEST_POSTGRES"
	EnvTestBus    = "MESH_TEST_NATS"
	EnvRepository = "MESH_CHECK_REPOSITORY"
	EnvChanged    = "MESH_CHECK_CHANGED"
	EnvVerdict    = "MESH_CHECK_VERDICT"
	EnvBeside     = "MESH_CHECK_BESIDE"
	EnvModules    = "MESH_CHECK_MODULES"
	// EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a
	// pull request checked alone.
	EnvGroup = "MESH_CHECK_GROUP"
	// EnvTests is a `go test -run` pattern and EnvStoreOnly is 1 when a branch tried before its pull request
	// asks for only those tests, or only the store tests (novox/hq issue 686). Never set for a pull request's
	// check. A script that reads neither runs whole.
	EnvTests     = "MESH_CHECK_TESTS"
	EnvStoreOnly = "MESH_CHECK_STORE_ONLY"
)

// CheckTimeout bounds one check; a check that runs past it is an error, not a pass.
var CheckTimeout = 45 * time.Minute

// reportLines is how much of what a check printed travels in its verdict.
const reportLines = 200

// noModule is the gate's word for a change that touches nothing of the mesh's graph: a fact, not a
// missing check, so a pass.
const noModule = "the change touches no module of the mesh's graph"

// noScript is the repository layer's word for a repository with no merge-check.sh of its own.
const noScript = "the repository declares no " + CheckScript + ": none of its own tests run before it merges"

// toolchainLine is how a merge-check.sh declares the toolchain it runs in.
var toolchainLine = regexp.MustCompile(`^#\s*mesh-check-toolchain:\s*([a-z0-9-]+)\s*$`)

// alsoLine is how a merge-check.sh declares a further part of the repository's check, run in a toolchain
// of its own: `# mesh-check-also: <toolchain> <script>`.
var alsoLine = regexp.MustCompile(`^#\s*mesh-check-also:\s*([a-z0-9-]+)\s+(\S+)\s*$`)

// ScriptPart is one part of a repository's own check: a script, run from the repository's root, and the
// toolchain it runs in.
type ScriptPart struct {
	Toolchain string
	Script    string
}

// ScriptParts is every part of a repository's own check: its merge-check.sh in the toolchain it declares,
// then each further part it declares among its first twenty lines (`# mesh-check-also: <toolchain>
// <script>`), in the order declared. A declared script that is not a path inside the repository is
// answered as a part all the same, for the check to refuse by name rather than to drop.
func ScriptParts(script []byte) []ScriptPart {
	parts := []ScriptPart{{Toolchain: ScriptToolchain(script), Script: CheckScript}}
	lines := bufio.NewScanner(bytes.NewReader(script))
	for i := 0; i < 20 && lines.Scan(); i++ {
		if m := alsoLine.FindStringSubmatch(strings.TrimSpace(lines.Text())); m != nil {
			parts = append(parts, ScriptPart{Toolchain: m[1], Script: m[2]})
		}
	}
	return parts
}

// ScriptToolchain is the language a merge-check.sh declares it runs in, among its first twenty lines;
// go when it declares none.
func ScriptToolchain(script []byte) string {
	lines := bufio.NewScanner(bytes.NewReader(script))
	for i := 0; i < 20 && lines.Scan(); i++ {
		if m := toolchainLine.FindStringSubmatch(strings.TrimSpace(lines.Text())); m != nil {
			return m[1]
		}
	}
	return "go"
}

// Check runs one merge check. An error is that it could not run; the verdict is then "error".
func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential,
	log Log) (CheckVerdict, error) {
	// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462).
	redact := redactorFor(forge)
	say := logging(func(step, message string) {
		if log != nil {
			log(step, redact.redact(message))
		}
	})
	began := time.Now()
	ctx, stop := context.WithTimeout(ctx, CheckTimeout)
	defer stop()

	root := filepath.Join(workspace, "check")
	if err := os.RemoveAll(root); err != nil {
		return CheckVerdict{}, err
	}
	if err := os.MkdirAll(root, 0o755); err != nil {
		return CheckVerdict{}, err
	}
	defer os.RemoveAll(root)
	// The forge credential lives outside the workspace the check's containers mount, and only while the
	// check clones (novox/hq issue 462).
	credentials, forget, err := storeCredential(workspace, forge)
	if err != nil {
		return CheckVerdict{}, err
	}
	defer forget()
	if err := forgetLeftCredentials(workspace); err != nil {
		return CheckVerdict{}, err
	}
	// cloneAt clones a repository and checks out ref — or, when the repository has no branch ref and orElse is said,
	// orElse (novox/hq ADR 0358 §3); readAt keeps which, by directory.
	readAt := map[string]string{}
	cloneAt := func(repository, ref, orElse, dir string) error {
		if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil {
			return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error()))
		}
		if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil {
			return err
		}
		if orElse != "" && ref != "" {
			if _, err := run(ctx, filepath.Join(root, dir), "git", "rev-parse", "--verify", "--quiet",
				"refs/remotes/origin/"+ref); err != nil {
				ref = orElse
			}
		}
		readAt[dir] = ref
		if ref != "" {
			if _, err := run(ctx, filepath.Join(root, dir), "git", "checkout", "--quiet", ref); err != nil {
				return fmt.Errorf("%s has no %s: %w", redact.redact(repository), ref, err)
			}
		}
		return nil
	}
	clone := func(repository, ref, dir string) error {
		return cloneAt(repository, ref, "", dir)
	}
	name := spec.Repo
	if name == "" || !safeName.MatchString(name) {
		name = "checked"
	}
	say("check", "%s/%s#%d at %s", spec.Owner, spec.Repo, spec.Number, short(spec.Ref))
	if err := clone(spec.Repository, spec.Ref, name); err != nil {
		return CheckVerdict{}, err
	}
	tree := filepath.Join(root, name)
	// **What merging makes is judged, not the head** (novox/hq ADR 0323, issue 341): the head merged into the
	// trunk's head as this clone found it, in this clone alone. A head that does not merge cleanly fails here.
	judged := CheckVerdict{Head: twelve(spec.Ref)}
	paths := spec.Paths
	if spec.Base != "" {
		m, err := mergeIntoTrunk(ctx, run, tree, spec.Base, spec.Ref)
		if err != nil {
			return CheckVerdict{}, err
		}
		judged.Head, judged.Trunk = twelve(m.Head), twelve(m.Trunk)
		if len(m.Conflicts) > 0 {
			said := conflictWords(m)
			say("check", "%s", said)
			return CheckVerdict{Verdict: "fail", Summary: said, Report: said, Took: time.Since(began),
				Gate: &Layer{Verdict: "fail", Summary: said}, Repo: &Layer{Verdict: "fail", Summary: said},
				Head: judged.Head, Trunk: judged.Trunk}, nil
		}
		judged.Merge = m.Merge
		if changed := changedByMerge(ctx, run, tree, m); len(changed) > 0 {
			paths = changed
		}
		say("check", "head %s merged into trunk %s (%s): the merge is judged, and never pushed", judged.Head,
			judged.Trunk, spec.Base)
	}
	named := func(v CheckVerdict) CheckVerdict {
		v.Head, v.Trunk, v.Merge, v.Beside, v.Members = judged.Head, judged.Trunk, judged.Merge, judged.Beside, judged.Members
		v.Release, v.JudgedBy, v.BesideAt, v.Validator = spec.Release, judged.JudgedBy, judged.BesideAt, judged.Validator
		return v
	}
	// **What the merge would move is built from a checkout nothing of the check touches** (novox/hq issue 703): the
	// gate and the repository's own check run in containers that mount the workspace read-write, so they can change
	// its tree, its objects and its caches. The builds are made from a copy of the merge's objects, cloned before
	// anything of the change runs, outside the workspace — so what is kept under a key is what the key describes.
	building, buildingTree := "", ""
	if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
		at := judged.Merge
		if at == "" {
			at = spec.Ref
		}
		building, buildingTree, err = checkoutForTheBuilds(ctx, run, tree, at, workspace)
		if err != nil {
			return CheckVerdict{}, err
		}
		defer os.RemoveAll(building)
	}
	// **The base branch's check judges the change** (novox/hq issue 310), never the one the change brings.
	judging, err := TheCheckThatJudges(ctx, tree, spec.Base)
	if err != nil {
		return CheckVerdict{}, err
	}
	script := judging.Script
	hasScript := script != nil
	if judging.Said != "" {
		say("check", "%s", judging.Said)
	}
	gated := spec.Gated()
	if !gated && !hasScript {
		// Nothing to run: said, never passed silently.
		v := CheckVerdict{Verdict: "pass", Summary: noModule, Took: time.Since(began),
			Gate: &Layer{Verdict: "pass", Summary: noModule}, Repo: &Layer{Verdict: "warning", Summary: noScript}}
		say("check", "%s; %s", noModule, noScript)
		return named(v), nil
	}
	for dir, b := range spec.Beside {
		if dir == name || !safeName.MatchString(dir) {
			continue
		}
		if dir == "mesh-lab" {
			// **mesh-lab is never read at a release branch** (novox/hq ADR 0358): its replays run with the container
			// runtime's socket, so a branch of a release's name — which anyone may create — is never what runs there,
			// whether it is asked with a fallback or without.
			if b.Else != "" || (spec.Release != "" && b.Ref == spec.Release) {
				b.Ref = b.Else
				if b.Ref == "" {
					b.Ref = "main"
				}
			}
			b.Else = ""
		}
		if err := cloneAt(b.Repository, b.Ref, b.Else, dir); err != nil {
			return CheckVerdict{}, fmt.Errorf("beside it, %w", err)
		}
		if spec.Release != "" {
			// The branch each was read at is named with its commit (ADR 0358 §4).
			if judged.BesideAt == nil {
				judged.BesideAt = map[string]string{}
			}
			judged.BesideAt[dir] = readAt[dir]
		}
		// The commit each one read is named in the verdict (ADR 0323 §3): its move is issue 568's question.
		if at := headAt(ctx, run, filepath.Join(root, dir)); at != "" {
			if judged.Beside == nil {
				judged.Beside = map[string]string{}
			}
			judged.Beside[dir] = at
		}
		say("check", "beside it %s at %s", dir, refWords(readAt[dir]))
	}

	// A delivery group's other heads (novox/hq ADR 0239), each at its head, for the gate to compose with this.
	groupFile := ""
	if len(spec.Group) > 0 {
		var heads []map[string]any
		for i, g := range spec.Group {
			dir := fmt.Sprintf("group-%d", i)
			if g.Repo != "" && safeName.MatchString(g.Repo) {
				dir = "group-" + g.Repo
			}
			if err := clone(g.Repository, g.Ref, dir); err != nil {
				return CheckVerdict{}, fmt.Errorf("a head of the group, %w", err)
			}
			say("check", "with it, of its group, %s/%s at %s", g.Owner, g.Repo, short(g.Ref))
			// Each member is composed as its merge into its own trunk's head (ADR 0323 §6).
			member := JudgedHead{Owner: g.Owner, Repo: g.Repo, Head: twelve(g.Ref)}
			if g.Base != "" {
				m, err := mergeIntoTrunk(ctx, run, filepath.Join(root, dir), g.Base, g.Ref)
				if err != nil {
					return CheckVerdict{}, fmt.Errorf("a head of the group, %s/%s: %w", g.Owner, g.Repo, err)
				}
				member.Head, member.Trunk = twelve(m.Head), twelve(m.Trunk)
				if len(m.Conflicts) > 0 {
					said := fmt.Sprintf("%s/%s: %s", g.Owner, g.Repo, conflictWords(m))
					say("check", "%s", said)
					judged.Members = append(judged.Members, member)
					return named(CheckVerdict{Verdict: "fail", Summary: said, Report: said, Took: time.Since(began),
						Gate: &Layer{Verdict: "fail", Summary: said}, Repo: &Layer{Verdict: "fail", Summary: said}}), nil
				}
				if changed := changedByMerge(ctx, run, filepath.Join(root, dir), m); len(changed) > 0 {
					g.Paths = changed
				}
			}
			judged.Members = append(judged.Members, member)
			heads = append(heads, map[string]any{"repository": g.Owner + "/" + g.Repo,
				"tree": filepath.Join(root, dir), "changed": g.Paths})
		}
		raw, err := json.Marshal(heads)
		if err != nil {
			return CheckVerdict{}, err
		}
		groupFile = filepath.Join(root, "group.json")
		if err := os.WriteFile(groupFile, raw, 0o644); err != nil {
			return CheckVerdict{}, err
		}
	}

	// Every clone is made: the credential is gone before anything of the check runs (novox/hq issue 462).
	forget()

	// The facts, and the versions they say the mesh runs.
	if registry == "" {
		return CheckVerdict{}, errors.New("no artifact store to read the facts snapshot from")
	}
	body, digest, err := (artifacts.Store{Address: registry}).GetTagged(ctx, facts.Repository, facts.Tag)
	if err != nil {
		return CheckVerdict{}, fmt.Errorf("the facts snapshot cannot be read, and a check without it judges nothing: %w", err)
	}
	f, err := facts.Decode(body)
	if err != nil {
		return CheckVerdict{}, err
	}
	factsFile := filepath.Join(root, "facts.json")
	if err := os.WriteFile(factsFile, body, 0o644); err != nil {
		return CheckVerdict{}, err
	}
	say("check", "the facts of %s (%s): %d machine(s), the bus at %s, the store at %s", f.Taken.Format(time.RFC3339),
		short(strings.TrimPrefix(digest, "sha256:")), len(f.Machines), f.Versions.Bus, f.Versions.Store)

	// **What an earlier delivery of this same ask left is removed first** (novox/hq issue 285): an ask is
	// redelivered when the holder that took it stopped mid-check — the build agent itself updated by the
	// rollout it is checking — and its containers, named by the ask's id, are still there. Raising the
	// store again under that name was refused, and the check said it could not run.
	if n, err := RemoveContainersOf(ctx, run, spec.ID); err == nil && n > 0 {
		say("check", "removed %d throwaway container(s) an earlier delivery of this check left", n)
	}
	// The throwaway store and bus, of the versions the mesh runs, removed whatever happens.
	defer func() {
		removing, done := context.WithTimeout(context.Background(), time.Minute)
		defer done()
		if n, err := RemoveContainersOf(removing, run, spec.ID); err == nil && n > 0 {
			say("check", "removed %d throwaway container(s)", n)
		}
	}()
	labelled := Labelled(run, spec.ID)
	store, err := throwaway(ctx, labelled, run, spec.ID+"-store", StoreImage(f.Versions.Store), 5432,
		[]string{"-e", "POSTGRES_PASSWORD=check"}, StoreSettings)
	if err != nil {
		return CheckVerdict{}, err
	}
	storeURL := "postgres://postgres:check@" + store + "/postgres?sslmode=disable"
	if err := waitFor(ctx, run, spec.ID+"-store", []string{"pg_isready", "-U", "postgres"}); err != nil {
		return CheckVerdict{}, err
	}
	bus, err := throwaway(ctx, labelled, run, spec.ID+"-bus", BusImage(f.Versions.Bus), 4222, nil, []string{"-js"})
	if err != nil {
		return CheckVerdict{}, err
	}
	if err := dialable(ctx, bus); err != nil {
		return CheckVerdict{}, err
	}
	say("check", "a throwaway store (%s) and bus (%s) of the versions the mesh runs", StoreImage(f.Versions.Store),
		BusImage(f.Versions.Bus))

	if spec.Toolchain == "" {
		return CheckVerdict{}, errors.New("the mesh holds no Go toolchain to run a check in")
	}
	user := spec.User
	if user == "" {
		user = fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid())
	}
	in := func(image, dir string, env []string, command ...string) []string {
		home := workspace
		for _, e := range env {
			if strings.HasPrefix(e, "HOME=") {
				home = strings.TrimPrefix(e, "HOME=")
			}
		}
		// As the builder itself: what a check writes into the workspace is the builder's to remove.
		args := []string{"run", "--rm", "--network", "host", "--volume", workspace + ":" + workspace, "--workdir", dir,
			"--user", user, "--env", "HOME=" + home,
			// The check's own checkouts, whoever cloned them: git in a container of another user than the
			// one that cloned refuses a repository it does not own ("dubious ownership"), and Go's build
			// stamps the version from git — a judge that would not build for want of it.
			"--env", "GIT_CONFIG_COUNT=1", "--env", "GIT_CONFIG_KEY_0=safe.directory", "--env", "GIT_CONFIG_VALUE_0=*"}
		for _, e := range env {
			if !strings.HasPrefix(e, "HOME=") {
				args = append(args, "--env", e)
			}
		}
		return append(append(args, image), command...)
	}
	inToolchain := func(dir string, env []string, command ...string) []string {
		return in(spec.Toolchain, dir, env, command...)
	}
	// **Every judge is built where no pull request can reach** (novox/hq issue 482, ADR 0358 review): the workspace's
	// Go cache, HOME and git configuration are written by pull requests' own scripts, and git runs during a Go build
	// to stamp its version — a planted hook would rewrite the judge's sources before they compile. So a judge is built
	// with a HOME, configuration and Go cache of its own, outside the workspace and made fresh for this check, no
	// global or system git configuration, no Go environment file, and no version stamping from git.
	judgeHome, err := os.MkdirTemp("", "mesh-check-judge-")
	if err != nil {
		return CheckVerdict{}, err
	}
	defer os.RemoveAll(judgeHome)
	if withinDir(workspace, judgeHome) {
		return CheckVerdict{}, fmt.Errorf("the temporary directory %s is inside the workspace %s, which a check's "+
			"containers mount: no judge is built where a pull request could reach it", judgeHome, workspace)
	}
	inJudgeBuild := func(dir string, _ []string, command ...string) []string {
		args := in(spec.Toolchain, dir, JudgeBuildEnv(judgeHome), command...)
		return append([]string{args[0], "--volume", judgeHome + ":" + judgeHome}, args[1:]...)
	}
	var out tail
	// running runs one container of the check, its output into the report, in a process group of its own.
	running := func(args []string) error {
		cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", args, spec.ID)...)
		inItsOwnGroup(cmd)
		cmd.Stdout, cmd.Stderr = &out, &out
		return cmd.Run()
	}

	// The judge. Its failing to build is the change's fault when the change is the judge or its validator,
	// and the check's when it is the controller the mesh runs.
	gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, judged.BesideAt, inJudgeBuild, say)
	if err != nil {
		return CheckVerdict{}, err
	}
	if spec.Release != "" {
		// **A release-branch judgement says so** (novox/hq ADR 0358 §4): the branch, the controller's commit that
		// judged, and each repository read beside it at its branch and commit.
		judgedBy := filepath.Join(root, "mesh-controller")
		if spec.Judge == "self" {
			judgedBy = tree
		}
		judged.JudgedBy = headAt(ctx, run, judgedBy)
		judged.Validator = releaseValidator(spec, judged.BesideAt, judged.Beside)
		said := releaseWords(spec.Release, judged.JudgedBy, judged.Validator, judged.BesideAt, judged.Beside)
		fmt.Fprintf(&out, "--- %s\n", said)
		say("check", "%s", said)
	}
	verdictFile := filepath.Join(root, "verdict.json")
	env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL,
		EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo,
		EnvChanged + "=" + strings.Join(paths, ","), EnvBeside + "=" + root,
		EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
		EnvGroup + "=" + groupFile,
		"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
	env = append(env, selection(spec)...)

	v := CheckVerdict{}
	modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...)
	timedOut := func() bool { return errors.Is(ctx.Err(), context.DeadlineExceeded) }

	// **The gate**, when the graph says the change touches it.
	if !gated {
		v.Gate = &Layer{Verdict: "pass", Summary: noModule}
	} else {
		v.Gate = &Layer{Modules: modules}
		switch {
		case judgeFault != "":
			v.Gate.Verdict, v.Gate.Summary = "fail", judgeFault
		default:
			say("check", "the gate: %d module(s) of the graph touched — %s", len(modules), strings.Join(modules, ", "))
			fmt.Fprintf(&out, "--- the gate: %s\n", strings.Join(modules, ", "))
			v.Gate.Verdict, v.Gate.Summary = gateLayer(ctx, spec, tree, root, gate, verdictFile, env, inToolchain,
				running, &out, bus, workspace, name, say)
		}
		if timedOut() {
			v.Gate.Verdict, v.Gate.Summary = "error", fmt.Sprintf("the check ran past %s and was ended", CheckTimeout)
		}
	}
	if ctx.Err() != nil && !timedOut() {
		return v, ctx.Err()
	}

	// **The repository's own check**, in the toolchain it declares.
	switch {
	case len(spec.Group) > 0:
		// A group's composed check judges the heads together; each member's own tests are its pull request's.
		v.Repo = &Layer{Verdict: "pass", Summary: "a group's composed check: each member's own " + CheckScript +
			" runs on its pull request"}
	case !hasScript:
		v.Repo = &Layer{Verdict: "warning", Summary: noScript}
	case timedOut():
		v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("the check ran past %s before its %s ran", CheckTimeout, CheckScript)}
	default:
		// The base branch's scripts in place of the change's, in the change's tree — after the gate, which
		// composes the change as it is.
		if err := judging.Put(tree); err != nil {
			return v, err
		}
		if judging.Said != "" {
			fmt.Fprintf(&out, "--- %s\n", judging.Said)
		}
		v.Repo = ownCheck(ctx, spec, ScriptParts(script), tree, &out, timedOut, func(image string, script string) *exec.Cmd {
			return exec.CommandContext(ctx, "docker", LabelledArgs("docker", in(image, tree, env, "sh", script), spec.ID)...)
		}, say)
		if v.Repo == nil {
			return v, ctx.Err()
		}
		if judging.Said != "" {
			v.Repo.Summary = judging.Said + "; " + v.Repo.Summary
		}
	}

	if spec.Release != "" {
		// The statuses name the release branch and the judge first, so the forge's short description keeps them.
		mark := releaseMark(spec.Release, judged.JudgedBy)
		v.Gate.Summary, v.Repo.Summary = mark+v.Gate.Summary, mark+v.Repo.Summary
	}
	// What the check printed travels in the verdict to the forge and the controller: redacted as the log is.
	v.Gate.Summary, v.Repo.Summary, v.Repo.Failed = redact.redact(v.Gate.Summary), redact.redact(v.Repo.Summary),
		redact.redact(v.Repo.Failed)
	v.Verdict, v.Summary = v.Gate.Verdict, v.Gate.Summary
	v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
	say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
		strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))

	// **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it
	// judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again.
	if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
		if why := skipsTheBuilds(v); why != "" {
			say("check", "%s", why)
		} else {
			began := time.Now()
			v.Built = buildForTheMerge(ctx, run, spec, *spec.BuildWith, building, buildingTree, judged.Merge, judged, forge,
				func(step, message string) {
					if log != nil {
						log(step, redact.redact(message))
					}
				}, say)
			say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(v.Built), len(v.Built),
				time.Since(began).Round(time.Second))
		}
	}
	return named(v), nil
}

// checkoutForTheBuilds is a clone of the commit a check builds for its merge, made before anything of the check runs,
// outside the workspace its containers mount (novox/hq issue 703): the directory to remove, and the clone in it.
// Cloned over git's own transport, never by linking the objects: what the check's containers later write into the
// workspace's objects cannot reach it.
func checkoutForTheBuilds(ctx context.Context, run Runner, tree, at, workspace string) (string, string, error) {
	dir, err := os.MkdirTemp("", "mesh-check-builds-")
	if err != nil {
		return "", "", err
	}
	if withinDir(workspace, dir) {
		os.RemoveAll(dir)
		return "", "", fmt.Errorf("the temporary directory %s is inside the workspace %s, which a check's containers "+
			"mount: the merge is not built from where its tests could change it", dir, workspace)
	}
	commit, err := run(ctx, tree, "git", "rev-parse", "--verify", at+"^{commit}")
	if err != nil {
		os.RemoveAll(dir)
		return "", "", fmt.Errorf("the commit to build for the merge, %s: %w", at, err)
	}
	commit = strings.TrimSpace(commit)
	if _, err := run(ctx, tree, "git", "update-ref", "refs/heads/mesh-check-builds", commit); err != nil {
		os.RemoveAll(dir)
		return "", "", err
	}
	cloned := filepath.Join(dir, "merged")
	if _, err := run(ctx, dir, "git", "clone", "--quiet", "--no-local", "--no-checkout", "--branch",
		"mesh-check-builds", tree, cloned); err != nil {
		os.RemoveAll(dir)
		return "", "", fmt.Errorf("cannot copy the merge to build it: %w", err)
	}
	return dir, cloned, nil
}

// builtCount is how many of a check's builds for its merge were built.
func builtCount(built []CheckBuilt) int {
	n := 0
	for _, b := range built {
		if b.NotBuilt == "" {
			n++
		}
	}
	return n
}

// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
// layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller
// answers as the context's error.
func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree string, out *tail, timedOut func() bool,
	command func(image, script string) *exec.Cmd, say func(step, format string, args ...any)) *Layer {
	var ran []string
	for i, part := range parts {
		named := part.Script
		if i > 0 {
			named = CheckScript + "'s part " + part.Script
		}
		image := spec.Toolchains[part.Toolchain]
		if part.Toolchain == "go" && image == "" {
			image = spec.Toolchain
		}
		if image == "" {
			return &Layer{Verdict: "error", Summary: fmt.Sprintf("%sits %s runs in the %s toolchain, which the mesh does "+
				"not hold", passedSoFar(ran), named, part.Toolchain)}
		}
		if i > 0 && !filepath.IsLocal(part.Script) {
			return &Layer{Verdict: "fail", Summary: fmt.Sprintf("%s declares a part %q that is not a path inside the "+
				"repository", CheckScript, part.Script)}
		}
		if i > 0 {
			if info, err := os.Stat(filepath.Join(tree, part.Script)); err != nil || info.IsDir() {
				return &Layer{Verdict: "fail", Summary: fmt.Sprintf("%s declares a part %s, which the repository does "+
					"not hold", CheckScript, part.Script)}
			}
		}
		say("check", "running its %s in the mesh's %s toolchain", part.Script, part.Toolchain)
		fmt.Fprintf(out, "--- its %s (%s toolchain)\n", part.Script, part.Toolchain)
		var own tail
		var picked failures
		logged := &toTheLog{say: say}
		cmd := command(image, part.Script)
		inItsOwnGroup(cmd)
		w := io.MultiWriter(out, &own, &picked, logged)
		cmd.Stdout, cmd.Stderr = w, w
		err := cmd.Run()
		logged.close(picked.String())
		switch {
		case timedOut():
			return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout),
				Failed: picked.String()}
		case ctx.Err() != nil:
			return nil
		case err != nil:
			// Named from the whole run, not the tail: what failed may be thousands of lines from the end
			// (novox/hq issue 460).
			said := picked.said()
			if said == "" {
				said = whatFailed(own.String())
			}
			return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + said,
				Failed: picked.String()}
		}
		ran = append(ran, fmt.Sprintf("%s (%s)", part.Script, part.Toolchain))
	}
	if len(ran) == 1 {
		return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed"}
	}
	return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed, each part in its toolchain: " + strings.Join(ran, ", ")}
}

// withWhatFailed is a check's report with what its repository's own check names as failed said last, whole:
// the report's tail is the last reportLines lines, and the controller keeps the end of a report it cuts, so
// what failed travels whatever came before or after it (novox/hq issue 460).
func withWhatFailed(report string, repo *Layer) string {
	if repo == nil || repo.Failed == "" {
		return report
	}
	return report + "\n--- what failed, picked from the whole of its output (the whole is in the build's log)\n" +
		repo.Failed
}

// logLines bounds the lines of a repository's own check that go into the build's log. The bus keeps 10 000
// messages per subject (EVENTS' MaxMsgsPerSubject), and a build's log is one subject: past that, its first
// lines would be lost. The build's own lines are a few hundred at most, so the check's output takes 8 000.
const logLines = 8000

// logLineBytes bounds one line of it, so a line that pastes a document does not fill a message.
const logLineBytes = 4 << 10

// toTheLog says each line a repository's own check prints in the build's log, as it prints it — the whole
// output kept for the build and read with `builds` and its id (novox/hq issue 460), where before only the
// verdict's tail of it was kept. Past logLines it says how many lines it left out, and the lines that name
// what failed after them.
type toTheLog struct {
	say     func(step, format string, args ...any)
	partial []byte
	said    int
	dropped int
}

func (l *toTheLog) Write(p []byte) (int, error) {
	l.partial = append(l.partial, p...)
	for {
		i := bytes.IndexByte(l.partial, '\n')
		if i < 0 {
			break
		}
		l.line(string(bytes.TrimRight(l.partial[:i], "\r")))
		l.partial = l.partial[i+1:]
	}
	if len(l.partial) > logLineBytes {
		l.line(string(l.partial))
		l.partial = nil
	}
	return len(p), nil
}

func (l *toTheLog) line(line string) {
	if l.said >= logLines {
		l.dropped++
		return
	}
	if len(line) > logLineBytes {
		line = line[:logLineBytes] + fmt.Sprintf(" … (%d bytes more)", len(line)-logLineBytes)
	}
	l.said++
	// Redacted by its shape before the bus keeps it (novox/hq issue 462); Check's own say adds the secrets
	// the builder knows.
	l.say("output", "%s", redactor{}.redact(line))
}

// close says the last line, and, when lines were left out, how many and what failed.
func (l *toTheLog) close(failed string) {
	if len(l.partial) > 0 {
		l.line(string(l.partial))
		l.partial = nil
	}
	if l.dropped == 0 {
		return
	}
	l.say("output", "… %d more line(s) of its output are not in this log, which keeps its first %d", l.dropped, logLines)
	if failed == "" {
		return
	}
	l.say("output", "--- what failed, picked from the whole of its output")
	for _, line := range strings.Split(failed, "\n") {
		l.say("output", "%s", redactor{}.redact(line))
	}
}

// passedSoFar is what of a check's parts passed before the one that did not, said first.
func passedSoFar(ran []string) string {
	if len(ran) == 0 {
		return ""
	}
	return strings.Join(ran, ", ") + " passed; "
}

// SomeManifestsEnv tells the judge's module check that the manifests it is given are only those a change touches,
// so a seat another module of the repository declares is a note there, not a refusal (novox/hq issue 364). A judge
// that predates it reads nothing of it and refuses as before.
const SomeManifestsEnv = "MESH_MODULE_CHECK_SOME=1"

// gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the
// change, and the replays of what the mesh runs. It answers the gate's verdict and summary.
func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string,
	inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus, workspace,
	name string, say func(step, format string, args ...any)) (string, string) {
	// 1. The manifests the change touches, as the judge reads them: a manifest it cannot read, or one with a
	// problem the change brings, fails here. **What was already so on the base branch is said and fails
	// nothing** — the gate's own rule: a module whose manifest the running controller already finds fault
	// with would otherwise fail every pull request that touches it, for a fault none of them made.
	var manifests []string
	for _, m := range spec.Manifests {
		if _, err := os.Stat(filepath.Join(tree, m)); err == nil {
			manifests = append(manifests, m)
		}
	}
	if len(manifests) > 0 {
		checked := func(dir string) (string, error) {
			var own tail
			cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker",
				inToolchain(dir, append(append([]string{}, env...), SomeManifestsEnv), append([]string{gate, "module", "check"}, manifests...)...), spec.ID)...)
			inItsOwnGroup(cmd)
			w := io.MultiWriter(out, &own)
			cmd.Stdout, cmd.Stderr = w, w
			err := cmd.Run()
			return own.String(), err
		}
		said, err := checked(tree)
		if err != nil {
			if ctx.Err() != nil {
				return "error", "the check was ended during the module check"
			}
			// The same manifests as the base branch has them, beside the change.
			was := ""
			if spec.Base != "" {
				base := filepath.Join(root, "base-manifests")
				for _, m := range manifests {
					body, err := gitShow(ctx, tree, "origin/"+spec.Base, m)
					if err != nil {
						continue // new on this branch: nothing was so before it
					}
					if err := os.MkdirAll(filepath.Dir(filepath.Join(base, m)), 0o755); err == nil {
						_ = os.WriteFile(filepath.Join(base, m), body, 0o644)
					}
				}
				fmt.Fprintf(out, "--- the same manifests on %s\n", spec.Base)
				if _, err := os.Stat(base); err == nil {
					was, _ = checked(base)
				}
			}
			brought := newProblems(said, was)
			if len(brought) > 0 {
				return "fail", "a manifest the change touches fails the module check: " + brought[0]
			}
			fmt.Fprintf(out, "the module check's problems were all so on %s already: said, not the change's\n", spec.Base)
		}
	}

	// 2. Every machine composed with the change.
	if err := running(inToolchain(tree, append(env, EnvVerdict+"="+verdictFile), "sh", "-c",
		`"$MESH_GATE" merge-gate --facts "$MESH_FACTS" --store "$MESH_GATE_POSTGRES" `+
			`--repository "$MESH_CHECK_REPOSITORY" --tree . --changed "$MESH_CHECK_CHANGED" `+
			`${MESH_CHECK_GROUP:+--group "$MESH_CHECK_GROUP"} --json > "$MESH_CHECK_VERDICT"`)); err != nil {
		if ctx.Err() != nil {
			return "error", "the check was ended during the merge gate"
		}
		var said struct {
			Verdict string `json:"verdict"`
			Summary string `json:"summary"`
		}
		if raw, err := os.ReadFile(verdictFile); err == nil && json.Unmarshal(raw, &said) == nil {
			switch said.Verdict {
			case "fail":
				return "fail", said.Summary
			case "error":
				// The gate could not raise the mesh as it is (novox/hq issue 285): said in its own words.
				return "error", said.Summary
			}
		}
		// The gate could not judge: not the change's fault, and never a pass.
		return "error", "the merge gate could not judge the change: " + lastLine(out.String())
	}
	raw, err := os.ReadFile(verdictFile)
	if err != nil {
		return "error", "the merge gate said no verdict"
	}
	said, ok := readGateVerdict(raw)
	if !ok {
		return "error", "the merge gate said no verdict"
	}
	if verdict, summary, raised := gateRaisedTheMesh(said); !raised {
		return verdict, summary
	}

	// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
	// so given the container runtime the resolver replay raises containers with — against the bus of the
	// release the mesh runs and the change's own catalogue when the change is to the catalogue.
	if lab := filepath.Join(root, "mesh-lab", "replays"); ctx.Err() == nil {
		if _, err := os.Stat(lab); err == nil {
			catalogue := filepath.Join(root, "mesh-catalog")
			if name == "mesh-catalog" {
				catalogue = tree
			}
			say("check", "the replays of what the mesh runs, from mesh-lab")
			fmt.Fprintln(out, "--- the replays (mesh-lab replays/)")
			args := inToolchain(lab, []string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue,
				"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")},
				"go", "test", "-count=1", "./...")
			// The socket goes to the replays alone, never to the change's own code above.
			args = append([]string{args[0], "--volume", "/var/run/docker.sock:/var/run/docker.sock"}, args[1:]...)
			if err := running(args); err != nil {
				if ctx.Err() != nil {
					return "error", "the check was ended during the replays"
				}
				return "fail", "a replay of a core incident fails with this change: " + lastLine(out.String())
			}
		}
	}
	if said.Verdict == "pass" || said.Verdict == "warning" || said.Verdict == "fail" {
		return said.Verdict, said.Summary
	}
	return "error", "the merge gate said " + said.Verdict
}

// judgeFor builds the judge of a check: the change's own controller (a change to the controller), the
// running controller built with the change's validator (a change to the node-engine), or the controller
// the mesh runs. It answers the judge's path; or, when the change makes its own judge unbuildable, why —
// the change's fault, a failing gate; or an error when the check cannot build a judge at all. A check
// whose gate does not run builds a judge only to hand its scripts one, and goes on without when it cannot.
//
// **A pull request into a release branch is judged by that branch's controller** (novox/hq ADR 0358 §2): the one
// beside it as mesh-controller, read at the release branch — with mesh-host's validator from beside it when mesh-host
// has that branch too, or with the change's own validator for a change to the node-engine — never the controller the
// mesh runs. besideAt is the branch each repository beside it was read at.
func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string, besideAt map[string]string,
	inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) {
	gated := spec.Gated()
	switch spec.Judge {
	case "self":
		bin := filepath.Join(root, "bin", "judge-of-itself")
		if _, err := labelled(ctx, root, "docker", inToolchain(tree, nil,
			"go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil {
			return "", "the change's controller, which judges itself, does not build: " + firstLine(err.Error()), nil
		}
		say("check", "judged by the change's own controller")
		return bin, "", nil
	}
	var bin, judgeTree string
	var err error
	if spec.Release != "" {
		withHost := spec.Judge != "validator" && besideAt["mesh-host"] == spec.Release
		bin, judgeTree, err = releaseJudge(ctx, labelled, run, spec, root, withHost, inToolchain, say)
	} else {
		bin, judgeTree, err = judge(ctx, labelled, run, root, inToolchain, say)
	}
	if err != nil {
		if !gated {
			say("check", "no judge for its script: %v", err)
			return "", "", nil
		}
		return "", "", err
	}
	if spec.Judge != "validator" {
		return bin, "", nil
	}
	// The node-engine's change: its validator in place of the one the judge vendors.
	vendored := filepath.Join(root, judgeTree, "vendor", "github.com", "novox", "mesh-host")
	for _, pkg := range []string{"validate", filepath.Join("internal", "declaration")} {
		if err := replaceGoFiles(filepath.Join(tree, pkg), filepath.Join(vendored, pkg)); err != nil {
			return "", "", fmt.Errorf("the change's validator could not be put in the judge: %w", err)
		}
	}
	withValidator := filepath.Join(root, "bin", "judge-with-this-validator")
	if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree), nil,
		"go", "build", "-o", withValidator, "./cmd/mesh-controller")...); err != nil {
		if spec.Release != "" {
			return "", spec.Release + "'s controller does not build with this change's validator: " +
				firstLine(err.Error()), nil
		}
		return "", "the controller the mesh runs does not build with this change's validator: " + firstLine(err.Error()), nil
	}
	if spec.Release != "" {
		say("check", "judged by %s's controller, with this change's validator", spec.Release)
		return withValidator, "", nil
	}
	say("check", "judged by the controller the mesh runs, with this change's validator")
	return withValidator, "", nil
}

// releaseJudge builds the controller that judges a pull request into a release branch (novox/hq ADR 0358 §2): the
// one beside the check as mesh-controller, read at that branch — with mesh-host's validator from beside it in place
// of the one it vendors when withHost. **Built once per release-branch head**: kept in the spec's judge cache by the
// commits it is built from and taken from there while they stand. It answers the binary and the directory it was
// built from.
func releaseJudge(ctx context.Context, run, plain Runner, spec CheckSpec, root string, withHost bool,
	inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) {
	const dir = "mesh-controller"
	if _, err := os.Stat(filepath.Join(root, dir)); err != nil {
		return "", "", fmt.Errorf("no controller of %s beside the check to judge it with", spec.Release)
	}
	key := fullHeadAt(ctx, plain, filepath.Join(root, dir))
	if key == "" {
		return "", "", fmt.Errorf("the controller of %s beside the check names no commit", spec.Release)
	}
	with := ""
	if withHost {
		host := fullHeadAt(ctx, plain, filepath.Join(root, "mesh-host"))
		if host == "" {
			return "", "", fmt.Errorf("mesh-host of %s beside the check names no commit", spec.Release)
		}
		key += "-with-mesh-host-" + host
		with = ", with mesh-host's validator of " + spec.Release + " at " + twelve(host)
	}
	bin := filepath.Join(root, "bin", "mesh-controller")
	if took := judgeKept(spec.JudgeCache, key, bin); took {
		say("check", "judged by %s's controller at %s%s, built for an earlier check of that head", spec.Release,
			twelve(key), with)
		return bin, dir, nil
	}
	if withHost {
		vendored := filepath.Join(root, dir, "vendor", "github.com", "novox", "mesh-host")
		for _, pkg := range []string{"validate", filepath.Join("internal", "declaration")} {
			if err := replaceGoFiles(filepath.Join(root, "mesh-host", pkg), filepath.Join(vendored, pkg)); err != nil {
				return "", "", fmt.Errorf("mesh-host's validator of %s could not be put in its controller: %w", spec.Release, err)
			}
		}
	}
	// Built where no pull request reaches (JudgeBuildEnv): a judge kept for this head is the head's, nothing else.
	if _, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), nil,
		"go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil {
		return "", "", fmt.Errorf("the controller of %s does not build%s: %w", spec.Release, with, err)
	}
	// Kept before anything of the change runs: nothing a pull request's script writes can reach the cache.
	keepJudge(spec.JudgeCache, key, bin, say)
	say("check", "judged by %s's controller at %s%s", spec.Release, twelve(key), with)
	return bin, dir, nil
}

// JudgeBuildEnv is the environment every judge is built in (novox/hq issue 482): home, a directory outside the
// workspace made fresh for the check, as HOME, configuration and Go cache; no global or system git configuration; no
// Go environment file; and no version stamped from git, so no git runs during the build at all.
func JudgeBuildEnv(home string) []string {
	return []string{"HOME=" + home, "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"),
		"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", "GOENV=off",
		"GOCACHE=" + filepath.Join(home, "go-cache"), "GOPATH=" + filepath.Join(home, "go"),
		"CGO_ENABLED=0", "GOFLAGS=-mod=vendor -buildvcs=false", "GOPROXY=off"}
}

// judgesKept is how many release-branch judges a judge cache keeps: a head moved past is not built from again.
const judgesKept = 4

// judgeKept puts the judge kept under key in the cache at bin, and says whether there was one.
func judgeKept(cache, key, bin string) bool {
	if cache == "" {
		return false
	}
	body, err := os.ReadFile(filepath.Join(cache, key, "mesh-controller"))
	if err != nil {
		return false
	}
	if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil {
		return false
	}
	if err := os.WriteFile(bin, body, 0o755); err != nil {
		return false
	}
	now := time.Now()
	_ = os.Chtimes(filepath.Join(cache, key), now, now)
	return true
}

// keepJudge keeps a judge just built under key, and lets go of all but the judgesKept most recently used. A judge
// that cannot be kept is said and built again next time: the cache only saves time.
func keepJudge(cache, key, bin string, say func(step, format string, args ...any)) {
	if cache == "" {
		return
	}
	body, err := os.ReadFile(bin)
	if err == nil {
		err = os.MkdirAll(filepath.Join(cache, key), 0o755)
	}
	if err == nil {
		kept := filepath.Join(cache, key, "mesh-controller")
		if err = os.WriteFile(kept+".part", body, 0o755); err == nil {
			err = os.Rename(kept+".part", kept)
		}
	}
	if err != nil {
		say("check", "the judge could not be kept for the next check, which builds it again: %v", err)
		return
	}
	entries, err := os.ReadDir(cache)
	if err != nil || len(entries) <= judgesKept {
		return
	}
	type used struct {
		name string
		at   time.Time
	}
	var all []used
	for _, e := range entries {
		if info, err := e.Info(); err == nil && e.IsDir() {
			all = append(all, used{e.Name(), info.ModTime()})
		}
	}
	sort.Slice(all, func(i, j int) bool { return all[i].at.After(all[j].at) })
	for _, u := range all[min(judgesKept, len(all)):] {
		if u.name != key {
			_ = os.RemoveAll(filepath.Join(cache, u.name))
		}
	}
}

// fullHeadAt is the whole commit a checkout stands at, empty when it names none.
func fullHeadAt(ctx context.Context, run Runner, tree string) string {
	out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", "HEAD^{commit}")
	if err != nil {
		return ""
	}
	return strings.TrimSpace(out)
}

// releaseValidator says which node-engine validator a release-branch check's judge is built with (novox/hq ADR 0358
// §2), by the rule judgeFor builds it by.
func releaseValidator(spec CheckSpec, at, commits map[string]string) string {
	switch {
	case spec.Judge == "self":
		return "the validator the change's own controller vendors"
	case spec.Judge == "validator":
		return "the change's own validator"
	case at["mesh-host"] == spec.Release:
		return fmt.Sprintf("mesh-host's validator of %s at %s", spec.Release, commits["mesh-host"])
	}
	return fmt.Sprintf("the validator %s's controller vendors (mesh-host has no branch %s)", spec.Release, spec.Release)
}

// releaseWords is how a release-branch check says what judged it and what it read (novox/hq ADR 0358 §4).
func releaseWords(release, judgedBy, validator string, at, commits map[string]string) string {
	var read []string
	for _, dir := range sortedKeys(at) {
		read = append(read, fmt.Sprintf("%s %s at %s", dir, refWords(at[dir]), commits[dir]))
	}
	said := fmt.Sprintf("judged for the release branch %s by its controller at %s, with %s", release, judgedBy, validator)
	if len(read) > 0 {
		said += "; read beside it: " + strings.Join(read, ", ")
	}
	return said
}

// releaseMark opens each layer's summary of a release-branch check, so a status says it first.
func releaseMark(release, judgedBy string) string {
	return fmt.Sprintf("[release %s, judged by its controller %s] ", release, judgedBy)
}

// refWords is a ref as a line says it: a commit by its first eight characters, a branch whole.
func refWords(ref string) string {
	if commitLike.MatchString(ref) {
		return short(ref)
	}
	return ref
}

// commitLike is a ref that names a commit.
var commitLike = regexp.MustCompile(`^[0-9a-f]{7,40}$`)

func sortedKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// replaceGoFiles puts a package's Go files — never its tests — in place of another copy's.
func replaceGoFiles(from, into string) error {
	old, err := filepath.Glob(filepath.Join(into, "*.go"))
	if err != nil {
		return err
	}
	for _, f := range old {
		if err := os.Remove(f); err != nil {
			return err
		}
	}
	if err := os.MkdirAll(into, 0o755); err != nil {
		return err
	}
	files, err := filepath.Glob(filepath.Join(from, "*.go"))
	if err != nil {
		return err
	}
	if len(files) == 0 {
		return fmt.Errorf("%s holds no Go files", from)
	}
	for _, f := range files {
		if strings.HasSuffix(f, "_test.go") {
			continue
		}
		body, err := os.ReadFile(f)
		if err != nil {
			return err
		}
		if err := os.WriteFile(filepath.Join(into, filepath.Base(f)), body, 0o644); err != nil {
			return err
		}
	}
	return nil
}

// problemLines are the lines of a module check's output that name a problem: not a manifest found ok, not
// the count, not the closing note.
func problemLines(s string) []string {
	var out []string
	for _, line := range strings.Split(s, "\n") {
		line = strings.TrimSpace(line)
		switch {
		case line == "", strings.Contains(line, ": ok"), strings.Contains(line, "problem(s) in"),
			strings.Contains(line, "manifest(s) checked"):
			continue
		}
		out = append(out, line)
	}
	return out
}

// newProblems are the problems a module check said of the change that it did not say of the base.
func newProblems(change, base string) []string {
	was := map[string]bool{}
	for _, p := range problemLines(base) {
		was[p] = true
	}
	var out []string
	for _, p := range problemLines(change) {
		if !was[p] {
			out = append(out, p)
		}
	}
	if len(out) == 0 && len(problemLines(change)) == 0 {
		// It failed and named nothing: not readable as already so.
		out = append(out, lastLine(change))
	}
	return out
}

// gateVerdict is what of the gate's verdict the seat reads: the verdict, and every machine — whether it
// composes on the mesh and whether it composed in the gate's store without the change.
type gateVerdict struct {
	Verdict  string `json:"verdict"`
	Summary  string `json:"summary"`
	Machines []struct {
		Described string `json:"described"`
		Live      bool   `json:"live-composes"`
		Base      struct {
			Composes bool `json:"composes"`
		} `json:"base"`
	} `json:"machines"`
}

// readGateVerdict reads the verdict the gate wrote, from its first line that opens a JSON document: a
// judge from before the verdict was alone on its output printed what composition said ahead of it.
func readGateVerdict(raw []byte) (gateVerdict, bool) {
	var v gateVerdict
	text := string(raw)
	if i := strings.Index(text, "\n{"); i >= 0 && !strings.HasPrefix(strings.TrimSpace(text), "{") {
		text = text[i+1:]
	}
	if json.Unmarshal([]byte(text), &v) != nil || v.Verdict == "" {
		return gateVerdict{}, false
	}
	return v, true
}

// gateRaisedTheMesh is the seat's own reading of a verdict that passes (novox/hq issue 285): **a machine
// the mesh composes that did not compose in the gate's store without the change makes the gate an error,
// never a pass** — the change was judged against a machine that is not the mesh's, broken against broken.
// The judge says so itself since issue 285, but the judge is the controller the mesh runs, and one from
// before it passed such a verdict; read here too, the rule holds whatever judged. It answers false, with
// the verdict to report, when the gate did not raise the mesh.
func gateRaisedTheMesh(v gateVerdict) (string, string, bool) {
	if v.Verdict != "pass" && v.Verdict != "warning" {
		return "", "", true
	}
	var unraised []string
	for _, m := range v.Machines {
		if m.Live && !m.Base.Composes {
			unraised = append(unraised, m.Described)
		}
	}
	if len(unraised) == 0 {
		return "", "", true
	}
	return "error", fmt.Sprintf("the mesh as it is could not be raised, so the change cannot be judged against it: "+
		"%d of %d machines compose on the mesh and not in the gate (the first: %s) — novox/hq issue 285",
		len(unraised), len(v.Machines), unraised[0]), false
}

// gitShow is a file as a ref has it.
func gitShow(ctx context.Context, dir, ref, file string) ([]byte, error) {
	cmd := exec.CommandContext(ctx, "git", "show", ref+":"+filepath.ToSlash(file))
	cmd.Dir = dir
	return cmd.Output()
}

func prefixed(prefix string, items []string) []string {
	out := make([]string, 0, len(items))
	for _, i := range items {
		out = append(out, prefix+i)
	}
	return out
}

func firstLine(s string) string {
	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
	return line
}

// safeName is a directory a repository beside a check may be cloned under.
var safeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)

// StoreImage is the store a check stands on: the major release the mesh's store runs (`17.11` → 17),
// on Alpine as the store module runs it.
func StoreImage(version string) string {
	major, _, _ := strings.Cut(strings.TrimSpace(version), ".")
	if major == "" || strings.ContainsAny(major, " (") {
		major = "17"
	}
	return "postgres:" + major + "-alpine"
}

// StoreSettings are the throwaway store's server settings: **no durability** (novox/hq issue 306). The
// store lives for one check and is removed after it, so nothing it writes has to survive a crash —
// and a test suite that creates a database per test asks the disk to flush thousands of times. On a
// machine whose disk the mesh's own store, bus and forge already keep busy — the control node — each
// flush waited its turn: the store-bound packages of the controller's suite ran five to seven times
// slower there than on any other holder, while those that touch no store ran alike, and its check ran
// past the suite's thirty minutes every time it landed there. The release stays the mesh's; only what
// a crash would need is switched off.
var StoreSettings = []string{"-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"}

// BusImage is the bus a check stands on: the release the mesh's bus server runs.
func BusImage(version string) string {
	v := strings.TrimPrefix(strings.TrimSpace(version), "v")
	if v == "" {
		v = "2.11"
	}
	return "nats:" + v + "-alpine"
}

// throwaway starts a container publishing one port on loopback, and answers where it is reached.
func throwaway(ctx context.Context, run, plain Runner, name, image string, port int, opts, args []string) (string, error) {
	invocation := []string{"run", "-d", "--rm", "--name", name, "-p", fmt.Sprintf("127.0.0.1::%d", port)}
	invocation = append(invocation, opts...)
	invocation = append(invocation, image)
	invocation = append(invocation, args...)
	if _, err := run(ctx, "", "docker", invocation...); err != nil {
		return "", fmt.Errorf("a throwaway %s could not be raised: %w", image, err)
	}
	out, err := plain(ctx, "", "docker", "port", name, fmt.Sprintf("%d/tcp", port))
	if err != nil {
		return "", err
	}
	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
		if strings.HasPrefix(line, "127.0.0.1:") {
			return strings.TrimSpace(line), nil
		}
	}
	return "", fmt.Errorf("%s published %d nowhere on loopback: %q", name, port, out)
}

// waitFor runs a readiness command in a container until it answers, for a minute.
func waitFor(ctx context.Context, run Runner, name string, ready []string) error {
	for i := 0; i < 60; i++ {
		if _, err := run(ctx, "", "docker", append([]string{"exec", name}, ready...)...); err == nil {
			return nil
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(time.Second):
		}
	}
	return fmt.Errorf("%s never became ready", name)
}

// dialable waits for an address to take a connection, for a minute.
func dialable(ctx context.Context, address string) error {
	for i := 0; i < 60; i++ {
		if c, err := net.DialTimeout("tcp", address, time.Second); err == nil {
			c.Close()
			return nil
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(time.Second):
		}
	}
	return fmt.Errorf("nothing took a connection at %s", address)
}

// judge builds the controller that judges a change: the one the mesh runs, beside the check as
// mesh-controller — or, when that one predates the merge gate, the controller's main, said. It answers
// the binary and the directory it was built from.
func judge(ctx context.Context, run, plain Runner, root string, inToolchain func(string, []string, ...string) []string,
	say func(step, format string, args ...any)) (string, string, error) {
	bin := filepath.Join(root, "bin", "mesh-controller")
	build := func(dir string) error {
		_, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), nil,
			"go", "build", "-o", bin, "./cmd/mesh-controller")...)
		return err
	}
	// Static, so it runs where the builder does.
	hasGate := func() bool {
		out, _ := plain(ctx, root, bin, "merge-gate")
		return strings.Contains(out, "merge-gate --facts")
	}
	if _, err := os.Stat(filepath.Join(root, "mesh-controller")); err == nil {
		if err := build("mesh-controller"); err != nil {
			return "", "", fmt.Errorf("the controller the mesh runs does not build: %w", err)
		}
		if hasGate() {
			say("check", "judged by the controller the mesh runs")
			return bin, "mesh-controller", nil
		}
	}
	if _, err := os.Stat(filepath.Join(root, "mesh-controller-main")); err != nil {
		return "", "", errors.New("no controller beside the check to judge it with")
	}
	if err := build("mesh-controller-main"); err != nil {
		return "", "", fmt.Errorf("the controller's main does not build: %w", err)
	}
	say("check", "judged by the controller's main: the one the mesh runs predates the merge gate")
	return bin, "mesh-controller-main", nil
}

// tail keeps the last lines written to it.
type tail struct{ buf bytes.Buffer }

func (t *tail) Write(p []byte) (int, error) {
	t.buf.Write(p)
	if t.buf.Len() > 1<<20 {
		keep := t.buf.Bytes()[t.buf.Len()-(512<<10):]
		t.buf = *bytes.NewBuffer(append([]byte(nil), keep...))
	}
	return len(p), nil
}

func (t *tail) String() string {
	lines := strings.Split(strings.TrimRight(t.buf.String(), "\n"), "\n")
	if len(lines) > reportLines {
		lines = lines[len(lines)-reportLines:]
	}
	return strings.Join(lines, "\n")
}

// whatFailed is the line of a failed script's output that says what failed, for the status a pull request
// shows: the first failing test, the first failing package, the files not formatted — a bare "FAIL" or a
// file's name said nothing a reader could act on (novox/hq issue 286) — and the last line otherwise.
func whatFailed(s string) string {
	lines := strings.Split(strings.TrimSpace(s), "\n")
	for i, line := range lines {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "not gofmt'd:") {
			var files []string
			for _, f := range lines[i+1:] {
				if f = strings.TrimSpace(f); f != "" {
					files = append(files, f)
				}
			}
			return "not gofmt'd by the toolchain's gofmt: " + strings.Join(files, ", ")
		}
	}
	// A panic, a data race or a build error outside any test says more than the package's bare FAIL line
	// (novox/hq issue 460).
	for _, said := range []func(string) bool{
		func(l string) bool { return strings.HasPrefix(l, "--- FAIL:") },
		func(l string) bool { return strings.HasPrefix(l, "panic:") || strings.HasPrefix(l, "fatal error:") },
		func(l string) bool { return l == "WARNING: DATA RACE" },
		goError.MatchString,
		func(l string) bool { return strings.HasPrefix(l, "FAIL\t") },
	} {
		for _, line := range lines {
			if line = strings.TrimSpace(line); said(line) {
				return line
			}
		}
	}
	return lastLine(s)
}

func lastLine(s string) string {
	lines := strings.Split(strings.TrimSpace(s), "\n")
	return strings.TrimSpace(lines[len(lines)-1])
}