mesh-controller/internal/builder/check_test.go

package builder

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/artifacts"
	"github.com/novox/mesh-controller/internal/facts"
)

// A merge check on the build seat (novox/hq to-be 45 §9), against a real container runtime and a real
// registry: the repository's own merge-check.sh runs with the facts the controller keeps, beside a
// throwaway store and bus of **the versions the mesh runs**, and everything raised is removed.
//
//	MESH_TEST_DOCKER=1 MESH_TEST_REGISTRY=127.0.0.1:15000 go test ./internal/builder -run Check

func TestTheStoreAndBusAChecksStandsOnAreTheOnesTheMeshRuns(t *testing.T) {
	if got := StoreImage("17.11"); got != "postgres:17-alpine" {
		t.Errorf("a store at 17.11 is checked against %s", got)
	}
	if got := StoreImage("16.4 (Debian 16.4-1.pgdg120+1)"); got != "postgres:16-alpine" {
		t.Errorf("a store at 16.4 is checked against %s", got)
	}
	if got := BusImage("2.11.17"); got != "nats:2.11.17-alpine" {
		t.Errorf("a bus at 2.11.17 is checked against %s", got)
	}
}

// aRepository is a git repository holding these files, committed, and its head.
func aCheckedRepository(t *testing.T, files map[string]string) (string, string) {
	t.Helper()
	dir := t.TempDir()
	git := func(args ...string) string {
		cmd := exec.Command("git", args...)
		cmd.Dir = dir
		cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
			"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Fatalf("git %v: %v\n%s", args, err, out)
		}
		return strings.TrimSpace(string(out))
	}
	git("init", "--quiet", "-b", "main")
	for name, body := range files {
		if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
			t.Fatal(err)
		}
	}
	git("add", "-A")
	git("commit", "--quiet", "-m", "x")
	return dir, git("rev-parse", "HEAD")
}

func checkEnvironment(t *testing.T) string {
	t.Helper()
	if os.Getenv("MESH_TEST_DOCKER") != "1" || os.Getenv("MESH_TEST_REGISTRY") == "" {
		t.Skip("MESH_TEST_DOCKER=1 and MESH_TEST_REGISTRY: a check raises containers and reads the registry")
	}
	registry := os.Getenv("MESH_TEST_REGISTRY")
	body, err := json.Marshal(facts.Facts{Format: facts.Format, Taken: time.Now().UTC(),
		Versions: facts.Versions{Bus: "2.11.17", Store: "17.11"},
		Machines: []facts.Machine{{Name: "abcdef", Length: 6}}})
	if err != nil {
		t.Fatal(err)
	}
	if _, err := (artifacts.Store{Address: registry}).PutTagged(t.Context(), facts.Repository, facts.Tag,
		facts.MediaType, body); err != nil {
		t.Fatal(err)
	}
	return registry
}

// goToolchain is the Go image a check's script runs in here: the base the controller's own image is built on.
const goToolchain = "golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c"

func labelled(id string) []string {
	out, _ := exec.Command("docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id).Output()
	return strings.Fields(string(out))
}

func TestACheckRunsTheRepositorysOwnScriptBesideTheMeshsVersionsAndLeavesNothing(t *testing.T) {
	registry := checkEnvironment(t)
	// The script proves what it was given: the facts, a store that answers, a bus that answers — and no
	// container runtime socket.
	script := `set -e
test -s "$MESH_FACTS"
grep -q '"bus": "2.11.17"' "$MESH_FACTS" || grep -q '"bus":"2.11.17"' "$MESH_FACTS"
case "$MESH_TEST_POSTGRES" in postgres://*127.0.0.1:*) ;; *) echo "no store: $MESH_TEST_POSTGRES"; exit 1;; esac
case "$MESH_TEST_NATS" in nats://127.0.0.1:*) ;; *) echo "no bus: $MESH_TEST_NATS"; exit 1;; esac
test "$MESH_CHECK_REPOSITORY" = "novox/hq"
test "$MESH_CHECK_CHANGED" = "a.go,b.go"
test ! -S /var/run/docker.sock || { echo "the check holds the container runtime's socket"; exit 1; }
echo checked
`
	repo, head := aCheckedRepository(t, map[string]string{CheckScript: script})
	id := fmt.Sprintf("check-test-%d", time.Now().UnixNano())
	v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox",
		Repo: "hq", Number: 7, Paths: []string{"a.go", "b.go"}, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
	if err != nil {
		t.Fatal(err)
	}
	// Nothing of the graph touched: the gate a pass that says so, the repository's own check run.
	if v.Gate == nil || v.Gate.Verdict != "pass" || v.Gate.Summary != noModule {
		t.Errorf("the gate answered %+v", v.Gate)
	}
	if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Report, "checked") {
		t.Fatalf("the repository's check answered %+v\n%s", v.Repo, v.Report)
	}
	if left := labelled(id); len(left) > 0 {
		t.Errorf("the check left %d container(s) behind", len(left))
	}
}

func TestAFailingCheckFailsAndOneThatCannotRunIsNeverAPass(t *testing.T) {
	registry := checkEnvironment(t)
	repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo 'resource \"x.service\": refused'; exit 3\n"})
	v, err := Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-fail-%d", time.Now().UnixNano()),
		Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
	if err != nil {
		t.Fatal(err)
	}
	if v.Repo == nil || v.Repo.Verdict != "fail" || !strings.Contains(v.Repo.Summary, "refused") {
		t.Fatalf("a failing script answered %+v", v.Repo)
	}

	// A script in a toolchain the mesh does not hold: an error, never a pass.
	repo, head = aCheckedRepository(t, map[string]string{CheckScript: "# mesh-check-toolchain: cobol\nexit 0\n"})
	v, err = Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-cobol-%d", time.Now().UnixNano()),
		Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
	if err != nil || v.Repo == nil || v.Repo.Verdict != "error" {
		t.Fatalf("a script in a toolchain nobody holds answered %+v, %v", v.Repo, err)
	}

	// Past its bound: an error, not a pass.
	was := CheckTimeout
	CheckTimeout = 25 * time.Second
	t.Cleanup(func() { CheckTimeout = was })
	repo, head = aCheckedRepository(t, map[string]string{CheckScript: "sleep 120\n"})
	v, err = Check(context.Background(), Command, CheckSpec{ID: fmt.Sprintf("check-slow-%d", time.Now().UnixNano()),
		Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
	if err == nil && (v.Repo == nil || v.Repo.Verdict != "error") {
		t.Fatalf("a check past its bound answered %+v", v.Repo)
	}

	// No facts: it cannot run, and says so.
	repo, head = aCheckedRepository(t, map[string]string{CheckScript: "exit 0\n"})
	_, err = Check(t.Context(), Command, CheckSpec{ID: "check-nofacts", Repository: repo, Ref: head, Owner: "novox",
		Repo: "hq", Toolchain: goToolchain}, t.TempDir(), "127.0.0.1:1", GitCredential{}, nil)
	if err == nil || !strings.Contains(err.Error(), "facts snapshot") {
		t.Fatalf("a check with no facts said %v", err)
	}
}

// A repository that touches nothing of the graph and has no script of its own runs nothing, and says
// both: the gate a pass that names why, the repository a warning — never silent.
func TestACheckWithNothingToRunSaysSo(t *testing.T) {
	repo, head := aCheckedRepository(t, map[string]string{"README.md": "x"})
	v, err := Check(t.Context(), Command, CheckSpec{ID: "check-nothing", Repository: repo, Ref: head, Owner: "novox",
		Repo: "hq"}, t.TempDir(), "", GitCredential{}, nil)
	if err != nil {
		t.Fatal(err)
	}
	if v.Gate == nil || v.Gate.Verdict != "pass" || v.Repo == nil || v.Repo.Verdict != "warning" ||
		!strings.Contains(v.Repo.Summary, CheckScript) {
		t.Fatalf("a check with nothing to run said %+v / %+v", v.Gate, v.Repo)
	}
	// A gated change never takes that path: with no store to read the facts from, it cannot run.
	_, err = Check(t.Context(), Command, CheckSpec{ID: "check-gated", Repository: repo, Ref: head, Owner: "novox",
		Repo: "mesh-catalog", Modules: []string{"gitea"}}, t.TempDir(), "", GitCredential{}, nil)
	if err == nil {
		t.Fatal("a change touching a module ran nothing and was not refused")
	}
}

// A script declares its toolchain among its first lines; go when it declares none.
func TestAScriptDeclaresItsToolchain(t *testing.T) {
	for script, want := range map[string]string{
		"#!/bin/sh\nset -eu\n": "go",
		"#!/bin/sh\n# mesh-check-toolchain: typescript\nnpm test\n": "typescript",
		"#!/bin/sh\n#mesh-check-toolchain:go\n":                     "go",
		"#!/bin/sh\necho '# mesh-check-toolchain: python'\n":        "go",
	} {
		if got := ScriptToolchain([]byte(script)); got != want {
			t.Errorf("%q declares %q, read as %q", script, want, got)
		}
	}
	held := map[string]string{"mesh-tools/build": "reg/mesh-tools-build@sha256:a", "mesh-tools-go/build": "reg/go@sha256:b"}
	chains := ToolchainsOf(held)
	if chains["typescript"] != "reg/mesh-tools-build@sha256:a" || chains["go"] != "reg/go@sha256:b" || ToolchainOf(held) != chains["go"] {
		t.Fatalf("the toolchains held read as %v", chains)
	}
	if _, held := chains["python"]; held {
		t.Error("a toolchain the mesh does not hold read as held")
	}
}

// A node-engine change's validator is put in place of the one the judge vendors: its Go files, never its tests.
func TestTheChangesValidatorReplacesTheVendoredOne(t *testing.T) {
	from, into := t.TempDir(), t.TempDir()
	for name, body := range map[string]string{"v.go": "package validate // new", "v_test.go": "package validate"} {
		if err := os.WriteFile(filepath.Join(from, name), []byte(body), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	if err := os.WriteFile(filepath.Join(into, "old.go"), []byte("package validate // old"), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := replaceGoFiles(from, into); err != nil {
		t.Fatal(err)
	}
	got, _ := filepath.Glob(filepath.Join(into, "*.go"))
	if len(got) != 1 || filepath.Base(got[0]) != "v.go" {
		t.Fatalf("the vendored validator holds %v", got)
	}
}

// aJudge is a controller that judges as told: `merge-gate` answers a warning, `module check` refuses a
// manifest that says it is broken. What the gate layer is tested against without building the real one.
var aJudge = map[string]string{
	"go.mod": "module example.org/judge\n\ngo 1.22\n",
	"cmd/mesh-controller/main.go": `package main

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
)

func main() {
	switch {
	case len(os.Args) == 2 && os.Args[1] == "merge-gate":
		fmt.Println("usage: merge-gate --facts <file> --store <postgres>")
		os.Exit(2)
	case len(os.Args) > 2 && os.Args[1] == "module":
		for _, m := range os.Args[3:] {
			body, _ := os.ReadFile(m)
			if strings.Contains(string(body), "broken") {
				fmt.Println(m + ": refused, it says it is broken")
				os.Exit(1)
			}
			fmt.Println(m + ": ok")
		}
	case os.Args[1] == "merge-gate":
		for i, a := range os.Args {
			if a == "--group" && i+1 < len(os.Args) {
				body, _ := os.ReadFile(os.Args[i+1])
				var heads []struct {
					Repository string ` + "`json:\"repository\"`" + `
					Tree       string ` + "`json:\"tree\"`" + `
				}
				_ = json.Unmarshal(body, &heads)
				var said []string
				for _, h := range heads {
					if _, err := os.Stat(h.Tree + "/module.json"); err == nil {
						said = append(said, h.Repository)
					}
				}
				fmt.Printf("{\"verdict\":\"pass\",\"summary\":\"composed with %s\"}\n", strings.Join(said, ","))
				return
			}
		}
		fmt.Println(` + "`" + `{"verdict":"warning","summary":"a merge rebuilds 14 module(s)"}` + "`" + `)
	}
}
`,
}

func TestTheGateRunsWhenTheGraphIsTouchedBesideTheRepositorysOwnCheck(t *testing.T) {
	registry := checkEnvironment(t)
	judgeRepo, judgeHead := aCheckedRepository(t, aJudge)
	beside := map[string]Beside{"mesh-controller": {Repository: judgeRepo, Ref: judgeHead}}
	check := func(files map[string]string, judge string) CheckVerdict {
		t.Helper()
		repo, head := aCheckedRepository(t, files)
		id := fmt.Sprintf("check-gate-%d", time.Now().UnixNano())
		v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox",
			Repo: "mesh-catalog", Number: 9, Paths: []string{"modules/gitea/index.ts"}, Beside: beside,
			Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Judge: judge,
			Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
		if err != nil {
			t.Fatal(err)
		}
		if left := labelled(id); len(left) > 0 {
			t.Errorf("the check left %d container(s) behind", len(left))
		}
		return v
	}
	v := check(map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "echo own; exit 0\n"}, "")
	if v.Gate == nil || v.Gate.Verdict != "warning" || v.Gate.Summary != "a merge rebuilds 14 module(s)" ||
		strings.Join(v.Gate.Modules, ",") != "gitea" || v.Verdict != "warning" {
		t.Fatalf("the gate answered %+v\n%s", v.Gate, v.Report)
	}
	if v.Repo == nil || v.Repo.Verdict != "pass" {
		t.Fatalf("its own check answered %+v", v.Repo)
	}

	v = check(map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}, "")
	if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "module check") || v.Repo.Verdict != "warning" {
		t.Fatalf("a manifest the judge refuses answered %+v / %+v\n%s", v.Gate, v.Repo, v.Report)
	}

	// A fault the base branch already had is said and fails nothing; one the change brings fails.
	repo, _ := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`})
	git := func(args ...string) string {
		cmd := exec.Command("git", args...)
		cmd.Dir = repo
		cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
			"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Fatalf("git %v: %v\n%s", args, err, out)
		}
		return strings.TrimSpace(string(out))
	}
	git("checkout", "--quiet", "-b", "change")
	if err := os.WriteFile(filepath.Join(repo, "modules/gitea/index.ts"), []byte("x"), 0o644); err != nil {
		t.Fatal(err)
	}
	git("add", "-A")
	git("commit", "--quiet", "-m", "y")
	id := fmt.Sprintf("check-base-%d", time.Now().UnixNano())
	v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: git("rev-parse", "HEAD"), Owner: "novox",
		Repo: "mesh-catalog", Base: "main", Paths: []string{"modules/gitea/index.ts"}, Beside: beside,
		Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain},
		t.TempDir(), registry, GitCredential{}, nil)
	if err != nil {
		t.Fatal(err)
	}
	if v.Gate.Verdict != "warning" || !strings.Contains(v.Report, "already") {
		t.Fatalf("a fault the base already had failed the change: %+v\n%s", v.Gate, v.Report)
	}

	// A delivery group (novox/hq ADR 0239): the other heads cloned beside it at their heads and handed to the
	// gate to compose with this one; the repository's own check is each pull request's, not the group's.
	app, appHead := aCheckedRepository(t, map[string]string{"module.json": `{"module":"app"}`})
	repo2, head2 := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
		CheckScript: "exit 1\n"})
	gid := fmt.Sprintf("check-group-%d", time.Now().UnixNano())
	v, err = Check(t.Context(), Command, CheckSpec{ID: gid, Repository: repo2, Ref: head2, Owner: "novox",
		Repo: "mesh-catalog", Paths: []string{"modules/gitea/module.json"}, Beside: beside, Modules: []string{"gitea"},
		Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain,
		Group: []GroupHead{{Owner: "novox", Repo: "app", Repository: app, Ref: appHead, Paths: []string{"module.json"}}}},
		t.TempDir(), registry, GitCredential{}, nil)
	if err != nil {
		t.Fatal(err)
	}
	if v.Gate.Verdict != "pass" || v.Gate.Summary != "composed with novox/app" {
		t.Fatalf("the group's other head was not composed: %+v\n%s", v.Gate, v.Report)
	}
	if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Repo.Summary, "group") {
		t.Fatalf("a group's check ran a member's own script: %+v", v.Repo)
	}

	// A change to the controller judges itself: one that does not build fails its own gate.
	v = check(map[string]string{"go.mod": "module x\n\ngo 1.22\n", "cmd/mesh-controller/main.go": "package main\nfunc main() { nope }\n",
		"modules/gitea/module.json": "{}"}, "self")
	if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "does not build") {
		t.Fatalf("a controller that does not build judged itself %+v", v.Gate)
	}
}

// **Only a commit on the trunk is published** (novox/hq ADR 0238): the build seat reads, from the clone it
// just made, the branch the forge names its default and whether the commit built is on it.
func TestABuildSaysWhetherItsCommitIsOnTheTrunk(t *testing.T) {
	repo, onMain := aCheckedRepository(t, map[string]string{"module.json": `{"module":"x","version":"1"}`})
	git := func(dir string, args ...string) string {
		cmd := exec.Command("git", args...)
		cmd.Dir = dir
		cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
			"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Fatalf("git %v: %v\n%s", args, err, out)
		}
		return strings.TrimSpace(string(out))
	}
	git(repo, "checkout", "--quiet", "-b", "feature")
	if err := os.WriteFile(filepath.Join(repo, "x"), []byte("x"), 0o644); err != nil {
		t.Fatal(err)
	}
	git(repo, "add", "-A")
	git(repo, "commit", "--quiet", "-m", "off the trunk")
	offMain := git(repo, "rev-parse", "HEAD")
	git(repo, "checkout", "--quiet", "main")

	clone := filepath.Join(t.TempDir(), "clone")
	git(filepath.Dir(clone), "clone", "--quiet", repo, clone)
	if trunk, on := trunkOf(t.Context(), Command, clone, onMain); trunk != "main" || !on {
		t.Errorf("a commit on main reads as on %q: %v", trunk, on)
	}
	if trunk, on := trunkOf(t.Context(), Command, clone, offMain); trunk != "main" || on {
		t.Errorf("a feature branch's commit reads as on %q: %v", trunk, on)
	}
	if got := strings.Join(branchesHolding(t.Context(), Command, clone, offMain), ","); got != "feature" {
		t.Errorf("the feature branch's commit is said to be on %q", got)
	}
	if got := strings.Join(branchesHolding(t.Context(), Command, clone, onMain), ","); got != "feature,main" {
		t.Errorf("main's commit is said to be on %q", got)
	}
	// A tree that says no trunk is not known — never read as on it.
	if trunk, on := trunkOf(t.Context(), Command, repo, onMain); trunk != "" || on {
		t.Errorf("a repository with no origin reads as trunk %q, on %v", trunk, on)
	}
}

// **Issue 283**: a failed merge-check.sh was said by its last line — a bare "FAIL", or the name of a file
// gofmt listed — which named nothing a reader could act on. The status says what failed.
func TestAFailedScriptIsSaidByWhatFailed(t *testing.T) {
	for out, want := range map[string]string{
		"not gofmt'd:\ninternal/bootstrap/publish_test.go\n":                                                                 "not gofmt'd by the toolchain's gofmt: internal/bootstrap/publish_test.go",
		"ok  \tx/a\t1s\n--- FAIL: TestTheInstallersFirstUserList (0.59s)\n    t.go:37: refused\nFAIL\nFAIL\tx/b\t1s\nFAIL\n": "--- FAIL: TestTheInstallersFirstUserList (0.59s)",
		"ok  \tx/a\t1s\nFAIL\tx/b [build failed]\nFAIL\n":                                                                    "FAIL\tx/b [build failed]",
		"npm ERR! missing script: typecheck\n":                                                                               "npm ERR! missing script: typecheck",
	} {
		if got := whatFailed(out); got != want {
			t.Errorf("%q is said as %q, not %q", out, got, want)
		}
	}
}

// **Issue 285**: a judge from before the rule passed "every machine composes with the change as it did
// without (0 of 4 compose)". The seat reads the machines itself: a machine the mesh composes that did not
// compose in the gate's store makes the gate an error, whatever judged it.
func TestTheSeatCallsAGateThatRaisedNoMachineAnError(t *testing.T) {
	old := "bus users without a credential: controller\n" + `{"verdict":"pass","summary":"every machine composes with the change as it did without (0 of 2 compose)",
		"machines":[{"described":"the hub","live-composes":true,"base":{"composes":false}},
		{"described":"a machine","live-composes":true,"base":{"composes":false}}]}`
	v, ok := readGateVerdict([]byte(old))
	if !ok {
		t.Fatal("a verdict after a line of composition's was not read")
	}
	verdict, summary, raised := gateRaisedTheMesh(v)
	if raised || verdict != "error" || !strings.Contains(summary, "2 of 2 machines") || !strings.Contains(summary, "the hub") {
		t.Errorf("0 of 2 composing is %q %q", verdict, summary)
	}
	good := `{"verdict":"pass","summary":"(2 of 2 compose)","machines":[{"described":"the hub","live-composes":true,"base":{"composes":true}},
		{"described":"a laptop","live-composes":false,"base":{"composes":false}}]}`
	v, _ = readGateVerdict([]byte(good))
	if _, _, raised := gateRaisedTheMesh(v); !raised {
		t.Error("a machine that does not compose on the mesh either was read as the gate's failure")
	}
	failed := `{"verdict":"fail","summary":"x","machines":[{"described":"the hub","live-composes":true,"base":{"composes":false}}]}`
	v, _ = readGateVerdict([]byte(failed))
	if _, _, raised := gateRaisedTheMesh(v); !raised {
		t.Error("a failing verdict is the change's, and stands")
	}
}

// **Issue 285**: an ask redelivered after its holder stopped mid-check found its own throwaway store still
// there under its name, and the check said it could not run. What an earlier delivery left goes first.
func TestARedeliveredCheckRemovesWhatItsEarlierDeliveryLeft(t *testing.T) {
	registry := checkEnvironment(t)
	repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"})
	id := fmt.Sprintf("check-again-%d", time.Now().UnixNano())
	if out, err := exec.Command("docker", "run", "-d", "--rm", "--label", BuildLabel+"="+id, "--name", id+"-store",
		"postgres:17-alpine", "sleep", "300").CombinedOutput(); err != nil {
		t.Skipf("no container for the earlier delivery: %v %s", err, out)
	}
	v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox",
		Repo: "hq", Number: 7, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
	if err != nil {
		t.Fatalf("a redelivered check could not run: %v", err)
	}
	if v.Repo == nil || v.Repo.Verdict != "pass" {
		t.Errorf("the repository's check answered %+v", v.Repo)
	}
	if left := labelled(id); len(left) > 0 {
		t.Errorf("the check left %d container(s) behind", len(left))
	}
}

// **A repository in two languages checks both, each in its own toolchain** (novox/hq issue 302): the lab's
// merge-check.sh runs in the TypeScript toolchain, and its Go replays were said "NOT CHECKED HERE" on every
// pull request, so a register that did not compile would have merged green. Its script declares the Go
// part; the check runs it in the Go toolchain's container after the script, and the layer passes only
// when both do.
func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) {
	script := "#!/bin/sh\n# mesh-check-toolchain: typescript\n# mesh-check-also: go replays/merge-check.sh\nnpm test\n"
	parts := ScriptParts([]byte(script))
	if len(parts) != 2 || parts[0] != (ScriptPart{"typescript", CheckScript}) ||
		parts[1] != (ScriptPart{"go", "replays/merge-check.sh"}) {
		t.Fatalf("the parts read as %+v", parts)
	}
	if got := ScriptParts([]byte("#!/bin/sh\nset -eu\n")); len(got) != 1 || got[0] != (ScriptPart{"go", CheckScript}) {
		t.Fatalf("a script declaring nothing reads as %+v", got)
	}

	// Each part run where its toolchain is: here, the image the part was given is what the command says.
	held := CheckSpec{Toolchain: "go-image", Toolchains: map[string]string{"typescript": "ts-image", "go": "go-image"}}
	run := func(t *testing.T, spec CheckSpec, files map[string]string, parts []ScriptPart) (*Layer, []string) {
		t.Helper()
		tree := t.TempDir()
		for name, body := range files {
			if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(filepath.Join(tree, name), []byte(body), 0o755); err != nil {
				t.Fatal(err)
			}
		}
		var images []string
		var out tail
		layer := ownCheck(t.Context(), spec, parts, tree, &out, func() bool { return false },
			func(image, script string) *exec.Cmd {
				images = append(images, image+" "+script)
				cmd := exec.CommandContext(t.Context(), "sh", script)
				cmd.Dir = tree
				return cmd
			}, func(string, string, ...any) {})
		return layer, images
	}
	twoParts := []ScriptPart{{"typescript", CheckScript}, {"go", "replays/merge-check.sh"}}

	layer, images := run(t, held, map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts)
	if layer.Verdict != "pass" || !strings.Contains(layer.Summary, "replays/merge-check.sh (go)") ||
		strings.Join(images, ", ") != "ts-image merge-check.sh, go-image replays/merge-check.sh" {
		t.Errorf("both parts passing answered %+v, ran %v", layer, images)
	}
	layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n",
		"replays/merge-check.sh": "echo 'not gofmt'\"'\"'d:'; echo register.go; exit 1\n"}, twoParts)
	if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "replays/merge-check.sh failed") ||
		!strings.Contains(layer.Summary, "register.go") || !strings.Contains(layer.Summary, "merge-check.sh (typescript) passed") {
		t.Errorf("a failing Go part answered %+v", layer)
	}
	layer, images = run(t, held, map[string]string{CheckScript: "exit 2\n", "replays/merge-check.sh": "exit 0\n"}, twoParts)
	if layer.Verdict != "fail" || len(images) != 1 {
		t.Errorf("a failing first part answered %+v and ran %v", layer, images)
	}
	layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, twoParts)
	if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "does not hold") {
		t.Errorf("a declared part the repository lacks answered %+v", layer)
	}
	layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, []ScriptPart{{"typescript", CheckScript},
		{"go", "../elsewhere.sh"}})
	if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "not a path inside") {
		t.Errorf("a part outside the repository answered %+v", layer)
	}
	layer, _ = run(t, CheckSpec{Toolchains: map[string]string{"typescript": "ts-image"}},
		map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts)
	if layer.Verdict != "error" || !strings.Contains(layer.Summary, "go toolchain") {
		t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer)
	}
}