Compile the workspace's own mesh-build-go and run digest and test as it under runner workspace, so a change to this module is judged by its candidate (issue 1898 K1, K2)

The checker asks runner workspace when a merge changes modules/mesh-build-go; the step's
shut-off process now takes a step spec (digest, unit, integration) and every answer says
judged_by. ADR 0535 §1, §3, §5.

modules/mesh-build-go/README.md

@@ -105,6 +105,16 @@ fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and th
 version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors
 is listed and tested from its vendor directory (`-mod=vendor`).
 
+## A change to this module is judged by its candidate
+
+`build_go_digest` and `build_go_test` take `runner`: `installed` (the default) runs the step as this program;
+`workspace` compiles the workspace's own `modules/mesh-build-go` (`go build` with `GOPROXY=off`, `GOTOOLCHAIN=local`, in a
+run step shut off) into the workspace's `candidate/mesh-build-go`, once per workspace, and runs the step as that program:
+`<candidate> level <spec>` in one bubblewrap, the spec naming the step (`digest`, `unit` or `integration`). This program
+keeps the fetch, the forge key and the bus login (ADR 0535 §1). A candidate that does not compile fails the step with the
+compiler's first error, and nothing runs (§3). Every digest and level answer says `judged_by`: `installed`, or
+`candidate <commit of the tree>`. The checker asks `workspace` when a merge changes `modules/mesh-build-go/` (issue 1898).
+
 ## Build
 
 `build_go_build {workspace, module, artifact}` builds one Go bundle artifact of a module of the workspace with the build

modules/mesh-build-go/cmd/mesh-build-go/candidate.go

@@ -0,0 +1,132 @@
+package main
+
+// A change to this module is judged by its own candidate (novox/hq ADR 0535; issue 1898): asked with the runner
+// workspace, the digest and test steps compile the program of mesh-build-go from the workspace's merged tree in a run
+// step, shut off, and run the step as that program in one shut-off process (`<candidate> level <spec>`, level.go), where
+// this program would run the step's commands. This program keeps the fetch, the forge key and the bus login; the
+// candidate's program holds none of them. Each answer says which program ran it (judged_by).
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+// The runners of a digest or test step: this installed program, or the candidate's compiled from the workspace.
+const (
+	RunnerInstalled = "installed"
+	RunnerWorkspace = "workspace"
+)
+
+// runnerProperty is the runner argument as the digest and test tools announce it: a checker reads it there, and an
+// installed module that does not announce it judges as it always did (ADR 0535 §2).
+var runnerProperty = map[string]any{"type": "string", "enum": []string{RunnerInstalled, RunnerWorkspace},
+	"description": "optional: installed (the default) runs the step as this installed program; workspace compiles the " +
+		"workspace's own mesh-build-go (modules/mesh-build-go of the merged tree) shut off and runs the step as it, for a " +
+		"change to this module (ADR 0535); the answer's judged_by names the program that ran"}
+
+// candidateModule is the directory of this module's go.mod in the catalogue's tree, and candidateMain its program's
+// package there.
+const (
+	candidateModule = "modules/mesh-build-go"
+	candidateMain   = "./cmd/mesh-build-go"
+)
+
+// runnerArg reads the runner argument: "" (the installed program), installed or workspace.
+func runnerArg(v any) (string, error) {
+	if v == nil {
+		return "", nil
+	}
+	s, ok := v.(string)
+	if !ok || (s != "" && s != RunnerInstalled && s != RunnerWorkspace) {
+		return "", fmt.Errorf("the runner %v is neither %s nor %s", v, RunnerInstalled, RunnerWorkspace)
+	}
+	return s, nil
+}
+
+// workspaceThere refuses a workspace id that is not one, or a workspace this node never fetched.
+func (m *Module) workspaceThere(workspace string) error {
+	if !workspaceOK.MatchString(workspace) {
+		return fmt.Errorf("%q is not a workspace id", workspace)
+	}
+	if _, err := os.Stat(filepath.Join(m.Work, workspace, "tree")); err != nil {
+		return fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
+	}
+	return nil
+}
+
+// candidateLocks hold one mutex per workspace, so its candidate is compiled once.
+var candidateLocks sync.Map
+
+// candidate is the program of mesh-build-go compiled from the workspace's tree, in the workspace beside its tree
+// (candidate/mesh-build-go), compiled once per workspace, in a run step shut off with the caches read-only and nothing
+// fetched (GOPROXY=off: the fetch step downloaded every module the tree's go.mod files name). A candidate that does not
+// compile is the step's failure, naming the compiler's first error (ADR 0535 §3).
+func (m *Module) candidate(ctx context.Context, workspace string) (string, error) {
+	exe := filepath.Join(m.Work, workspace, "candidate", "mesh-build-go")
+	mu, _ := candidateLocks.LoadOrStore(exe, &sync.Mutex{})
+	mu.(*sync.Mutex).Lock()
+	defer mu.(*sync.Mutex).Unlock()
+	if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
+		return exe, nil
+	}
+	dir := filepath.Join(m.Work, workspace, "tree", candidateModule)
+	if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
+		return "", fmt.Errorf("the workspace holds no %s/go.mod, so it has no candidate mesh-build-go to run", candidateModule)
+	}
+	if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil {
+		return "", err
+	}
+	run, err := m.stepRunner(workspace, false)
+	if err != nil {
+		return "", err
+	}
+	env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local")
+	r := run(ctx, env, "go", "-C", dir, "build", "-o", exe, candidateMain)
+	if r.Status != 0 || r.Err != "" {
+		return "", fmt.Errorf("the candidate's mesh-build-go does not compile: %s", compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
+	}
+	return exe, nil
+}
+
+// compileError is the first line of the compiler's output that says an error: not the `# package` heading go build
+// prints above a package's errors, nor Go's progress.
+func compileError(out string) string {
+	for _, l := range strings.Split(out, "\n") {
+		if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") && !strings.HasPrefix(l, "go: downloading ") {
+			return l
+		}
+	}
+	return firstLine(out)
+}
+
+// mergeCommit is the commit the workspace's tree holds: the merge the fetch made, or the head it fetched.
+func (m *Module) mergeCommit(ctx context.Context, workspace string) (string, error) {
+	r := m.Run(ctx, nil, "git", "-C", filepath.Join(m.Work, workspace, "tree"), "rev-parse", "HEAD")
+	commit := strings.TrimSpace(r.Stdout)
+	if r.Status != 0 || r.Err != "" || commit == "" {
+		return "", fmt.Errorf("the workspace's commit: git rev-parse: %s", firstLine(r.Err+"\n"+r.Stderr))
+	}
+	return commit, nil
+}
+
+// asCandidate runs the spec's step as the candidate's program, shut off, and answers what it said and the judged_by of
+// the answer: "candidate <merge commit>".
+func (m *Module) asCandidate(ctx context.Context, spec levelSpec) (levelAnswer, string, error) {
+	commit, err := m.mergeCommit(ctx, spec.Workspace)
+	if err != nil {
+		return levelAnswer{}, "", err
+	}
+	exe, err := m.candidate(ctx, spec.Workspace)
+	if err != nil {
+		return levelAnswer{}, "", err
+	}
+	a, err := m.shutOff(ctx, exe, spec)
+	if err != nil {
+		return levelAnswer{}, "", fmt.Errorf("the candidate's mesh-build-go: %w", err)
+	}
+	return a, "candidate " + commit, nil
+}

modules/mesh-build-go/cmd/mesh-build-go/candidate_integration_test.go

@@ -0,0 +1,127 @@
+//go:build integration
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// candidateMainSource is a candidate's program that answers a mark: asked `level <spec>`, it tries what a run step must
+// not reach (the account's key and login, a name, a socket of the machine) and answers a passing level of 42 tests whose
+// why is "mark" and what it reached.
+const candidateMainSource = `package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"time"
+)
+
+func main() {
+	if len(os.Args) != 3 || os.Args[1] != "level" {
+		os.Exit(2)
+	}
+	seen := map[string]bool{}
+	_, err := os.ReadFile(%q)
+	seen["key"] = err == nil
+	_, err = os.ReadFile(%q)
+	seen["login"] = err == nil
+	_, err = net.LookupHost("git.novox.be")
+	seen["resolved"] = err == nil
+	c, err := net.DialTimeout("tcp", %q, 2*time.Second)
+	if err == nil {
+		c.Close()
+	}
+	seen["tcp"] = err == nil
+	raw, _ := json.Marshal(seen)
+	var spec struct {
+		Step string ` + "`json:\"step\"`" + `
+	}
+	json.Unmarshal([]byte(os.Args[2]), &spec)
+	out, _ := json.Marshal(map[string]any{"result": map[string]any{"level": spec.Step, "passed": true, "tests": 42,
+		"packages": []any{}, "why": "mark " + string(raw)}})
+	fmt.Println(string(out))
+}
+`
+
+// 1898/AC4 (ADR 0535 §1): a tree whose mesh-build-go program answers a mark: the unit level with the runner workspace
+// compiles that program and runs it shut off, answers its mark judged by the candidate at the tree's commit, and the
+// program reached no key, no login, no name and no socket of the machine.
+func TestTheUnitLevelUnderTheWorkspaceRunnerRunsTheTreesProgramShutOff(t *testing.T) {
+	needTools(t, "the candidate's program shut off", "go", "git", "bwrap")
+	home := t.TempDir()
+	key, login := filepath.Join(home, ".ssh", "mesh_ed25519"), filepath.Join(home, "broker")
+	for _, f := range []string{key, login} {
+		if err := os.MkdirAll(filepath.Dir(f), 0o700); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(f, []byte("secret"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	tcp, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tcp.Close()
+
+	work, err := os.MkdirTemp("", "w")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { os.RemoveAll(work) })
+	tree := filepath.Join(work, "ws1", "tree")
+	dir := filepath.Join(tree, candidateModule)
+	if err := os.MkdirAll(filepath.Join(dir, "cmd", "mesh-build-go"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module meshbuildgo\n\ngo 1.25.0\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	src := fmt.Sprintf(candidateMainSource, key, login, tcp.Addr().String())
+	if err := os.WriteFile(filepath.Join(dir, "cmd", "mesh-build-go", "main.go"), []byte(src), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range [][]string{{"init", "-q"}, {"add", "-A"}, {"-c", "user.email=a@b", "-c", "user.name=x", "commit", "-qm", "x"}} {
+		if out, err := exec.Command("git", append([]string{"-C", tree}, c...)...).CombinedOutput(); err != nil {
+			t.Fatalf("git %v: %v %s", c, err, out)
+		}
+	}
+	head, err := exec.Command("git", "-C", tree, "rev-parse", "HEAD").Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
+	m.Run = StepRunner(home)
+	out, err := m.test(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace})
+	if err != nil {
+		t.Fatal(err)
+	}
+	res := out.(*TestResult)
+	if res.Tests != 42 || !strings.HasPrefix(res.Why, "mark ") {
+		t.Fatalf("result = %+v, want the candidate's mark", res)
+	}
+	if want := "candidate " + strings.TrimSpace(string(head)); res.JudgedBy != want {
+		t.Errorf("judged by %q, want %q", res.JudgedBy, want)
+	}
+	var seen map[string]bool
+	if err := json.Unmarshal([]byte(strings.TrimPrefix(res.Why, "mark ")), &seen); err != nil || len(seen) != 4 {
+		t.Fatalf("the program said %q: %v", res.Why, err)
+	}
+	for what, reached := range seen {
+		if reached {
+			t.Errorf("the candidate's program reached %s", what)
+		}
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/candidate_test.go

@@ -0,0 +1,262 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+)
+
+// Issue 1898 K1, K2 (ADR 0535): asked with the runner workspace, the digest and test steps compile the workspace's own
+// mesh-build-go shut off and run the step as it, in one shut-off process; the answer says which program ran.
+
+const candidateSHA = "fedcba9876543210fedcba9876543210fedcba98"
+
+// candidateRig is a module over a workspace whose tree holds modules/mesh-build-go, with a runner standing for git, go
+// and the candidate's program; what ran shut off is in seen, every command in ran.
+type candidateRig struct {
+	m        *Module
+	states   *keptStates
+	seen     []shutOff
+	ran      []string
+	compile  forgekey.Ran                     // what go build answers; Status 0 writes the program
+	answer   func(spec levelSpec) levelAnswer // what the candidate's program answers
+	specs    []levelSpec
+	compiled int
+}
+
+func newCandidateRig(t *testing.T) *candidateRig {
+	t.Helper()
+	cr := &candidateRig{states: newStates()}
+	cr.m = moduleWith(cr.states, "go1.27.1")
+	cr.m.Home, cr.m.Work, cr.m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
+	cr.m.Executable = "/opt/bundle/mesh-build-go"
+	dir := filepath.Join(cr.m.Work, "ws1", "tree", candidateModule)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module meshbuildgo\n\ngo 1.26.0\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	cr.answer = func(spec levelSpec) levelAnswer {
+		if spec.Step == stepDigest {
+			return levelAnswer{Digest: &DigestResult{Level: spec.Level, Packages: map[string]string{"meshbuildgo/cmd/mesh-build-go": "mark"}}}
+		}
+		return levelAnswer{Result: &TestResult{Level: spec.Step, Passed: true, Tests: 42, Packages: []PackageResult{}, Why: "mark"}}
+	}
+	exe := filepath.Join(cr.m.Work, "ws1", "candidate", "mesh-build-go")
+	cr.m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		line := name + " " + strings.Join(args, " ")
+		cr.ran = append(cr.ran, line)
+		switch {
+		case name == "git" && slices.Contains(args, "rev-parse"):
+			return forgekey.Ran{Stdout: candidateSHA + "\n"}
+		case name == "git" && slices.Contains(args, "ls-files"):
+			return forgekey.Ran{Stdout: "go.mod\n"}
+		case name == "postgres":
+			return forgekey.Ran{Stdout: "postgres (PostgreSQL) 18.6\n"}
+		case name == "go" && slices.Contains(args, "build"):
+			cr.compiled++
+			for _, want := range []string{"GOPROXY=off", "GOTOOLCHAIN=local", "GOFLAGS=-mod=mod"} {
+				if !slices.Contains(env, want) {
+					t.Errorf("the candidate compiled without %s: %v", want, env)
+				}
+			}
+			if cr.compile.Status == 0 && cr.compile.Err == "" {
+				if err := os.WriteFile(args[slices.Index(args, "-o")+1], []byte("program"), 0o755); err != nil {
+					t.Fatal(err)
+				}
+			}
+			return cr.compile
+		case name == exe && len(args) == 2 && args[0] == levelArg:
+			var spec levelSpec
+			if err := json.Unmarshal([]byte(args[1]), &spec); err != nil {
+				t.Errorf("the spec %s: %v", args[1], err)
+			}
+			cr.specs = append(cr.specs, spec)
+			raw, _ := json.Marshal(cr.answer(spec))
+			return forgekey.Ran{Stdout: string(raw) + "\n"}
+		}
+		return forgekey.Ran{Status: 1, Err: "not expected here: " + line}
+	}, &cr.seen)
+	return cr
+}
+
+func (cr *candidateRig) test(t *testing.T, args map[string]any) (TestResult, delivery.Step) {
+	t.Helper()
+	if _, err := stepTool(t, cr.m, "build_go_test")(args); err != nil {
+		t.Fatal(err)
+	}
+	st := cr.states.waitEnd(t)
+	var res TestResult
+	if st.State == delivery.StepPassed {
+		if err := json.Unmarshal(st.Result, &res); err != nil {
+			t.Fatalf("result %s: %v", st.Result, err)
+		}
+	}
+	return res, st
+}
+
+func TestTheDigestAndTestToolsAnnounceTheRunner(t *testing.T) { // 1898 K1, ADR 0535 §2
+	m := moduleWith(newStates(), "go1.27.1")
+	for _, s := range m.StepTools() {
+		if s.Name != "build_go_digest" && s.Name != "build_go_test" {
+			continue
+		}
+		p, _ := s.Input["properties"].(map[string]any)["runner"].(map[string]any)
+		if p == nil || !slices.Equal(p["enum"].([]string), []string{RunnerInstalled, RunnerWorkspace}) {
+			t.Errorf("%s announces the runner %v, want installed and workspace", s.Name, p)
+		}
+	}
+}
+
+func TestTheUnitLevelUnderTheWorkspaceRunnerRunsTheCandidateShutOff(t *testing.T) { // 1898 K1, K2 (AC4's unit part)
+	cr := newCandidateRig(t)
+	res, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace, "packages": []any{"./..."}})
+	if st.State != delivery.StepPassed {
+		t.Fatalf("the step ended %s: %s", st.State, st.Reason)
+	}
+	if res.Tests != 42 || res.Why != "mark" || res.JudgedBy != "candidate "+candidateSHA {
+		t.Errorf("result = %+v, want the candidate's mark judged by candidate %s", res, candidateSHA)
+	}
+	if len(cr.specs) != 1 || cr.specs[0].Step != stepUnit || cr.specs[0].Workspace != "ws1" || !slices.Equal(cr.specs[0].Packages, []string{"./..."}) || cr.specs[0].Bound != UnitBound {
+		t.Errorf("the candidate was asked %+v, want the unit level of ws1 with its packages and bound", cr.specs)
+	}
+	for _, l := range cr.ran {
+		if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") {
+			t.Errorf("the installed program ran %q: the level is the candidate's to run", l)
+		}
+	}
+	// Two commands shut off: the compile, then the candidate's process; each as a run step is (ADR 0448 §2).
+	if len(cr.seen) != 2 || !strings.Contains(cr.seen[0].command, " build ") || !strings.Contains(cr.seen[0].command, "-C "+filepath.Join(cr.m.Work, "ws1", "tree", candidateModule)) ||
+		!strings.HasPrefix(cr.seen[1].command, filepath.Join(cr.m.Work, "ws1", "candidate", "mesh-build-go")+" level ") {
+		t.Fatalf("shut off %+v, want the compile and the candidate's process", cr.seen)
+	}
+	for _, s := range cr.seen {
+		checkShutOff(t, cr.m, s, false)
+	}
+}
+
+func TestTheIntegrationLevelUnderTheWorkspaceRunnerRunsTheCandidate(t *testing.T) { // 1898 K2
+	cr := newCandidateRig(t)
+	res, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelIntegration, "runner": RunnerWorkspace})
+	if st.State != delivery.StepPassed || res.JudgedBy != "candidate "+candidateSHA {
+		t.Fatalf("the step ended %s (%s): %+v", st.State, st.Reason, res)
+	}
+	if len(cr.specs) != 1 || cr.specs[0].Step != stepIntegration || cr.specs[0].Bound != IntegrationRun || cr.specs[0].Setup != IntegrationSetup {
+		t.Errorf("the candidate was asked %+v, want the integration level with its bounds", cr.specs)
+	}
+}
+
+func TestTheDigestUnderTheWorkspaceRunnerIsTheCandidatesAndTheCandidateIsCompiledOnce(t *testing.T) { // 1898 K1, K2
+	cr := newCandidateRig(t)
+	out, err := stepTool(t, cr.m, "build_go_digest")(map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace})
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := out.(DigestResult)
+	if d.Packages["meshbuildgo/cmd/mesh-build-go"] != "mark" || d.JudgedBy != "candidate "+candidateSHA {
+		t.Errorf("digest = %+v, want the candidate's, judged by candidate %s", d, candidateSHA)
+	}
+	if len(cr.specs) != 1 || cr.specs[0].Step != stepDigest || cr.specs[0].Level != LevelUnit {
+		t.Errorf("the candidate was asked %+v", cr.specs)
+	}
+	cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace})
+	if cr.compiled != 1 {
+		t.Errorf("compiled the candidate %d times for two steps of one workspace, want once", cr.compiled)
+	}
+}
+
+func TestACandidateThatDoesNotCompileFailsTheStepWithTheCompilersFirstErrorAndRunsNoLevel(t *testing.T) { // 1898/AC5
+	cr := newCandidateRig(t)
+	cr.compile = forgekey.Ran{Status: 1, Stderr: "# meshbuildgo/cmd/mesh-build-go\ncmd/mesh-build-go/digest.go:12:2: undefined: x\ncmd/mesh-build-go/digest.go:13:2: undefined: y\n"}
+	_, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace})
+	if st.State == delivery.StepPassed || !strings.Contains(st.Reason, "does not compile: cmd/mesh-build-go/digest.go:12:2: undefined: x") || strings.Contains(st.Reason, "undefined: y") {
+		t.Fatalf("the step ended %s: %q, want it failed naming the first compile error", st.State, st.Reason)
+	}
+	if len(cr.specs) != 0 {
+		t.Errorf("the candidate ran %+v after it failed to compile", cr.specs)
+	}
+	for _, l := range cr.ran {
+		if strings.Contains(l, " test ") {
+			t.Errorf("a level ran: %q", l)
+		}
+	}
+	if _, err := stepTool(t, cr.m, "build_go_digest")(map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}); err == nil ||
+		!strings.Contains(err.Error(), "undefined: x") {
+		t.Errorf("the digest answered %v, want the compile error", err)
+	}
+}
+
+func TestTheInstalledProgramSaysItJudged(t *testing.T) { // 1898 K1, ADR 0535 §5
+	m, states, ws, _ := workspaceFor(t, "slow")
+	for _, runner := range []any{nil, RunnerInstalled} {
+		args := map[string]any{"workspace": ws, "level": LevelUnit}
+		if runner != nil {
+			args["runner"] = runner
+		}
+		res, st := runUnitWith(t, m, states, args)
+		if st.State != delivery.StepPassed || res.JudgedBy != RunnerInstalled {
+			t.Errorf("runner %v: the step ended %s, judged by %q, want installed", runner, st.State, res.JudgedBy)
+		}
+	}
+	dm, _ := digestFixture(t)
+	if d := digests(t, dm, LevelUnit); d.JudgedBy != RunnerInstalled {
+		t.Errorf("the installed digest is judged by %q, want installed", d.JudgedBy)
+	}
+}
+
+func TestAnUnknownRunnerIsRefused(t *testing.T) { // 1898 K1
+	cr := newCandidateRig(t)
+	if _, err := cr.m.digest(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": "elsewhere"}); err == nil {
+		t.Error("the digest took the runner elsewhere")
+	}
+	if _, err := cr.m.test(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": "elsewhere"}); err == nil {
+		t.Error("the test step took the runner elsewhere")
+	}
+}
+
+func TestTheShutOffProcessRunsTheStepItIsAsked(t *testing.T) { // 1898 K2: the candidate's side of the spec
+	m, _, ws, calls := workspaceFor(t, "slow")
+	m.Inside = true
+	a := m.answer(context.Background(), levelSpec{Step: stepUnit, Workspace: ws, Packages: []string{"./..."}, Bound: UnitBound})
+	if a.Error != "" || a.Result == nil || a.Result.Level != LevelUnit || a.Result.Tests == 0 {
+		t.Fatalf("the unit step answered %+v", a)
+	}
+	ranTest := false
+	for _, c := range *calls {
+		ranTest = ranTest || strings.HasPrefix(c.line, "go -C ") && strings.Contains(c.line, " test -json ")
+	}
+	if !ranTest {
+		t.Errorf("the process ran no go test: %+v", *calls)
+	}
+	dm, _ := digestFixture(t)
+	dm.Inside = true
+	if a := dm.answer(context.Background(), levelSpec{Step: stepDigest, Level: LevelUnit, Workspace: "ws1"}); a.Error != "" || a.Digest == nil || len(a.Digest.Packages) == 0 {
+		t.Errorf("the digest step answered %+v", a)
+	}
+	if a := m.answer(context.Background(), levelSpec{Step: "deploy", Workspace: ws}); a.Error == "" {
+		t.Error("the process took a step that is not one")
+	}
+}
+
+func runUnitWith(t *testing.T, m *Module, states *keptStates, args map[string]any) (TestResult, delivery.Step) {
+	t.Helper()
+	if _, err := stepTool(t, m, "build_go_test")(args); err != nil {
+		t.Fatal(err)
+	}
+	st := states.waitEnd(t)
+	var res TestResult
+	if st.State == delivery.StepPassed {
+		if err := json.Unmarshal(st.Result, &res); err != nil {
+			t.Fatalf("result %s: %v", st.Result, err)
+		}
+	}
+	return res, st
+}

modules/mesh-build-go/cmd/mesh-build-go/digest.go

@@ -89,6 +89,9 @@ type DigestResult struct {
 	// only when they are the siblings it gave the fetch.
 	Beside       []string `json:"beside,omitempty"`
 	WithoutTests []string `json:"without_tests,omitempty"`
+	// JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
+	// runner workspace.
+	JudgedBy string `json:"judged_by,omitempty"`
 }
 
 func (m *Module) digestTool() StepTool {
@@ -103,6 +106,7 @@ func (m *Module) digestTool() StepTool {
 			"properties": map[string]any{
 				"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
 				"level":     map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
+				"runner":    runnerProperty,
 			},
 			"required": []string{"workspace", "level"},
 		},
@@ -113,12 +117,49 @@ func (m *Module) digestTool() StepTool {
 func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
 	workspace, _ := args["workspace"].(string)
 	level, _ := args["level"].(string)
+	runner, err := runnerArg(args["runner"])
+	if err != nil {
+		return nil, err
+	}
+	if runner == RunnerWorkspace {
+		if err := m.workspaceThere(workspace); err != nil {
+			return nil, err
+		}
+		switch level {
+		case LevelUnit:
+		case LevelIntegration:
+			if _, err := m.checkPostgres(ctx); err != nil {
+				return nil, err // a *buildstep.Refusal, as the installed program's digest says it
+			}
+		default:
+			return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
+		}
+		a, judged, err := m.asCandidate(ctx, levelSpec{Step: stepDigest, Level: level, Workspace: workspace})
+		if err != nil {
+			return nil, err
+		}
+		if a.Digest == nil {
+			return nil, fmt.Errorf("the candidate's mesh-build-go answered no digest")
+		}
+		a.Digest.JudgedBy = judged
+		return *a.Digest, nil
+	}
+	res, err := m.digestHere(ctx, workspace, level)
+	if err != nil {
+		return nil, err
+	}
+	res.JudgedBy = RunnerInstalled
+	return res, nil
+}
+
+// digestHere is the digest step as this program runs it.
+func (m *Module) digestHere(ctx context.Context, workspace, level string) (DigestResult, error) {
 	if !workspaceOK.MatchString(workspace) {
-		return nil, fmt.Errorf("%q is not a workspace id", workspace)
+		return DigestResult{}, fmt.Errorf("%q is not a workspace id", workspace)
 	}
 	tree := filepath.Join(m.Work, workspace, "tree")
 	if _, err := os.Stat(tree); err != nil {
-		return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
+		return DigestResult{}, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
 	}
 	var tags []string
 	header := []string{"digest " + DigestVersion, "level " + level}
@@ -126,17 +167,17 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
 	case LevelUnit:
 	case LevelIntegration:
 		if _, err := m.checkPostgres(ctx); err != nil {
-			return nil, err
+			return DigestResult{}, err
 		}
 		tags = integrationTags
 		header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer)
 	default:
-		return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
+		return DigestResult{}, fmt.Errorf("the level %q is neither unit nor integration", level)
 	}
 	header = append(header, "tags "+strings.Join(tags, ","))
 	beside, err := m.besideLines(workspace)
 	if err != nil {
-		return nil, err
+		return DigestResult{}, err
 	}
 	besideHeader := make([]string, 0, len(beside))
 	for _, b := range beside {
@@ -145,16 +186,16 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
 
 	dirs, err := m.goModDirs(ctx, tree)
 	if err != nil {
-		return nil, err
+		return DigestResult{}, err
 	}
 	res := DigestResult{Level: level, Packages: map[string]string{}, Beside: beside}
 	for _, d := range dirs {
 		pkgs, err := m.listPackages(ctx, workspace, tree, d, tags)
 		if err != nil {
-			return nil, err
+			return DigestResult{}, err
 		}
 		if err := digestModule(&res, pkgs, tree, header, besideHeader); err != nil {
-			return nil, err
+			return DigestResult{}, err
 		}
 	}
 	sort.Strings(res.WithoutTests)

modules/mesh-build-go/cmd/mesh-build-go/level.go

@@ -18,11 +18,22 @@ import (
 	"git.novox.be/novox/mesh-sdk/go/buildstep"
 )
 
-// levelArg is the first argument that makes this binary an integration level's process.
+// levelArg is the first argument that makes this binary a step's shut-off process: the integration level's, and, when
+// the merge changes this module, the candidate's program running a digest or a level (ADR 0535 §1, candidate.go).
 const levelArg = "level"
 
-// levelSpec is what the shut-off process is asked: the workspace, the packages and the -run pattern, the level's bounds.
+// The steps a shut-off process runs (levelSpec.Step); empty is the integration level, as a spec said before the runner.
+const (
+	stepDigest      = "digest"
+	stepUnit        = LevelUnit
+	stepIntegration = LevelIntegration
+)
+
+// levelSpec is what the shut-off process is asked: the step, the workspace, the packages and the -run pattern, the
+// level's bounds; for a digest, its level.
 type levelSpec struct {
+	Step      string        `json:"step,omitempty"`
+	Level     string        `json:"level,omitempty"`
 	Work      string        `json:"work"`
 	Cache     string        `json:"cache"`
 	Workspace string        `json:"workspace"`
@@ -32,10 +43,11 @@ type levelSpec struct {
 	Bound     time.Duration `json:"bound"`
 }
 
-// levelAnswer is the one line the process prints: the level's result, or why it could not run.
+// levelAnswer is the one line the process prints: the level's result or the digest, or why it could not run.
 type levelAnswer struct {
-	Result *TestResult `json:"result,omitempty"`
-	Error  string      `json:"error,omitempty"`
+	Result *TestResult   `json:"result,omitempty"`
+	Digest *DigestResult `json:"digest,omitempty"`
+	Error  string        `json:"error,omitempty"`
 }
 
 // executable is this program's path, bound read-only into the step: Executable when a test set it.
@@ -50,35 +62,63 @@ func (m *Module) executable() (string, error) {
 	return filepath.EvalSymlinks(exe)
 }
 
-// integrationShutOff runs the integration level in one shut-off process and reads its answer.
+// integrationShutOff runs the integration level in one shut-off process of this program and reads its answer.
 func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string) (*TestResult, error) {
 	exe, err := m.executable()
 	if err != nil {
 		return nil, fmt.Errorf("the integration level's program: %w", err)
 	}
 	b := m.integrationBounds()
-	spec, err := json.Marshal(levelSpec{Work: m.Work, Cache: m.Cache, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run})
+	a, err := m.shutOff(ctx, exe, levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run})
 	if err != nil {
 		return nil, err
 	}
+	if a.Result == nil {
+		return nil, fmt.Errorf("the integration level's shut-off process answered no result")
+	}
+	return a.Result, nil
+}
+
+// shutOff runs the program exe as one shut-off process of the workspace (`exe level <spec>`) and reads its answer: the
+// workspace read-write, the program alone read-only, the shared caches beneath a throwaway layer, the Node build
+// module's toolchains read-only for the integration level.
+func (m *Module) shutOff(ctx context.Context, exe string, spec levelSpec) (levelAnswer, error) {
+	spec.Work, spec.Cache = m.Work, m.Cache
+	raw, err := json.Marshal(spec)
+	if err != nil {
+		return levelAnswer{}, err
+	}
 	for _, c := range m.caches() {
 		if err := os.MkdirAll(c, 0o700); err != nil {
-			return nil, fmt.Errorf("the shared cache %s: %w", c, err)
+			return levelAnswer{}, fmt.Errorf("the shared cache %s: %w", c, err)
 		}
 	}
-	iso := m.isolation(workspace, false)
+	iso := m.isolation(spec.Workspace, false)
 	iso.ReadOnly = append(iso.ReadOnly, exe) // the program alone, not the directory it lies in
-	iso.ReadOnly = append(iso.ReadOnly, m.nodeToolchains()...)
-	r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(spec))
+	if spec.Step == stepIntegration || spec.Step == "" {
+		iso.ReadOnly = append(iso.ReadOnly, m.nodeToolchains()...)
+	}
+	r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(raw))
 	var answer levelAnswer
 	line := lastLine(r.Stdout)
-	if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Error == "") {
-		return nil, fmt.Errorf("the integration level's shut-off process answered nothing it says (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
+	if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Digest == nil && answer.Error == "") {
+		return levelAnswer{}, fmt.Errorf("the %s step's shut-off process answered nothing it says (status %d %s): %s", stepName(spec.Step), r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
 	}
 	if answer.Error != "" {
-		return nil, fmt.Errorf("%s", answer.Error)
+		return levelAnswer{}, fmt.Errorf("%s", answer.Error)
 	}
-	return answer.Result, nil
+	return answer, nil
+}
+
+// stepName names a spec's step in words.
+func stepName(step string) string {
+	switch step {
+	case stepDigest:
+		return "digest"
+	case stepUnit:
+		return "unit level's"
+	}
+	return "integration level's"
 }
 
 // NodeToolchains is the Node build module's toolchains directory on this node: its cache directory (module.json
@@ -124,18 +164,40 @@ func runLevelProcess(args []string) int {
 	if !workspaceOK.MatchString(spec.Workspace) {
 		return say(levelAnswer{Error: fmt.Sprintf("%q is not a workspace id", spec.Workspace)})
 	}
-	m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"),
+	m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"), UnitBound: spec.Bound,
 		IntegrationBounds: Bounds{Setup: spec.Setup, Run: spec.Bound}, Say: func(string, ...any) {}}
 	m.Run = StepRunner(m.Home)
-	ctx := context.Background()
-	major, err := m.checkPostgres(ctx)
-	if err != nil {
-		return say(levelAnswer{Error: err.Error()})
+	return say(m.answer(context.Background(), spec))
+}
+
+// answer runs the spec's step in this process, already shut off: a digest of a level, the unit level or the integration
+// level of the workspace.
+func (m *Module) answer(ctx context.Context, spec levelSpec) levelAnswer {
+	tree := filepath.Join(m.Work, spec.Workspace, "tree")
+	switch spec.Step {
+	case stepDigest:
+		d, err := m.digestHere(ctx, spec.Workspace, spec.Level)
+		if err != nil {
+			return levelAnswer{Error: err.Error()}
+		}
+		return levelAnswer{Digest: &d}
+	case stepUnit:
+		res, err := m.runLevel(ctx, tree, levelOpts{workspace: spec.Workspace, level: LevelUnit, packages: spec.Packages, run: spec.Run,
+			bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(spec.Workspace)})
+		if err != nil {
+			return levelAnswer{Error: err.Error()}
+		}
+		return levelAnswer{Result: res}
+	case stepIntegration, "":
+		major, err := m.checkPostgres(ctx)
+		if err != nil {
+			return levelAnswer{Error: err.Error()}
+		}
+		res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major)
+		if err != nil {
+			return levelAnswer{Error: err.Error()}
+		}
+		return levelAnswer{Result: res}
 	}
-	tree := filepath.Join(spec.Work, spec.Workspace, "tree")
-	res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major)
-	if err != nil {
-		return say(levelAnswer{Error: err.Error()})
-	}
-	return say(levelAnswer{Result: res})
+	return levelAnswer{Error: fmt.Sprintf("the step %q is not digest, unit or integration", spec.Step)}
 }

modules/mesh-build-go/cmd/mesh-build-go/levels.go

@@ -83,6 +83,9 @@ type TestResult struct {
 	OverWhich string `json:"over_which,omitempty"`
 	Store     string `json:"store,omitempty"`
 	Bus       string `json:"bus,omitempty"`
+	// JudgedBy is the program that ran the level (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
+	// runner workspace.
+	JudgedBy string `json:"judged_by,omitempty"`
 }
 
 // event is one line of `go test -json` (test2json, and the build events go test adds).

modules/mesh-build-go/cmd/mesh-build-go/test.go

@@ -37,7 +37,8 @@ func (m *Module) testTool() StepTool {
 				"level":     map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
 				"packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
 					"description": "package patterns of each module, ./... when none"},
-				"run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"},
+				"run":    map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"},
+				"runner": runnerProperty,
 			},
 			"required": []string{"workspace", "level"},
 		},
@@ -72,13 +73,48 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
 	if _, err := os.Stat(tree); err != nil {
 		return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
 	}
-	switch level {
-	case LevelUnit:
-		return m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
-	case LevelIntegration:
-		return m.integration(ctx, workspace, tree, packages, pattern)
+	runner, err := runnerArg(args["runner"])
+	if err != nil {
+		return nil, err
 	}
-	return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
+	var res *TestResult
+	switch {
+	case runner == RunnerWorkspace && (level == LevelUnit || level == LevelIntegration):
+		return m.testAsCandidate(ctx, workspace, level, packages, pattern)
+	case level == LevelUnit:
+		res, err = m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
+	case level == LevelIntegration:
+		res, err = m.integration(ctx, workspace, tree, packages, pattern)
+	default:
+		return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
+	}
+	if err != nil {
+		return nil, err
+	}
+	res.JudgedBy = RunnerInstalled
+	return res, nil
+}
+
+// testAsCandidate runs a level as the candidate's program (ADR 0535 §1): the integration level is refused first as the
+// installed program would refuse it (no postgres), then the whole level runs in the candidate's one shut-off process.
+func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string) (*TestResult, error) {
+	spec := levelSpec{Step: stepUnit, Workspace: workspace, Packages: packages, Run: pattern, Bound: m.unitBound()}
+	if level == LevelIntegration {
+		if _, err := m.checkPostgres(ctx); err != nil {
+			return nil, err // a *buildstep.Refusal: this level only
+		}
+		b := m.integrationBounds()
+		spec = levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run}
+	}
+	a, judged, err := m.asCandidate(ctx, spec)
+	if err != nil {
+		return nil, err
+	}
+	if a.Result == nil {
+		return nil, fmt.Errorf("the candidate's mesh-build-go answered no result of the %s level", level)
+	}
+	a.Result.JudgedBy = judged
+	return a.Result, nil
 }
 
 // besideEnv is MESH_CHECK_BESIDE, naming the directory build_go_fetch placed the workspace's siblings in (ADR 0484