modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go

@@ -145,7 +145,8 @@ func Tools(r check.Reader) []stdio.Tool {
 		{Name: check.StatusReadTool,
 			Description: "What holds the checker back and what runs: the capacity per node (the setting, the default one step per node), whether the controller's " +
 				"delivery plans in their build phase hold a key (no new check starts while one does), the steps in flight on the build modules by node, and the " +
-				"candidates without a verdict for their heads (a group of hq's pull requests alone is left out: merge-check.sh judges it). A key a build " +
+				"candidates without a verdict for their heads (a group of hq's pull requests alone is left out: merge-check.sh judges it), and the " +
+				"candidates whose check waits by kind (waiting_by_kind: new, nodes, and trunk, which start after the other two). A key a build " +
 				"module left running (started past the step bound, or its build module not running on its node) is listed apart under left_running " +
 				"with why, and not counted. The places the checker holds are in its process, not in a bucket, and the answer says so. Read-only.",
 			Input: map[string]any{"type": "object", "properties": map[string]any{}},
@@ -184,19 +185,21 @@ func main() {
 		Capacity:   capacity, HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
 	tries := stdio.State(check.TriesState)
 	source := stdio.State("mesh-delivery-gate.source")
+	trunks := func() (map[string]string, error) {
+		e, err := source.Get(delivery.SourceMainKey)
+		if err != nil || e == nil {
+			return nil, err
+		}
+		s, err := delivery.DecodeSource(e.Value)
+		return s.Heads, err
+	}
+	r.Trunks = trunks
 	trying := Trying{
 		Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
 			_, err := tries.Create(key, json.RawMessage(value))
 			return err
 		}},
-		Trunks: func() (map[string]string, error) {
-			e, err := source.Get(delivery.SourceMainKey)
-			if err != nil || e == nil {
-				return nil, err
-			}
-			s, err := delivery.DecodeSource(e.Value)
-			return s.Heads, err
-		},
+		Trunks: trunks,
 	}
 	if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
 		fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)

modules/mesh-delivery-checker/internal/check/checker.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -40,6 +41,9 @@ type Outcome struct {
 	Builds  []BuildNote
 	// Slowest names the slowest tests of a level that went over its bound ("unit pkg.Test 1.2s").
 	Slowest []string
+	// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
+	// that changes none is not named.
+	Judged map[string]string
 }
 
 // BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
@@ -65,7 +69,7 @@ type Checker struct {
 	Trunks   interface{ Heads() map[string]string }
 	// Bound is how long one check may run; DefaultBound when 0.
 	Bound time.Duration
-	// Limit is how many checks run at once; two when 0.
+	// Limit is how many checks run at once; two when 0. A freed place goes to the first waiting check by rank (ADR 0536 §4).
 	Limit int
 	// Hold says whether a delivery builds, so no new check starts (ADR 0484 clause 12); nil never holds.
 	Hold func() bool
@@ -77,7 +81,10 @@ type Checker struct {
 	Logf     func(format string, args ...any)
 
 	once      sync.Once
-	sem       chan struct{}
+	limit     int
+	running   int       // checks holding a place
+	waiting   []*waiter // checks waiting for a place, in the order they asked
+	seq       uint64
 	wg        sync.WaitGroup
 	mu        sync.Mutex
 	known     map[string]delivery.Candidate // by candidates key
@@ -94,7 +101,7 @@ func (ch *Checker) init() {
 		if limit <= 0 {
 			limit = 2
 		}
-		ch.sem = make(chan struct{}, limit)
+		ch.limit = limit
 		ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
 		ch.nodesFail = map[string]uint64{}
 	})
@@ -199,10 +206,10 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
 	ch.wg.Add(1)
 	go func() {
 		defer ch.wg.Done()
-		ch.sem <- struct{}{}
-		defer func() { <-ch.sem }()
 		for {
+			ch.admit(cs, ch.kindOf(ctx, cs))
 			ch.try(ctx, key, cs)
+			ch.leave()
 			ch.mu.Lock()
 			more := ch.again[cs.Key]
 			delete(ch.again, cs.Key)
@@ -215,10 +222,121 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
 			if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil {
 				cs = next
 			}
+			cs.Queued = ch.now() // asked again: it waits for a place by its rank like any other
 		}
 	}()
 }
 
+// The kinds of a check waiting for a place, by the rank they start in (ADR 0536 §4): a head never judged and a head asked
+// again after a failure of the nodes first, then a head asked again only because its trunk moved.
+const (
+	KindNew   = "new"
+	KindNodes = "nodes"
+	KindTrunk = "trunk"
+)
+
+var kindRank = map[string]int{KindNew: 0, KindNodes: 0, KindTrunk: 1}
+
+// waiter is a check waiting for a place.
+type waiter struct {
+	group, key, kind string
+	queued           time.Time
+	seq              uint64
+	ready            chan struct{}
+}
+
+// Waiting names a check waiting for a place: its group, its verdict key, its kind and when it was asked.
+type Waiting struct {
+	Group  string    `json:"group"`
+	Key    string    `json:"key"`
+	Kind   string    `json:"kind"`
+	Queued time.Time `json:"queued"`
+}
+
+// Waiting are the checks waiting for a place, in the order they would start.
+func (ch *Checker) Waiting() []Waiting {
+	ch.init()
+	ch.mu.Lock()
+	defer ch.mu.Unlock()
+	ws := slices.Clone(ch.waiting)
+	slices.SortStableFunc(ws, before)
+	out := make([]Waiting, len(ws))
+	for i, w := range ws {
+		out[i] = Waiting{Group: w.group, Key: w.key, Kind: w.kind, Queued: w.queued}
+	}
+	return out
+}
+
+// before orders waiting checks: the lower rank, then the one asked first, then the one that asked for a place first.
+func before(a, b *waiter) int {
+	if ra, rb := kindRank[a.kind], kindRank[b.kind]; ra != rb {
+		return ra - rb
+	}
+	if c := a.queued.Compare(b.queued); c != 0 {
+		return c
+	}
+	return int(a.seq) - int(b.seq)
+}
+
+// kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
+// failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
+// move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
+func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
+	value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
+	if err != nil || !found {
+		return KindNew
+	}
+	v, err := delivery.DecodeVerdict(value)
+	switch {
+	case err != nil || v.State == delivery.VerdictRunning:
+		return KindNew
+	case IsNodesFailure(v):
+		return KindNodes
+	}
+	return KindTrunk
+}
+
+// admit returns when the check holds a place: at once when one is free and no check waits, else when a check's end hands its
+// place to this one as the first by rank. No timer: a check's end wakes the next (leave).
+func (ch *Checker) admit(cs *Case, kind string) {
+	ch.mu.Lock()
+	if ch.running < ch.limit && len(ch.waiting) == 0 {
+		ch.running++
+		ch.mu.Unlock()
+		return
+	}
+	ch.seq++
+	w := &waiter{group: cs.Group, key: cs.Key, kind: kind, queued: cs.Queued, seq: ch.seq, ready: make(chan struct{})}
+	ch.waiting = append(ch.waiting, w)
+	ch.handOn()
+	ch.mu.Unlock()
+	<-w.ready
+}
+
+// leave frees the check's place and hands it to the first waiting check by rank.
+func (ch *Checker) leave() {
+	ch.mu.Lock()
+	ch.running--
+	ch.handOn()
+	ch.mu.Unlock()
+}
+
+// handOn gives each free place to the first waiting check by rank; it is called with mu held.
+func (ch *Checker) handOn() {
+	for ch.running < ch.limit && len(ch.waiting) > 0 {
+		first := 0
+		for i := range ch.waiting {
+			if before(ch.waiting[i], ch.waiting[first]) < 0 {
+				first = i
+			}
+		}
+		w := ch.waiting[first]
+		ch.waiting = slices.Delete(ch.waiting, first, first+1)
+		ch.running++
+		close(w.ready)
+	}
+}
+
 func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
 	ch.mu.Lock()
 	defer ch.mu.Unlock()
@@ -311,7 +429,7 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
 		return
 	}
 	began := ch.now()
-	ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil)
+	ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
 	out := ch.judge(ctx, cs)
 	if ctx.Err() != nil {
 		return
@@ -342,11 +460,11 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
 	if ch.movedSince(cs) {
 		// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
 		// check of the new trunk, which Reconsider starts, ends.
-		ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil)
+		ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
 		ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
 		return
 	}
-	ch.writeOwed(ctx, cs, v, out.Builds, out.Slowest)
+	ch.writeOwed(ctx, cs, v, out)
 	ch.logf("%s: %s", cs.Key, v.State)
 }
 
@@ -366,7 +484,7 @@ func failure(machine, words string) delivery.Verdict {
 // writeOwed writes, for each pull request judged, the status the forge bridge performs: pending while the check runs, success
 // or failure at its end (ADR 0484 clause 2). A key already holding the same value is not written again, so the bridge
 // performs one act for each change; an older head's key is never touched.
-func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, builds []BuildNote, slowest []string) {
+func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, out Outcome) {
 	for _, p := range cs.Pulls {
 		key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head)
 		if err != nil {
@@ -375,7 +493,7 @@ func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict,
 		}
 		state := map[string]string{delivery.VerdictRunning: "pending", delivery.VerdictPass: "success", delivery.VerdictFail: "failure"}[v.State]
 		o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state,
-			Target: strings.TrimSuffix(ch.Link, "/") + "/" + cs.Key, Text: Line(v, cs, p, builds, slowest)}
+			Target: strings.TrimSuffix(ch.Link, "/") + "/" + cs.Key, Text: Line(v, cs, p, out.Builds, out.Slowest, out.Judged)}
 		data, err := delivery.EncodeCheckerOwed(o)
 		if err != nil {
 			ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err)
@@ -486,7 +604,7 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
 		if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
 			continue
 		}
-		ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil)
+		ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
 	}
 	ch.Reconsider(ctx)
 }

modules/mesh-delivery-checker/internal/check/judge.go

@@ -17,6 +17,35 @@ import (
 // goModule is the build module the Go repositories are judged by.
 const goModule = buildrun.GoBuildModule
 
+// RunnerWorkspace is the runner the Go build module's digest and test steps are asked with when the merge changes the build
+// module itself: the installed module compiles the candidate's program from the merged tree and runs the step as it, shut
+// off (ADR 0535 §1).
+const RunnerWorkspace = "workspace"
+
+// buildGoDir is the Go build module's directory in the catalogue: a merge changing a path under it is judged by its
+// candidate (ADR 0535 §1). mesh-build-node's own directory is judged by the installed module until its part of issue 1898.
+const buildGoDir = "modules/mesh-build-go/"
+
+// changesBuildGo says whether a merge of the repository with these changed paths changes the Go build module.
+func changesBuildGo(repository string, changed []string) bool {
+	if repository != catalogueRepository {
+		return false
+	}
+	for _, p := range changed {
+		if strings.HasPrefix(p, buildGoDir) {
+			return true
+		}
+	}
+	return false
+}
+
+// The words of the line on the program that judged a change to the Go build module (ADR 0535 §2, §5).
+const (
+	judgedByOldInstalled = "judged by the installed mesh-build-go: it does not run a candidate yet"
+	judgedByInstalled    = "judged by the installed mesh-build-go"
+	judgedByCandidate    = "judged by the candidate's mesh-build-go at "
+)
+
 // verdictPtr is a verdict of failure a step of the check ends with.
 type verdictPtr = *delivery.Verdict
 
@@ -118,6 +147,34 @@ type target struct {
 	// so a verdict is kept or taken only under a digest that says it folded exactly these in (lead decision L5).
 	beside []string
 	plans  map[string]*goPlan // by level
+	// runner is RunnerWorkspace while the target's steps are asked to run the candidate (ADR 0535 §1), empty otherwise.
+	runner string
+	// judged is what the line says of the program that judged a change to the build module; empty for any other change.
+	judged string
+}
+
+// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
+// installed module (a module that does not take the runner says nothing, and is asked without it from now on: ADR 0535 §2).
+// It says whether the candidate ran.
+func (t *target) judgedBy(answer string) bool {
+	switch {
+	case strings.HasPrefix(answer, "candidate "):
+		t.judged = judgedByCandidate + short12(strings.TrimPrefix(answer, "candidate "))
+		return true
+	case answer == "":
+		t.judged, t.runner = judgedByOldInstalled, ""
+	default:
+		t.judged, t.runner = judgedByInstalled, ""
+	}
+	return false
+}
+
+// stepArgs are a step's arguments with the target's runner.
+func (t *target) stepArgs(args map[string]any) map[string]any {
+	if t.runner != "" {
+		args["runner"] = t.runner
+	}
+	return args
 }
 
 // goPlan is what a Go target has to run at one level: the digests, the packages without a kept verdict and the tests taken.
@@ -216,7 +273,16 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
 			slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
 		}
 	}
-	return Outcome{Verdict: v, Slowest: slow}
+	out := Outcome{Verdict: v, Slowest: slow}
+	for _, t := range st.targets {
+		if t.judged != "" {
+			if out.Judged == nil {
+				out.Judged = map[string]string{}
+			}
+			out.Judged[pullKey(Pull{Repository: t.pull.Repository, Number: t.pull.Number})] = t.judged
+		}
+	}
+	return out
 }
 
 // fail is a verdict of failure at a machine.
@@ -326,6 +392,9 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict {
 				return fail(node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
 			}
 			t.changed = t.fetched.Changed
+			if bm == buildrun.GoBuildModule && changesBuildGo(p.Repository, t.changed) {
+				t.runner = RunnerWorkspace
+			}
 			merged = append(merged, t.fetched)
 			st.targets = append(st.targets, t)
 		}
@@ -467,7 +536,7 @@ func (st *state) planGo(ctx context.Context, t *target, level string) *delivery.
 		return nil
 	}
 	raw, err := st.step(ctx, t, "digest", fmt.Sprintf("digest the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node),
-		map[string]any{"workspace": t.fetched.Workspace, "level": level})
+		t.stepArgs(map[string]any{"workspace": t.fetched.Workspace, "level": level}))
 	var d Digested
 	if err == nil {
 		err = json.Unmarshal(raw, &d)
@@ -475,6 +544,11 @@ func (st *state) planGo(ctx context.Context, t *target, level string) *delivery.
 	if ctx.Err() != nil {
 		return fail(t.node, ctx.Err().Error())
 	}
+	if err == nil && t.runner != "" && t.judgedBy(d.JudgedBy) {
+		// The candidate's digest decides which verdicts would be taken: every package runs and nothing is kept (ADR 0535 §4).
+		plan.all = true
+		return nil
+	}
 	if err != nil || len(d.Packages) == 0 {
 		// The build module named no digest (a level it cannot digest here): the level runs whole and keeps nothing.
 		if err != nil {
@@ -532,7 +606,7 @@ func (st *state) runGo(ctx context.Context, t *target, level string, priority bo
 			return nil
 		}
 	}
-	args := map[string]any{"workspace": t.fetched.Workspace, "level": level}
+	args := t.stepArgs(map[string]any{"workspace": t.fetched.Workspace, "level": level})
 	if run != nil {
 		args["packages"] = run
 	}
@@ -547,6 +621,9 @@ func (st *state) runGo(ctx context.Context, t *target, level string, priority bo
 	if err := json.Unmarshal(raw, &res); err != nil {
 		return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err))
 	}
+	if t.runner != "" {
+		t.judgedBy(res.JudgedBy)
+	}
 	a := st.levels[level]
 	a.run += res.Tests
 	a.skipped += res.Skipped

modules/mesh-delivery-checker/internal/check/judged_test.go

@@ -0,0 +1,207 @@
+package check
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// The judged_by answer of mesh-build-go's digest and test steps (ADR 0535 §5; the milestone file's interface for issue
+// 1898): "installed" for a step its installed program ran, "candidate <merge commit>" for one the candidate's program ran
+// under runner workspace; an installed module older than the runner names nothing (it ignores the unknown argument: the
+// SDK's tool server checks no argument against the input schema).
+
+const buildGoFile = "modules/mesh-build-go/cmd/mesh-build-go/digest.go"
+
+// candidateBench is a bench whose fetch changes the Go build module's directory, and whose build module runs the candidate
+// when asked to (a module with the runner) or ignores the runner (one without).
+func candidateBench(t *testing.T, takesRunner bool) *bench {
+	b := newBench(t)
+	b.fetch.Changed = []string{"README.md", buildGoFile}
+	judged := func(c stepCall) string {
+		if !takesRunner {
+			return ""
+		}
+		if c.args["runner"] == RunnerWorkspace {
+			return "candidate " + b.fetch.Merge
+		}
+		return "installed"
+	}
+	b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
+		d := b.digest[c.args["level"].(string)]
+		d.JudgedBy = judged(c)
+		return d, nil
+	}
+	b.test = func(c stepCall) (any, error) {
+		r := captured[Tested](t, "go-test-slow")
+		r.Level, _ = c.args["level"].(string)
+		r.JudgedBy = judged(c)
+		return r, nil
+	}
+	return b
+}
+
+func TestAMergeChangingTheGoBuildModuleIsAskedWithTheCandidateRunner(t *testing.T) { // 1898/AC1
+	b := candidateBench(t, true)
+	b.judge(t, caseOf(t, pullOf(catalogueRepository, 9, head1)))
+	asked := append(b.steps.tool("digest"), b.steps.tool("test")...)
+	if len(asked) < 3 {
+		t.Fatalf("asked %+v, want the digest and both levels", asked)
+	}
+	for _, c := range asked {
+		if c.args["runner"] != RunnerWorkspace {
+			t.Errorf("%s %v was asked with runner %v, want %s", c.tool, c.args["level"], c.args["runner"], RunnerWorkspace)
+		}
+	}
+}
+
+func TestAMergeNotChangingTheGoBuildModuleIsAskedWithoutARunner(t *testing.T) { // 1898/AC1
+	for _, tc := range []struct {
+		name, repo, path string
+	}{
+		{"another module of the catalogue", catalogueRepository, "modules/mesh-build-node/cmd/mesh-build-node/main.go"},
+		{"a look-alike path in another repository", repoA, buildGoFile},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			b := candidateBench(t, true)
+			b.fetch.Changed = []string{tc.path}
+			out := b.judge(t, caseOf(t, pullOf(tc.repo, 9, head1)))
+			for _, c := range append(b.steps.tool("digest"), b.steps.tool("test")...) {
+				if _, ok := c.args["runner"]; ok {
+					t.Errorf("%s was asked with runner %v", c.tool, c.args["runner"])
+				}
+			}
+			if len(out.Judged) != 0 {
+				t.Errorf("judged = %v, want nothing said of the program", out.Judged)
+			}
+		})
+	}
+}
+
+func TestAnInstalledModuleWithoutTheRunnerJudgesAsTodayAndTheLineSaysSo(t *testing.T) { // 1898/AC2
+	b := candidateBench(t, false)
+	cs := caseOf(t, pullOf(catalogueRepository, 9, head1))
+	out := b.judge(t, cs)
+	if out.Verdict.State != delivery.VerdictPass {
+		t.Fatalf("verdict = %+v", out.Verdict)
+	}
+	// As today: the digests key kept verdicts, and the levels are asked without the runner the module ignored.
+	for _, c := range b.steps.tool("test") {
+		if _, ok := c.args["runner"]; ok {
+			t.Errorf("the %v level was asked again with the runner the module does not take", c.args["level"])
+		}
+	}
+	if keys, _ := b.kept.Keys(context.Background()); len(keys) != 2 {
+		t.Errorf("kept %v, want the passing package at both levels, as today", keys)
+	}
+	p := cs.Pulls[0]
+	want := "judged by the installed mesh-build-go: it does not run a candidate yet"
+	if got := out.Judged[pullKey(p)]; got != want {
+		t.Fatalf("judged = %q, want %q", got, want)
+	}
+	if l := Line(out.Verdict, cs, p, out.Builds, out.Slowest, out.Judged); !strings.Contains(l, want) {
+		t.Errorf("line %q does not say %q", l, want)
+	}
+}
+
+func TestACandidateJudgedCheckTakesNoKeptVerdictAndWritesNone(t *testing.T) { // 1898/AC3
+	b := candidateBench(t, true)
+	if _, err := (Kept{Store: b.kept}).Keep(context.Background(), "unit", digA, keptValue("fixture/slow", 2)); err != nil {
+		t.Fatal(err)
+	}
+	out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 9, head1)))
+	if out.Verdict.State != delivery.VerdictPass {
+		t.Fatalf("verdict = %+v", out.Verdict)
+	}
+	tests := b.steps.tool("test")
+	if len(tests) != 2 {
+		t.Fatalf("test calls = %+v, want both levels", tests)
+	}
+	for _, c := range tests {
+		if _, ok := c.args["packages"]; ok {
+			t.Errorf("the %v level was asked for %v, want it whole", c.args["level"], c.args["packages"])
+		}
+	}
+	if u := out.Verdict.Levels[delivery.LevelUnit]; u.Kept != 0 {
+		t.Errorf("unit = %+v, want nothing taken", u)
+	}
+	if keys, _ := b.kept.Keys(context.Background()); len(keys) != 1 || keys[0] != "unit."+digA {
+		t.Errorf("kept keys = %v, want only the one kept before", keys)
+	}
+}
+
+func TestTheLineNamesTheCandidateThatJudged(t *testing.T) { // 1898/AC6
+	b := candidateBench(t, true)
+	cs := caseOf(t, pullOf(catalogueRepository, 9, head1))
+	out := b.judge(t, cs)
+	p := cs.Pulls[0]
+	want := "judged by the candidate's mesh-build-go at " + b.fetch.Merge[:12]
+	for name, l := range map[string]string{
+		"line":   Line(out.Verdict, cs, p, out.Builds, out.Slowest, out.Judged),
+		"report": Report(out.Verdict, cs, p, out.Builds, out.Slowest, out.Judged),
+	} {
+		if !strings.Contains(l, want) {
+			t.Errorf("the %s %q does not say %q", name, l, want)
+		}
+	}
+	// A long failing line keeps the program it was judged by; only the lesser parts go.
+	v := out.Verdict
+	v.State = delivery.VerdictFail
+	v.Failed = delivery.FailedFirst{Test: "unit: git.novox.be/novox/mesh-catalog/modules/mesh-build-go/cmd/mesh-build-go.TestADigestFoldsTheSiblings", Words: strings.Repeat("x", 60)}
+	if l := Line(v, cs, p, out.Builds, out.Slowest, out.Judged); !strings.Contains(l, "judged by the candidate's") {
+		t.Errorf("the long line %q dropped the program", l)
+	}
+}
+
+func TestInAGroupOnlyTheBuildModulesPullRequestNamesItsProgram(t *testing.T) { // 1898/AC6
+	b := candidateBench(t, true)
+	cs := caseOf(t, pullOf(catalogueRepository, 9, head1), pullOf(repoA, 7, head1))
+	out := b.judge(t, cs)
+	for _, p := range cs.Pulls {
+		l := Report(out.Verdict, cs, p, out.Builds, out.Slowest, out.Judged)
+		says := strings.Contains(l, "judged by")
+		if says != (p.Repository == catalogueRepository) {
+			t.Errorf("%s#%d: %q", p.Repository, p.Number, l)
+		}
+	}
+}
+
+func TestACandidateThatDoesNotCompileFailsTheCheckWithTheCompilersWords(t *testing.T) { // ADR 0535 §3, checker side
+	b := candidateBench(t, true)
+	refusal := "the candidate's mesh-build-go does not compile: modules/mesh-build-go/cmd/mesh-build-go/digest.go:12:2: undefined: x"
+	b.steps.handlers["build_go_digest"] = func(stepCall) (any, error) { return nil, errorString(refusal) }
+	b.test = func(stepCall) (any, error) { return nil, errorString(refusal) }
+	out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 9, head1)))
+	if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Words, "undefined: x") {
+		t.Fatalf("verdict = %+v, want a failure naming the compile error", out.Verdict)
+	}
+	if n := len(b.steps.tool("test")); n != 1 {
+		t.Errorf("%d test steps, want the first one's failure to end the check", n)
+	}
+}
+
+type errorString string
+
+func (e errorString) Error() string { return string(e) }
+
+func TestTheStatusTheForgeShowsNamesTheProgramThatJudged(t *testing.T) { // 1898/AC6, the forge's line
+	ctx := context.Background()
+	words := judgedByCandidate + "fedcba987654"
+	j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
+		return Outcome{Verdict: passing(), Judged: map[string]string{repoA + "#7": words}}, nil
+	}}
+	r := newRig(t, j)
+	r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
+	r.ch.Wait()
+	key, _ := delivery.CheckerOwedKey(repoA, 7, head1)
+	raw, _, _, _ := r.owed.Get(ctx, key)
+	o, err := delivery.DecodeCheckerOwed(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o.State != "success" || !strings.Contains(o.Text, words) {
+		t.Errorf("status %+v, want success saying %q", o, words)
+	}
+}

modules/mesh-delivery-checker/internal/check/line.go

@@ -52,15 +52,20 @@ func firstFailed(v delivery.Verdict) string {
 // pieces are the parts of a line, from the most to the least needed.
 type pieces struct {
 	head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
+	// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
+	judged string
 }
 
+// pullKey names a pull request of a case: `<owner/repo>#<number>`.
+func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
+
 func (p pieces) join(drop int) string {
-	parts := []string{p.head, p.commits, p.counts}
+	parts := []string{p.head, p.commits, p.counts, p.judged}
 	switch {
 	case drop < 3 && p.times != "":
 		parts = append(parts, p.times)
 	case drop == 3 && p.shortTimes != "":
-		parts = []string{p.head, p.commits, p.shortCounts, p.shortTimes}
+		parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
 	}
 	if p.over != "" {
 		parts = append(parts, p.over)
@@ -77,13 +82,7 @@ func (p pieces) join(drop int) string {
 	case p.composition != "":
 		parts = append(parts, p.composition)
 	}
-	out := make([]string, 0, len(parts))
-	for _, s := range parts {
-		if s != "" {
-			out = append(out, s)
-		}
-	}
-	return strings.Join(out, " | ")
+	return joinSet(parts...)
 }
 
 func levelWords(name string, r delivery.LevelResult) string {
@@ -139,8 +138,9 @@ func buildWords(builds []BuildNote) string {
 }
 
 // parts makes the pieces of the line of a pull request of a case.
-func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) pieces {
+func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
 	var pc pieces
+	pc.judged = judged[pullKey(p)]
 	commits := short12(p.Head) + " on " + short12(p.Trunk)
 	switch v.State {
 	case delivery.VerdictRunning:
@@ -221,22 +221,48 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
 }
 
 // Report is the full line of a pull request's status, with no limit: what the read tools and the full report say.
-func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) string {
-	return parts(v, cs, p, builds, slowest).join(0)
+func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) string {
+	return parts(v, cs, p, builds, slowest, judged).join(0)
 }
 
 // Line is the words of a status for one pull request of a case (ADR 0447 clause 3, ADR 0484 clause 7): the verdict first (a
 // failure names the conflict, the refused step or the test before anything), the head and the trunk commit as twelve
 // characters each, the tests run, kept and skipped per level, the time of each part, a level over its bound with its slowest
-// tests, the builds of the touched modules, and that the composition was not yet run. It is at most MaxLine characters: the
+// tests, the builds of the touched modules, and that the composition was not yet run; for a change to a build module, the
+// program that judged it (judged, by pullKey; ADR 0535 §5). It is at most MaxLine characters: the
 // builds, then the group drop first, then the times are given in their compact form, then dropped, and the line is cut at last.
-func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) string {
-	pc := parts(v, cs, p, builds, slowest)
+// The program that judged is never dropped: a line naming it gives the counts in their compact form, then drops them and
+// what follows, and is cut in its first part rather than lose it.
+func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) string {
+	pc := parts(v, cs, p, builds, slowest, judged)
 	for drop := 0; drop <= 4; drop++ {
 		if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine {
 			return s
 		}
 	}
+	if pc.judged != "" {
+		for _, counts := range []string{pc.shortCounts, ""} {
+			if s := joinSet(pc.head, pc.commits, counts, pc.judged); utf8.RuneCountInString(s) <= MaxLine {
+				return s
+			}
+		}
+		rest := " | " + joinSet(pc.commits, pc.judged)
+		room := MaxLine - utf8.RuneCountInString(rest) - 1
+		if head := []rune(pc.head); room > 0 && len(head) > room {
+			return string(head[:room]) + "…" + rest
+		}
+	}
 	runes := []rune(pc.join(4))
 	return string(runes[:MaxLine-1]) + "…"
 }
+
+// joinSet joins the parts that are not empty.
+func joinSet(parts ...string) string {
+	out := make([]string, 0, len(parts))
+	for _, s := range parts {
+		if s != "" {
+			out = append(out, s)
+		}
+	}
+	return strings.Join(out, " | ")
+}

modules/mesh-delivery-checker/internal/check/line_test.go

@@ -22,7 +22,7 @@ func passVerdict() delivery.Verdict {
 
 func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCompositionWasNotRun(t *testing.T) { // 1300/AC1
 	cs, p := lineCase(t, pullOf(repoA, 7, head1))
-	full := Report(passVerdict(), cs, p, nil, nil)
+	full := Report(passVerdict(), cs, p, nil, nil, nil)
 	for _, want := range []string{"pass 111111111111 on aaaaaaaaaaaa", "unit 12 run, 301 kept", "integration 3 run, 40 kept", "queue 800ms", "clone 4.1s", "unit 400ms", "int setup 3.2s", "int run 12s", "composition not yet run"} {
 		if !strings.Contains(full, want) {
 			t.Errorf("the report %q lacks %q", full, want)
@@ -31,7 +31,7 @@ func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCom
 	if !strings.HasPrefix(full, "pass ") || strings.Contains(full, "composition passed") {
 		t.Errorf("report = %q", full)
 	}
-	line := Line(passVerdict(), cs, p, nil, nil)
+	line := Line(passVerdict(), cs, p, nil, nil, nil)
 	if n := utf8.RuneCountInString(line); n > MaxLine {
 		t.Errorf("the line is %d characters, want at most %d: %q", n, MaxLine, line)
 	}
@@ -51,7 +51,7 @@ func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { //
 		"conflict": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "shanks", Words: "novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}}, "failed: shanks: novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"},
 		"refused":  {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "mesh-build-go", Words: "no go build node qualifies: builders disagree"}}, "failed: mesh-build-go: no go build node qualifies"},
 	} {
-		line := Line(c.v, cs, p, nil, nil)
+		line := Line(c.v, cs, p, nil, nil, nil)
 		if !strings.HasPrefix(line, c.want) || !strings.Contains(line, "111111111111 on aaaaaaaaaaaa") {
 			t.Errorf("%s: line = %q, want it to open with %q and name the commits", name, line, c.want)
 		}
@@ -62,8 +62,8 @@ func TestALevelOverItsBoundIsNamedWithItsSlowestTestsAndTheVerdictIsNotChanged(t
 	cs, p := lineCase(t, pullOf(repoA, 7, head1))
 	v := passVerdict()
 	v.OverBound = []string{"unit"}
-	line := Line(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"})
-	if !strings.HasPrefix(line, "pass ") || !strings.Contains(Report(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"}), "over bound: unit (slowest unit pkg.TestSlow 7.0s)") {
+	line := Line(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"}, nil)
+	if !strings.HasPrefix(line, "pass ") || !strings.Contains(Report(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"}, nil), "over bound: unit (slowest unit pkg.TestSlow 7.0s)") {
 		t.Errorf("line = %q", line)
 	}
 }
@@ -72,16 +72,16 @@ func TestSkippedTestsAreSaidAndNeverAFlatPass(t *testing.T) { // 1300/AC6
 	cs, p := lineCase(t, pullOf(repoA, 7, head1))
 	v := passVerdict()
 	v.Levels[delivery.LevelUnit] = delivery.LevelResult{Run: 12, Kept: 301, Skipped: 2}
-	if line := Line(v, cs, p, nil, nil); !strings.Contains(line, "unit 12 run, 301 kept, 2 skipped") {
+	if line := Line(v, cs, p, nil, nil, nil); !strings.Contains(line, "unit 12 run, 301 kept, 2 skipped") {
 		t.Errorf("line = %q", line)
 	}
 }
 
 func TestAGroupsMembersEachCarryTheSameVerdict(t *testing.T) { // 1300/AC4
 	cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
-	a := Line(passVerdict(), cs, cs.Pulls[0], nil, nil)
-	b := Line(passVerdict(), cs, cs.Pulls[1], nil, nil)
-	if !strings.HasPrefix(a, "pass ") || !strings.HasPrefix(b, "pass ") || !strings.Contains(Report(passVerdict(), cs, cs.Pulls[0], nil, nil), "group feat/x of 2 pull requests") {
+	a := Line(passVerdict(), cs, cs.Pulls[0], nil, nil, nil)
+	b := Line(passVerdict(), cs, cs.Pulls[1], nil, nil, nil)
+	if !strings.HasPrefix(a, "pass ") || !strings.HasPrefix(b, "pass ") || !strings.Contains(Report(passVerdict(), cs, cs.Pulls[0], nil, nil, nil), "group feat/x of 2 pull requests") {
 		t.Errorf("lines %q and %q", a, b)
 	}
 	if !strings.Contains(a, "111111111111") || !strings.Contains(b, "222222222222") {
@@ -92,16 +92,16 @@ func TestAGroupsMembersEachCarryTheSameVerdict(t *testing.T) { // 1300/AC4
 func TestTheBuildsOfTheTouchedModulesAreSaidAndDroppedFirstWhenTheLineIsTooLong(t *testing.T) { // ADR 0484 clause 7
 	cs, p := lineCase(t, pullOf(repoA, 7, head1))
 	builds := []BuildNote{{"avahi", "built"}, {"mesh-tools", "waiting"}}
-	if r := Report(passVerdict(), cs, p, builds, nil); !strings.Contains(r, "builds: avahi built, mesh-tools waiting") {
+	if r := Report(passVerdict(), cs, p, builds, nil, nil); !strings.Contains(r, "builds: avahi built, mesh-tools waiting") {
 		t.Errorf("report = %q", r)
 	}
-	line := Line(passVerdict(), cs, p, builds, nil)
+	line := Line(passVerdict(), cs, p, builds, nil, nil)
 	if utf8.RuneCountInString(line) > MaxLine {
 		t.Errorf("line %q is too long", line)
 	}
 	short := passVerdict()
 	short.Took = delivery.Took{}
-	if l := Line(short, cs, p, builds[:1], nil); !strings.Contains(l, "builds: avahi built") {
+	if l := Line(short, cs, p, builds[:1], nil, nil); !strings.Contains(l, "builds: avahi built") {
 		t.Errorf("a short line keeps its builds: %q", l)
 	}
 }
@@ -109,14 +109,14 @@ func TestTheBuildsOfTheTouchedModulesAreSaidAndDroppedFirstWhenTheLineIsTooLong(
 func TestALineIsNeverLongerThanTheLimit(t *testing.T) { // 1300/AC1
 	cs, p := lineCase(t, pullOf(repoA, 7, head1))
 	v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "integration: " + strings.Repeat("pkg/", 50) + ".TestSomething", Words: strings.Repeat("because ", 40)}}
-	if line := Line(v, cs, p, nil, nil); utf8.RuneCountInString(line) > MaxLine || !strings.HasPrefix(line, "failed: integration: ") {
+	if line := Line(v, cs, p, nil, nil, nil); utf8.RuneCountInString(line) > MaxLine || !strings.HasPrefix(line, "failed: integration: ") {
 		t.Errorf("line = %q (%d)", line, utf8.RuneCountInString(line))
 	}
 }
 
 func TestARunningCheckIsPending(t *testing.T) {
 	cs, p := lineCase(t, pullOf(repoA, 7, head1))
-	if line := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil); line != "checking 111111111111 on aaaaaaaaaaaa" {
+	if line := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil); line != "checking 111111111111 on aaaaaaaaaaaa" {
 		t.Errorf("line = %q", line)
 	}
 }
@@ -125,11 +125,11 @@ func TestThePassedCompositionIsSaidOnlyWhenItWasRun(t *testing.T) { // ADR 0484
 	cs, p := lineCase(t, pullOf(repoA, 7, head1))
 	v := passVerdict()
 	v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "afbe371003a570f088be4a394d83aa7a9458f7ab"}
-	if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition passed with afbe371003a5") {
+	if r := Report(v, cs, p, nil, nil, nil); !strings.Contains(r, "composition passed with afbe371003a5") {
 		t.Errorf("report = %q", r)
 	}
 	v.Composition = nil
-	if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition not yet run") || strings.Contains(r, "composition passed") {
+	if r := Report(v, cs, p, nil, nil, nil); !strings.Contains(r, "composition not yet run") || strings.Contains(r, "composition passed") {
 		t.Errorf("a verdict with no composition: %q", r)
 	}
 }
@@ -139,18 +139,18 @@ func TestTheCompositionsTimeAndMachinesAreSaidOnceTheAssignerReportsThem(t *test
 	v := passVerdict()
 	v.Took.Composition = 2400
 	v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "afbe371003a570f088be4a394d83aa7a9458f7ab", Machines: []string{"g14", "ace", "novox", "shanks"}}
-	r := Report(v, cs, p, nil, nil)
+	r := Report(v, cs, p, nil, nil, nil)
 	for _, want := range []string{"composition passed in 2.4s with afbe371003a5 on 4 machines: ace, g14, novox, shanks"} {
 		if !strings.Contains(r, want) {
 			t.Errorf("report %q lacks %q", r, want)
 		}
 	}
-	if l := Line(v, cs, p, nil, nil); !strings.Contains(l, "composition passed in 2.4s") || !strings.Contains(l, "on 4 machines") || utf8.RuneCountInString(l) > MaxLine {
+	if l := Line(v, cs, p, nil, nil, nil); !strings.Contains(l, "composition passed in 2.4s") || !strings.Contains(l, "on 4 machines") || utf8.RuneCountInString(l) > MaxLine {
 		t.Errorf("line = %q", l)
 	}
 	v.State, v.Failed = delivery.VerdictFail, delivery.FailedFirst{Machine: "novox", Words: "composition refused"}
 	v.Composition = &delivery.Composition{State: delivery.CompositionFail, Words: "novox: two seats claim dns", Machines: []string{"novox"}}
-	if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition failed in 2.4s on novox: novox: two seats claim dns") {
+	if r := Report(v, cs, p, nil, nil, nil); !strings.Contains(r, "composition failed in 2.4s on novox: novox: two seats claim dns") {
 		t.Errorf("failed composition report = %q", r)
 	}
 }

modules/mesh-delivery-checker/internal/check/rank_test.go

@@ -0,0 +1,217 @@
+package check
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"sync"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// The checker's ranked start of checks (ADR 0536 §4, issue 1897): a freed place goes to a head never judged (or asked again
+// after a failure of the nodes) before a head asked again only because its trunk moved; among equals the oldest asked first.
+// A running check is never stopped.
+
+// rankRig is a checker of one check at a time whose judge records the order checks start in and holds the first check
+// after hold is set until release is closed.
+type rankRig struct {
+	*rig
+	mu      sync.Mutex
+	started []*Case
+	hold    bool
+	held    chan struct{} // closed when the held check started
+	release chan struct{}
+	errs    []error // ctx errors the held check saw when released
+}
+
+func newRankRig(t *testing.T) *rankRig {
+	rr := &rankRig{held: make(chan struct{}), release: make(chan struct{})}
+	j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) {
+		rr.mu.Lock()
+		rr.started = append(rr.started, c)
+		hold := rr.hold
+		rr.hold = false
+		rr.mu.Unlock()
+		if hold {
+			close(rr.held)
+			<-rr.release
+			rr.mu.Lock()
+			rr.errs = append(rr.errs, ctx.Err())
+			rr.mu.Unlock()
+		}
+		return Outcome{Verdict: passing()}, nil
+	}}
+	rr.rig = newRig(t, j)
+	rr.ch.Limit = 1
+	now := time.Date(2026, 10, 23, 14, 0, 0, 0, time.UTC)
+	var clock sync.Mutex
+	rr.ch.Now = func() time.Time {
+		clock.Lock()
+		defer clock.Unlock()
+		now = now.Add(time.Second)
+		return now
+	}
+	return rr
+}
+
+func group(n int) string { return fmt.Sprintf("feat/g%d", n) }
+
+// judged puts n candidates in and judges them on the current trunk.
+func (rr *rankRig) judged(t *testing.T, n int) {
+	for i := 1; i <= n; i++ {
+		rr.ch.Candidate(context.Background(), group(i), candidateOf(t, group(i), pullOf(repoA, i, head1)))
+	}
+	rr.ch.Wait()
+	rr.mu.Lock()
+	rr.started = nil
+	rr.mu.Unlock()
+}
+
+// waitFor returns when n checks wait for a place.
+func (rr *rankRig) waitFor(t *testing.T, n int) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		if w := len(rr.ch.Waiting()); w == n {
+			return
+		} else if time.Now().After(deadline) {
+			t.Fatalf("%d checks wait, want %d", w, n)
+		}
+		runtime.Gosched()
+	}
+}
+
+// moveTrunkHoldingOne moves the trunk, so every judged candidate is asked again; the first re-judge to start is held.
+func (rr *rankRig) moveTrunkHoldingOne(t *testing.T, judged int) {
+	rr.mu.Lock()
+	rr.hold = true
+	rr.mu.Unlock()
+	rr.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
+	rr.ch.TrunkMoved(context.Background())
+	<-rr.held
+	rr.waitFor(t, judged-1)
+}
+
+func TestANewHeadStartsBeforeTheReJudgesOfATrunkMove(t *testing.T) { // 1897/AC1
+	rr := newRankRig(t)
+	rr.judged(t, 4)
+	rr.moveTrunkHoldingOne(t, 4) // one runs, three re-judges wait
+	rr.ch.Candidate(context.Background(), group(9), candidateOf(t, group(9), pullOf(repoA, 9, head2)))
+	rr.waitFor(t, 4)
+	close(rr.release)
+	rr.ch.Wait()
+	if len(rr.started) != 5 {
+		t.Fatalf("%d checks started, want 5", len(rr.started))
+	}
+	if g := rr.started[1].Group; g != group(9) {
+		t.Errorf("after the running check, %s started, want the new head's %s; order %v", g, group(9), groups(rr.started))
+	}
+}
+
+func TestTheReJudgeQueuedFirstStartsFirst(t *testing.T) { // 1897/AC2
+	rr := newRankRig(t)
+	rr.judged(t, 3)
+	rr.moveTrunkHoldingOne(t, 3) // one runs, two re-judges wait
+	close(rr.release)
+	rr.ch.Wait()
+	if len(rr.started) != 3 {
+		t.Fatalf("%d checks started, want 3", len(rr.started))
+	}
+	if a, b := rr.started[1], rr.started[2]; !a.Queued.Before(b.Queued) {
+		t.Errorf("%s (queued %s) started before %s (queued %s)", a.Group, a.Queued.Format(time.TimeOnly), b.Group, b.Queued.Format(time.TimeOnly))
+	}
+}
+
+func TestANewHeadDoesNotStopARunningReJudge(t *testing.T) { // 1897/AC3
+	rr := newRankRig(t)
+	rr.judged(t, 1)
+	rr.moveTrunkHoldingOne(t, 1) // the re-judge runs
+	rr.ch.Candidate(context.Background(), group(9), candidateOf(t, group(9), pullOf(repoA, 9, head2)))
+	rr.waitFor(t, 1)
+	close(rr.release)
+	rr.ch.Wait()
+	if len(rr.errs) != 1 || rr.errs[0] != nil {
+		t.Fatalf("the running re-judge saw %v, want it never stopped", rr.errs)
+	}
+	if len(rr.started) != 2 || rr.started[0].Group != group(1) || rr.started[1].Group != group(9) {
+		t.Errorf("started %v, want the re-judge to end before the new head starts", groups(rr.started))
+	}
+	key := keyOf(t, group(1), candidateOf(t, group(1), pullOf(repoA, 1, head1)))
+	if v := rr.verdict(t, key); v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 {
+		t.Errorf("the re-judge's verdict = %+v, want its pass on the moved trunk", v)
+	}
+}
+
+func TestWaitingChecksAreNamedByKind(t *testing.T) { // 1897/AC4, the checker's own view
+	rr := newRankRig(t)
+	rr.judged(t, 2)
+	rr.moveTrunkHoldingOne(t, 2)
+	rr.ch.Candidate(context.Background(), group(9), candidateOf(t, group(9), pullOf(repoA, 9, head2)))
+	rr.waitFor(t, 2)
+	kinds := map[string]string{}
+	for _, w := range rr.ch.Waiting() {
+		kinds[w.Group] = w.Kind
+	}
+	close(rr.release)
+	rr.ch.Wait()
+	running := rr.started[0].Group
+	other := group(1)
+	if running == other {
+		other = group(2)
+	}
+	if kinds[group(9)] != KindNew || kinds[other] != KindTrunk || len(kinds) != 2 {
+		t.Errorf("waiting = %v, want %s %s and %s %s", kinds, group(9), KindNew, other, KindTrunk)
+	}
+}
+
+func groups(cs []*Case) []string {
+	out := make([]string, len(cs))
+	for i, c := range cs {
+		out[i] = c.Group
+	}
+	return out
+}
+
+func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4
+	ctx := context.Background()
+	j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
+		if c.Group == group(2) {
+			return Outcome{Verdict: failure("checker", "no "+NodesWords+": every node is offline")}, nil
+		}
+		return Outcome{Verdict: passing()}, nil
+	}}
+	r := newRig(t, j)
+	cands := fakeState{}
+	put := func(n int, head string) {
+		cand := candidateOf(t, group(n), pullOf(repoA, n, head))
+		ckey, _ := delivery.CandidatesKey(group(n))
+		cands[ckey] = cand
+		r.ch.Candidate(ctx, group(n), cand)
+	}
+	put(1, head1) // judged on the old trunk: a re-judge waits
+	put(2, head1) // failed for the nodes: waits for them
+	r.ch.Wait()
+	r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
+	put(4, head1) // judged on the new trunk: stands
+	r.ch.Wait()
+	cand3 := candidateOf(t, group(3), pullOf(repoA, 3, head1)) // never judged
+	ckey3, _ := delivery.CandidatesKey(group(3))
+	cands[ckey3] = cand3
+	verdicts := fakeState{}
+	for k, v := range r.verdicts.vals {
+		verdicts[k] = v
+	}
+	rd := Reader{CandidateState: cands.read(), VerdictState: verdicts.read(),
+		Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA2, repoSDK: trunkS}, nil }}
+	st, err := rd.Status()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := map[string][]string{KindNew: {group(3)}, KindNodes: {group(2)}, KindTrunk: {group(1)}}
+	if fmt.Sprint(st.Waiting) != fmt.Sprint(want) {
+		t.Errorf("waiting = %v, want %v", st.Waiting, want)
+	}
+}

modules/mesh-delivery-checker/internal/check/read.go

@@ -40,7 +40,9 @@ type Reader struct {
 	Capacity     func() (string, error) // the capacity setting as written, `node=steps,...`
 	HaveState    StateRead              // the node-engine's have: whether a build module is on a node and running
 	StepBound    time.Duration          // how long the checker waits for one step; DefaultStepBound when 0
-	Now          func() time.Time
+	// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
+	Trunks func() (map[string]string, error)
+	Now    func() time.Time
 }
 
 // VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
@@ -82,7 +84,7 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
 		return out, nil
 	}
 	for _, p := range cs.Pulls {
-		out.Reports[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = Report(v, cs, p, nil, nil)
+		out.Reports[pullKey(p)] = Report(v, cs, p, nil, nil, nil)
 	}
 	for _, l := range cs.Left {
 		out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1]
@@ -280,7 +282,10 @@ type Status struct {
 	LeftRunning   []LeftStep     `json:"left_running"`
 	RunningByNode map[string]int `json:"running_by_node"`
 	Unjudged      []string       `json:"unjudged_candidates"`
-	Notes         []string       `json:"notes,omitempty"`
+	// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
+	// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
+	Waiting map[string][]string `json:"waiting_by_kind"`
+	Notes   []string            `json:"notes,omitempty"`
 }
 
 // Status reads the capacity, the plan bucket, the build modules' steps and the candidates against their verdicts.
@@ -353,6 +358,15 @@ func (r Reader) Status() (Status, error) {
 		st.Notes = append(st.Notes, "left_running are keys a build module left running (it stopped, restarted or was unassigned before it wrote the step's end): the checker's wait for each ended at the step bound, so none holds a place; the build module never ends them")
 	}
 	st.Unjudged = []string{}
+	st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
+	var heads map[string]string
+	if r.Trunks == nil {
+		st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
+	} else if h, err := r.Trunks(); err != nil {
+		st.Notes = append(st.Notes, "the trunk heads cannot be read, so no check is named as waiting for a trunk move: "+err.Error())
+	} else {
+		heads = h
+	}
 	if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
 		keys, err := r.CandidateState.Keys()
 		if err != nil {
@@ -375,8 +389,24 @@ func (r Reader) Status() (Status, error) {
 			if !checksAny(c) {
 				continue // hq's pull requests alone: merge-check.sh judges them, the checker never does
 			}
-			if v, err := r.VerdictState.Get(vkey); err == nil && len(v) == 0 {
+			raw, err = r.VerdictState.Get(vkey)
+			if err != nil {
+				continue
+			}
+			if len(raw) == 0 {
 				st.Unjudged = append(st.Unjudged, c.Group)
+				st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
+				continue
+			}
+			v, err := delivery.DecodeVerdict(raw)
+			switch {
+			case err != nil || v.State == delivery.VerdictRunning:
+			case IsNodesFailure(v):
+				st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
+			case heads != nil:
+				if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
+					st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
+				}
 			}
 		}
 	}

modules/mesh-delivery-checker/internal/check/results.go

@@ -87,6 +87,9 @@ type Tested struct {
 	Failed     []Failure   `json:"failed,omitempty"`
 	OverBound  bool        `json:"over_bound"`
 	OverWhich  string      `json:"over_which,omitempty"`
+	// JudgedBy is the program that ran the step (ADR 0535 §5): "installed", or "candidate <merge commit>" under runner
+	// workspace; absent from a build module that does not take the runner.
+	JudgedBy string `json:"judged_by,omitempty"`
 }
 
 // Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import
@@ -98,4 +101,6 @@ type Digested struct {
 	Level        string            `json:"level"`
 	Packages     map[string]string `json:"packages"`
 	WithoutTests []string          `json:"without_tests,omitempty"`
+	// JudgedBy is the program that ran the digest, as Tested.JudgedBy says.
+	JudgedBy string `json:"judged_by,omitempty"`
 }

modules/mesh-delivery-checker/internal/check/try.go

@@ -336,7 +336,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
 	t.Ended = tr.now().UTC()
 	t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
 	t.Levels, t.Machines = v.Levels, v.Machines
-	t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest)
+	t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
 	switch {
 	case !judged || IsNodesFailure(v):
 		t.State = TryError