Make checker_status count only steps that can still run, so a key a build module left running is not read as a full node (issue 1890)

The checker's own places are released when a step ends or its wait passes the step bound; checker_status read every
'running' key in the build modules' steps buckets as a step in flight, so keys left by stopped, restarted or unassigned
build modules looked like full nodes. It now lists them under left_running with why (past the bound, module not on the
node, module not running), and leaves hq-only groups out of the unjudged candidates.

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go

@@ -7,6 +7,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -155,6 +156,8 @@ type bed struct {
 	tools      map[string]jetstream.KeyValue
 	presence   jetstream.KeyValue
 	have       jetstream.KeyValue
+	instances  jetstream.KeyValue
+	stop       func() // stops the builder started last
 	fake       *fakeBuildModules
 	srvURL     string
 	dir        string
@@ -227,6 +230,7 @@ func startBed(t *testing.T) *bed {
 	b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
 	presence := mk(delivery.PresenceBucket, delivery.PresenceHistory)
 	b.presence, b.have = presence, mk(delivery.HaveBucket, delivery.HaveHistory)
+	b.instances = mk(instancesBucket, 1)
 	for _, node := range []string{"n1", "n2"} {
 		b.setOnline(t, node, true)
 		for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
@@ -344,7 +348,9 @@ func (b *bed) start(t *testing.T, node string, env map[string]string) {
 	runCtx, stop := context.WithCancel(b.ctx)
 	done := make(chan error, 1)
 	go func() { done <- run(runCtx, func(k string) string { return e[k] }, io.Discard) }()
-	t.Cleanup(func() { stop(); <-done })
+	var once sync.Once
+	b.stop = func() { once.Do(func() { stop(); <-done }) }
+	t.Cleanup(b.stop)
 }
 
 // await reads the build key until pred holds of it, within 20 s.
@@ -713,3 +719,26 @@ func TestAManifestFallingBackToAPredefinedKeyIsBuilt(t *testing.T) {
 		t.Errorf("build = %+v, want done: the manifest is read", v)
 	}
 }
+
+// 1886/AC5: a builder on n1 whose Go, Node and image build modules each answered a reference says under n1, in its bucket
+// of instances, that it writes references, and takes the say away when it stops: the controller delivers from its keys
+// only while it holds.
+func TestABuilderWhoseBuildModulesAnsweredReferencesSaysItAndStoppedSaysNothing(t *testing.T) {
+	b := startBed(t)
+	b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")              // a Go and a TypeScript bundle
+	b.moduleCommit(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit) // an image, for the trunk
+	b.start(t, "n1", nil)
+	var said string
+	for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && said == ""; time.Sleep(50 * time.Millisecond) {
+		if e, err := b.instances.Get(b.ctx, "n1"); err == nil {
+			said = string(e.Value())
+		}
+	}
+	if !strings.Contains(said, `"writes":["references"]`) {
+		t.Fatalf("the builder said %q", said)
+	}
+	b.stop()
+	if _, err := b.instances.Get(b.ctx, "n1"); !errors.Is(err, jetstream.ErrKeyNotFound) {
+		t.Fatalf("the stopped builder still says: %v", err)
+	}
+}

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go

@@ -178,6 +178,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		Manifests:  manifestReader{manifests},
 		Logf:       logger.Printf,
 	}
+	// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
+	// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
+	// yet (the manifest's state not made) is said, and the controller keeps asking build-agent.
+	switch kv, err := js.KeyValue(ctx, instancesBucket); {
+	case errors.Is(err, jetstream.ErrBucketNotFound):
+		logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
+	case err != nil:
+		return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
+	default:
+		says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
+		b.Answered = says.Answered
+		defer func() {
+			stopping, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+			defer cancel()
+			if err := says.unsay(stopping); err != nil {
+				logger.Printf("%v", err)
+			}
+		}()
+	}
 	// The toolchains first, so a build that starts from the first modules value finds the nodes.
 	watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
 	if err := watch.start(); err != nil {

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// What the builder says it writes (novox/hq issue 1886, ADR 0522): each running instance keeps a key, named by its node,
+// in its own bucket of instances (`mesh-delivery-builder_instances`, the manifest's state `instances`), saying that every
+// done key it writes names where its artifacts are (ADR 0498 §1). The references come from the build modules' own pushes,
+// so an instance says it only once a step of every build module it calls answered one since it started: a builder beside
+// a build module from before R4e says nothing. The controller delivers from the builder's keys only while an instance
+// says so, and asks build-agent as before R4e otherwise. The bucket is read by the controller alone, so its name and shape
+// are not in mesh-sdk.
+const (
+	instancesBucket  = "mesh-delivery-builder_instances"
+	writesReferences = `{"v":1,"writes":["references"]}`
+)
+
+// sayStore is the bucket of instances as the builder writes it.
+type sayStore interface {
+	Put(ctx context.Context, key string, value []byte) (uint64, error)
+	Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error
+}
+
+// sayer puts this instance's say once every build module in need answered a reference, and takes it away at the stop.
+type sayer struct {
+	ctx   context.Context
+	store sayStore
+	node  string
+	need  []string
+	logf  func(string, ...any)
+
+	mu       sync.Mutex
+	answered map[string]bool
+	said     bool
+}
+
+func newSayer(ctx context.Context, store sayStore, node string, need []string, logf func(string, ...any)) *sayer {
+	return &sayer{ctx: ctx, store: store, node: node, need: need, logf: logf, answered: map[string]bool{}}
+}
+
+// Answered is told the build module whose step answered a reference (build.Builder.Answered).
+func (s *sayer) Answered(module string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.answered[module] = true
+	if s.said {
+		return
+	}
+	for _, m := range s.need {
+		if !s.answered[m] {
+			return
+		}
+	}
+	if _, err := s.store.Put(s.ctx, s.node, []byte(writesReferences)); err != nil {
+		s.logf("saying in %s that this builder writes references: %v; said again at the next reference", instancesBucket, err)
+		return
+	}
+	s.said = true
+	s.logf("every build module answered a reference: said in %s that this builder writes references", instancesBucket)
+}
+
+// unsay takes the say away, if it was made: a builder replaced by one that does not write references leaves none.
+func (s *sayer) unsay(ctx context.Context) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if !s.said {
+		return nil
+	}
+	if err := s.store.Delete(ctx, s.node); err != nil {
+		return fmt.Errorf("taking away the say in %s: %w", instancesBucket, err)
+	}
+	s.said = false
+	return nil
+}

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says_test.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+type saysOf map[string]string
+
+func (s saysOf) Put(_ context.Context, key string, value []byte) (uint64, error) {
+	s[key] = string(value)
+	return 1, nil
+}
+
+func (s saysOf) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
+	delete(s, key)
+	return nil
+}
+
+// 1886/AC5: a builder says, under its node, that it writes references only once every build module it calls answered
+// one; stopped, it says nothing any more.
+func TestTheBuilderSaysItWritesReferencesOnceEveryBuildModuleAnsweredOne(t *testing.T) {
+	s := saysOf{}
+	say := newSayer(t.Context(), s, "shanks", []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"}, t.Logf)
+	say.Answered("mesh-build-go")
+	say.Answered("mesh-build-node")
+	if v, ok := s["shanks"]; ok {
+		t.Fatalf("said before the image build module answered a reference: %q", v)
+	}
+	say.Answered("mesh-build-image")
+	var v struct {
+		Writes []string `json:"writes"`
+	}
+	if err := json.Unmarshal([]byte(s["shanks"]), &v); err != nil || len(v.Writes) != 1 || v.Writes[0] != "references" {
+		t.Fatalf("the builder's say: %q %v", s["shanks"], err)
+	}
+	if err := say.unsay(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := s["shanks"]; ok {
+		t.Fatalf("the stopped builder still says: %v", s)
+	}
+}

modules/mesh-delivery-builder/internal/build/r4e_test.go

@@ -383,3 +383,21 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A
 		t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
 	}
 }
+
+// 1886/AC5: the builder tells who answered a reference: the build module whose step named one, and none for a step that
+// answered a digest alone (a build module from before R4e).
+func TestTheBuilderTellsWhichBuildModuleAnsweredAReference(t *testing.T) {
+	r := newRig(t, goBundle)
+	var answered []string
+	r.b.Answered = func(module string) { answered = append(answered, module) }
+	r.bm.noRef["ace/tool"] = true
+	r.consider(t, "modules/m")
+	if len(answered) != 0 {
+		t.Fatalf("a step that answered no reference was told as one: %v", answered)
+	}
+	delete(r.bm.noRef, "ace/tool")
+	r.consider(t, "modules/m")
+	if len(answered) != 1 || answered[0] != "mesh-build-go" {
+		t.Fatalf("the build module that answered a reference: %v, want mesh-build-go", answered)
+	}
+}

modules/mesh-delivery-builder/internal/build/run.go

@@ -45,6 +45,9 @@ type Builder struct {
 	// Limit is how many builds run at once; four when 0.
 	Limit int
 	Logf  func(format string, args ...any)
+	// Answered is told the build module whose step answered a reference (issue 1886): what the builder says it writes
+	// rests on it. Nil tells nobody.
+	Answered func(buildModule string)
 
 	once     sync.Once
 	sem      chan struct{}
@@ -749,6 +752,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
 		if built.Reference != "" && (stored(t.Kind) || !candidate) {
 			l.references[t.Artifact] = built.Reference
 		}
+		if built.Reference != "" && b.Answered != nil {
+			b.Answered(l.module)
+		}
 	}
 }
 

modules/mesh-delivery-builder/manifest_test.go

@@ -83,7 +83,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 	if m.Module != "mesh-delivery-builder" {
 		t.Errorf("module = %q", m.Module)
 	}
-	if want := []state{{Name: "builds", History: 5}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
+	if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
 		t.Errorf("state = %+v, want %+v", m.State, want)
 	}
 	if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains",

modules/mesh-delivery-builder/module.json

@@ -7,7 +7,8 @@
     "mesh-build-node.toolchain.reported"
   ],
   "state": [
-    { "name": "builds", "history": 5 }
+    { "name": "builds", "history": 5 },
+    { "name": "instances", "history": 1 }
   ],
   "reads": [
     "mesh-build-go.steps",

modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go

@@ -145,8 +145,9 @@ func Tools(r check.Reader) []stdio.Tool {
 		{Name: check.StatusReadTool,
 			Description: "What holds the checker back and what runs: the capacity per node (the setting, the default one step per node), whether the controller's " +
 				"delivery plans in their build phase hold a key (no new check starts while one does), the steps in flight on the build modules by node, and the " +
-				"candidates without a verdict for their heads. The steps the checker holds waiting for a place are in its process, not in a bucket, and the " +
-				"answer says so. Read-only.",
+				"candidates without a verdict for their heads (a group of hq's pull requests alone is left out: merge-check.sh judges it). A key a build " +
+				"module left running (started past the step bound, or its build module not running on its node) is listed apart under left_running " +
+				"with why, and not counted. The places the checker holds are in its process, not in a bucket, and the answer says so. Read-only.",
 			Input: map[string]any{"type": "object", "properties": map[string]any{}},
 			Run:   func(map[string]any) (any, error) { return r.Status() }},
 	}
@@ -167,11 +168,20 @@ func capacity() (string, error) {
 	return s.Capacity, json.Unmarshal(raw, &s)
 }
 
+// stepBound is the checker's step bound as the checker reads it (MESH_CHECKER_STEP_BOUND), its default when unset or not a
+// duration.
+func stepBound() time.Duration {
+	if d, err := time.ParseDuration(os.Getenv("MESH_CHECKER_STEP_BOUND")); err == nil && d > 0 {
+		return d
+	}
+	return check.DefaultStepBound
+}
+
 func main() {
 	r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
 		PreviewState: read(stdio.State("previews")), PlansState: read(stdio.State("mesh-controller.plans-building")),
 		StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
-		Capacity:   capacity}
+		Capacity:   capacity, HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
 	tries := stdio.State(check.TriesState)
 	source := stdio.State("mesh-delivery-gate.source")
 	trying := Trying{

modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go

@@ -40,9 +40,6 @@ func eventSubject(event string) string {
 }
 
 const (
-	// defaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a
-	// test level runs a repository's tests).
-	defaultStepBound = 30 * time.Minute
 	// defaultLink is the base of the address a status points at; the verdict key follows.
 	defaultLink = "mesh-delivery-view/checks"
 )
@@ -184,7 +181,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		logger.Printf("what the checker keeps could not be said on the bus, so readers of owed, previews and verdicts are not granted until it is: %v", err)
 	}
 
-	stepBound, err := durationEnv(getenv, "MESH_CHECKER_STEP_BOUND", defaultStepBound)
+	stepBound, err := durationEnv(getenv, "MESH_CHECKER_STEP_BOUND", check.DefaultStepBound)
 	if err != nil {
 		return err
 	}

modules/mesh-delivery-checker/internal/check/examiner_test.go

@@ -723,3 +723,41 @@ func TestNoStepOfACheckCarriesPublish(t *testing.T) { // ADR 0500 §4, 1309 (the
 		}
 	}
 }
+
+// keyCaller is a build module that starts every long step and answers its key on the node it was asked on.
+type keyCaller struct{}
+
+func (keyCaller) CallOn(node, key string, _ any) (json.RawMessage, error) {
+	return json.Marshal(map[string]string{"step": "lost1", "key": node + ".lost1", "state": delivery.StepRunning})
+}
+
+// lostSteps is a steps state whose key never ends: the build module stopped, restarted or was unassigned with the step
+// running, and nothing writes its end.
+type lostSteps struct{}
+
+func (lostSteps) Wait(ctx context.Context, _, _ string) (delivery.Step, error) {
+	<-ctx.Done()
+	return delivery.Step{}, ctx.Err()
+}
+
+func TestAStepThatNeverEndsEndsTheCheckLoudlyAndFreesItsNodesPlace(t *testing.T) { // 1890/AC1
+	b := newBench(t)
+	places := &buildrun.Places{Capacity: map[string]int{"shanks": 1}}
+	b.ex.Places = places
+	b.ex.Steps = buildrun.Runner{Caller: keyCaller{}, Steps: lostSteps{}, Bound: 20 * time.Millisecond}
+	out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
+	if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "shanks.lost1 did not end within") {
+		t.Fatalf("verdict = %+v", out.Verdict)
+	}
+	if r, res := places.Running(), places.Reserved(); len(r) != 0 || len(res) != 0 {
+		t.Fatalf("the lost step still holds shanks: running %v, reserved %v", r, res)
+	}
+	// The node's one place is free: the next check takes it at once.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	release, err := places.Acquire(ctx, "shanks")
+	if err != nil {
+		t.Fatalf("shanks's place was not freed: %v", err)
+	}
+	release()
+}

modules/mesh-delivery-checker/internal/check/read.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/delivery"
 	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
@@ -37,6 +38,9 @@ type Reader struct {
 	PlansState   StateRead              // the controller's delivery plans in their build phase; zero where the bucket is absent
 	StepStates   map[string]StateRead   // the build modules' steps states, by module
 	Capacity     func() (string, error) // the capacity setting as written, `node=steps,...`
+	HaveState    StateRead              // the node-engine's have: whether a build module is on a node and running
+	StepBound    time.Duration          // how long the checker waits for one step; DefaultStepBound when 0
+	Now          func() time.Time
 }
 
 // VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
@@ -254,6 +258,12 @@ type RunningStep struct {
 	Since  string `json:"since,omitempty"`
 }
 
+// LeftStep is a key a build module left running that no step behind it runs any more, and why.
+type LeftStep struct {
+	RunningStep
+	Why string `json:"why"`
+}
+
 // Status is what holds the checker back and what runs now, read from the buckets: the checker's held and running steps live in
 // its process, so they are shown as what the buckets say (the steps in flight on the build modules, the candidates without a
 // verdict for their heads, and the delivery plans in their build phase), not as the process's own counters.
@@ -267,6 +277,7 @@ type Status struct {
 		Holds    bool     `json:"holds"`
 	} `json:"plans_building"`
 	Running       []RunningStep  `json:"running_steps"`
+	LeftRunning   []LeftStep     `json:"left_running"`
 	RunningByNode map[string]int `json:"running_by_node"`
 	Unjudged      []string       `json:"unjudged_candidates"`
 	Notes         []string       `json:"notes,omitempty"`
@@ -295,7 +306,14 @@ func (r Reader) Status() (Status, error) {
 		mods = append(mods, m)
 	}
 	sort.Strings(mods)
-	st.Running = []RunningStep{}
+	st.Running, st.LeftRunning = []RunningStep{}, []LeftStep{}
+	now, bound := time.Now(), r.StepBound
+	if r.Now != nil {
+		now = r.Now()
+	}
+	if bound <= 0 {
+		bound = DefaultStepBound
+	}
 	for _, m := range mods {
 		sr := r.StepStates[m]
 		if sr.Keys == nil || sr.Get == nil {
@@ -320,10 +338,20 @@ func (r Reader) Status() (Status, error) {
 			if !step.Started.IsZero() {
 				rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
 			}
+			if why := r.leftWhy(m, step, now, bound); why != "" {
+				st.LeftRunning = append(st.LeftRunning, LeftStep{RunningStep: rs, Why: why})
+				continue
+			}
 			st.Running = append(st.Running, rs)
 			st.RunningByNode[step.Node]++
 		}
 	}
+	if r.HaveState.Get == nil {
+		st.Notes = append(st.Notes, "the node-engine's have is not readable here, so a step is not checked against its build module being on its node")
+	}
+	if len(st.LeftRunning) > 0 {
+		st.Notes = append(st.Notes, "left_running are keys a build module left running (it stopped, restarted or was unassigned before it wrote the step's end): the checker's wait for each ended at the step bound, so none holds a place; the build module never ends them")
+	}
 	st.Unjudged = []string{}
 	if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
 		keys, err := r.CandidateState.Keys()
@@ -344,11 +372,56 @@ func (r Reader) Status() (Status, error) {
 			if err != nil {
 				continue
 			}
+			if !checksAny(c) {
+				continue // hq's pull requests alone: merge-check.sh judges them, the checker never does
+			}
 			if v, err := r.VerdictState.Get(vkey); err == nil && len(v) == 0 {
 				st.Unjudged = append(st.Unjudged, c.Group)
 			}
 		}
 	}
-	st.Notes = append(st.Notes, "the steps the checker holds waiting for a place are in its process and in no bucket: a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held")
+	st.Notes = append(st.Notes, "the places the checker holds are in its process and in no bucket: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held")
 	return st, nil
 }
+
+// DefaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a test
+// level runs a repository's tests); past it the check fails naming the step, and the step's place on its node is free.
+const DefaultStepBound = 30 * time.Minute
+
+// leftWhy says why a key in the running state runs no step any more, or "" when it can still be running: its build module
+// is not on its node or is not running there (the node-engine's have), or it started longer ago than the step bound, when
+// the checker stopped waiting for it.
+func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound time.Duration) string {
+	if r.HaveState.Get != nil {
+		key, err := delivery.HaveKey(step.Node, module)
+		if err == nil {
+			raw, err := r.HaveState.Get(key)
+			switch {
+			case err != nil:
+			case len(raw) == 0:
+				return fmt.Sprintf("%s is not on %s", module, step.Node)
+			default:
+				if h, err := delivery.DecodeHave(raw); err == nil && h.Phase != delivery.PhaseRunning {
+					return fmt.Sprintf("%s on %s is %s", module, step.Node, h.Phase)
+				}
+			}
+		}
+	}
+	if step.Started.IsZero() {
+		return fmt.Sprintf("it says no start, so no wait of the checker's (bound %s) can be behind it", bound)
+	}
+	if end := step.Started.Add(bound); now.After(end) {
+		return fmt.Sprintf("started %s, past the step bound of %s: the checker stopped waiting for it at %s", step.Started.UTC().Format(time.RFC3339), bound, end.UTC().Format(time.RFC3339))
+	}
+	return ""
+}
+
+// checksAny says whether a candidate has a pull request the checker judges: one outside hq.
+func checksAny(c delivery.Candidate) bool {
+	for _, p := range c.Pulls {
+		if p.Repository != HQRepository {
+			return true
+		}
+	}
+	return false
+}

modules/mesh-delivery-checker/internal/check/read_test.go

@@ -142,6 +142,8 @@ func TestTheStatusToolReadsCapacityHoldsAndRunningSteps(t *testing.T) {
 		StepStates:     map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()},
 		CandidateState: fakeState{ckey: cand}.read(),
 		VerdictState:   fakeState{}.read(),
+		HaveState:      fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
+		Now:            func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
 	}
 	st, err := rd.Status()
 	if err != nil {
@@ -165,3 +167,77 @@ func TestTheStatusToolReadsCapacityHoldsAndRunningSteps(t *testing.T) {
 		t.Errorf("empty status = %+v, %v", st, err)
 	}
 }
+
+func haveOf(t *testing.T, phase delivery.Phase) []byte {
+	t.Helper()
+	raw, err := delivery.EncodeHave(delivery.Have{Phase: phase, Health: delivery.HealthOK})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return raw
+}
+
+func stepOf(t *testing.T, id, node string, started time.Time) []byte {
+	t.Helper()
+	raw, err := delivery.EncodeStep(delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: delivery.StepRunning, Started: started})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return raw
+}
+
+// 1890: a key a build module left running (it stopped, restarted or was unassigned before it wrote the step's end) holds
+// no place in the checker, whose wait for it ended at the step bound; the status says it is left running and why, and
+// counts only the steps that can still run.
+func TestTheStatusCountsOnlyStepsThatCanStillRunAndSaysWhyTheOthersAreLeft(t *testing.T) { // 1890/AC2, AC3
+	now := time.Date(2026, 10, 23, 7, 0, 0, 0, time.UTC)
+	rd := Reader{
+		StepStates: map[string]StateRead{
+			"mesh-build-go": fakeState{
+				"g14.fresh1":   stepOf(t, "fresh1", "g14", now.Add(-5*time.Minute)),
+				"g14.old1":     stepOf(t, "old1", "g14", now.Add(-33*time.Hour)),
+				"ace.gone1":    stepOf(t, "gone1", "ace", now.Add(-2*time.Minute)),
+				"shanks.down1": stepOf(t, "down1", "shanks", now.Add(-time.Minute)),
+			}.read(),
+		},
+		HaveState: fakeState{
+			"g14.mesh-build-go":    haveOf(t, delivery.PhaseRunning),
+			"shanks.mesh-build-go": haveOf(t, delivery.PhaseFailed),
+		}.read(),
+		StepBound: 30 * time.Minute,
+		Now:       func() time.Time { return now },
+	}
+	st, err := rd.Status()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(st.Running) != 1 || st.Running[0].ID != "fresh1" || len(st.RunningByNode) != 1 || st.RunningByNode["g14"] != 1 {
+		t.Fatalf("running = %+v by node %v", st.Running, st.RunningByNode)
+	}
+	why := map[string]string{}
+	for _, l := range st.LeftRunning {
+		why[l.ID] = l.Why
+	}
+	if len(why) != 3 {
+		t.Fatalf("left running = %+v", st.LeftRunning)
+	}
+	for id, want := range map[string]string{"old1": "past the step bound of 30m0s", "gone1": "mesh-build-go is not on ace", "down1": "mesh-build-go on shanks is failed"} {
+		if !strings.Contains(why[id], want) {
+			t.Errorf("why %s = %q, want it to say %q", id, why[id], want)
+		}
+	}
+}
+
+// 1890: a group whose pull requests are all hq's is never judged by the checker (merge-check.sh judges hq), so it is not
+// a candidate held back.
+func TestTheStatusDoesNotCountAnHQOnlyGroupAsUnjudged(t *testing.T) { // 1890/AC4
+	cand := candidateOf(t, "pb-x", pullOf(HQRepository, 3, head2))
+	ckey, _ := delivery.CandidatesKey("pb-x")
+	st, err := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{}.read()}.Status()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(st.Unjudged) != 0 {
+		t.Errorf("unjudged = %v", st.Unjudged)
+	}
+}