issue 1841/S11: judge the upgrades' health from the runs, rounds and held kernels on the bus, so a node that slept through a run is judged by what happened, not by a unit's age (folds issue 1579)

modules/pacman/cmd/pacman-tools/main.go

@@ -236,13 +236,11 @@ func tools(m *Machine) []stdio.Tool {
 		},
 		{
 			Name: "pacman_upgrade_health",
-			Description: "Whether the weekly full upgrade (Saturday 04:00 local time, ADR 0477) last passed, as the module's health check: unhealthy " +
+			Description: "This node's upgrades, read from the bus, as the module's health check (ADR 0508 §11): unhealthy while no upgrade-order " +
-				"with the exit and the last error of the journal when it failed, healthy while it runs, after it passed and before it ever ran. (r)",
+				"is set, while the node's latest run is refused, failed or did not pass its after-upgrade check, hung past two hours, or held a " +
+				"kernel, and while the week's round stands still at this node or at the node it waits for, naming it and since when. (r)",
 			Input: schema(map[string]any{}),
 			Run: func(map[string]any) (any, error) {
-				if h, said := m.InstalledHealth(); said {
-					return h, nil
-				}
 				return m.UpgradeHealth(), nil
 			},
 		},

modules/pacman/cmd/pacman-tools/pacman_test.go

@@ -319,57 +319,3 @@ func TestConfigIsReadAsPacmanConfResolvesIt(t *testing.T) {
 		t.Fatalf("%+v", c)
 	}
 }
-
-// The weekly upgrade's outcome is the module's health (issue 1574, ADR 0477 clause 5).
-func upgradeHealthMachine(props, journal string) *Machine {
-	return machine(byLine(map[string]Ran{
-		"systemctl show " + WeeklyUnit + " --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecMainExitTimestamp --property=ExecMainStartTimestamp": {Stdout: props},
-		"sudo -n journalctl --no-pager -o cat -n 40 -u " + WeeklyUnit: {Stdout: journal},
-		"sudo -n journalctl --no-pager -o cat -n 1 -u " + WeeklyUnit:  {Stdout: journal},
-	}, nil), 1000)
-}
-
-func TestAFailedWeeklyUpgradeIsUnhealthyNamingTheError(t *testing.T) {
-	h := upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=Sat 2026-10-24 04:03:11 CEST\n",
-		":: Synchronizing package databases...\nerror: failed to commit transaction (conflicting files)\nErrors occurred, no packages were upgraded.\n").UpgradeHealth()
-	if h.Healthy || !strings.Contains(h.Why, "failed to commit transaction (conflicting files)") || !strings.Contains(h.Why, "exit-code") || !strings.Contains(h.Why, "Sat 2026-10-24 04:03:11 CEST") {
-		t.Fatalf("%+v", h)
-	}
-	h = upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=\n", "").UpgradeHealth()
-	if h.Healthy || !strings.Contains(h.Why, "status 1") {
-		t.Fatalf("a failure without a journal is still unhealthy: %+v", h)
-	}
-}
-
-func TestAPassedRunningOrNeverRunWeeklyUpgradeIsHealthy(t *testing.T) {
-	for name, props := range map[string]string{
-		"passed":    "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=Sat 2026-10-24 04:09:00 CEST\n",
-		"running":   "LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n",
-		"never run": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n",
-	} {
-		if h := upgradeHealthMachine(props, "").UpgradeHealth(); !h.Healthy {
-			t.Errorf("%s: %+v", name, h)
-		}
-	}
-	if h := machine(fake(func(call) Ran { return Ran{Status: 1, Stderr: "boom"} }, nil), 1000).UpgradeHealth(); h.Healthy {
-		t.Errorf("a unit that cannot be read is not healthy: %+v", h)
-	}
-}
-
-// A run that goes on past RunLimit is hung and holds pacman's lock: unhealthy, naming its start (issue 1574).
-func TestAHungWeeklyUpgradeIsUnhealthyNamingItsStart(t *testing.T) {
-	// the test machine's clock is 2026-10-04 12:00 UTC
-	h := upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 09:00:00 UTC\n",
-		"downloading linux-6.99...\n").UpgradeHealth()
-	if h.Healthy || !strings.Contains(h.Why, "Wed 2026-10-04 09:00:00 UTC") || !strings.Contains(h.Why, "downloading linux-6.99") {
-		t.Fatalf("started 3 h ago: %+v", h)
-	}
-	h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", "").UpgradeHealth()
-	if !h.Healthy {
-		t.Fatalf("started 10 min ago: %+v", h)
-	}
-	h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=\n", "").UpgradeHealth()
-	if h.Healthy {
-		t.Fatalf("a start that cannot be read is not healthy: %+v", h)
-	}
-}

modules/pacman/cmd/pacman-tools/upgrade_health.go

@@ -1,18 +1,30 @@
 package main
 
+// The module's health, read from the bus (novox/hq ADR 0508 §11; issue 1579). The health check of the keeper's
+// process reads this node's latest run, the week's round and any held kernel from the module's states, never a
+// unit's age, so a laptop that slept through a run, or a node that was off, is judged by what the bus holds. It
+// is unhealthy, naming why, when:
+//
+//   - `installed.<node>` is marked not current (ADR 0507 §2);
+//   - no `upgrade-order` is set: no round starts, and every node says so (ADR 0508 §5);
+//   - the node's latest run was refused, failed, or its after-upgrade check did not pass, until a later run passes;
+//   - the latest run's transaction goes on past RunLimit: hung, holding pacman's lock;
+//   - the latest run held a kernel, naming the package, the DKMS module and its bound (ADR 0508 §3);
+//   - the week's round stands still: this node's turn came more than round-start-minutes ago and it has not
+//     started, or the node it waits for stalled or failed, named with since when (ADR 0508 §6).
+//
+// A state that cannot be read is unhealthy: failure is loud.
+
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
+	"os"
+	"sort"
 	"strings"
 	"time"
 )
 
-// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node
-// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager
-// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of
-// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and,
-// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an
-// upgrade passes.
-
 // Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why.
 type Health struct {
 	Healthy bool     `json:"healthy"`
@@ -20,66 +32,190 @@ type Health struct {
 	Found   []string `json:"found,omitempty"`
 }
 
-// WeeklyUnit is the service the weekly timer starts.
+// RunLimit is how long a run's transaction may go on before it is unhealthy. A transaction has no timeout on
-const WeeklyUnit = "mesh-pacman-weekly-upgrade.service"
+// purpose: killing pacman in the middle of one leaves a half-upgraded machine. One past the limit is a hung one
-
+// (a stuck hook, a stalled download), which holds pacman's database lock so no later run can start; it is said,
-// RunLimit is how long a weekly upgrade may go on before it is unhealthy. The service has no start timeout on
+// not killed.
-// purpose: killing pacman in the middle of a transaction leaves a half-upgraded machine. A run past the limit
-// is a hung one (a stuck hook, a stalled transaction), which holds pacman's database lock so that no later run
-// can start; it is said, not killed.
 const RunLimit = 2 * time.Hour
 
-// systemdTime is how systemctl shows a timestamp, in the machine's local time.
+// healthTime is how the health writes a moment: the machine's local time.
-const systemdTime = "Mon 2006-01-02 15:04:05 MST"
+const healthTime = "2006-01-02 15:04 MST"
 
-// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit
+// keeperConfig is the keeper's config as the tools bundle reads it, from the file the manifest names to the
-// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A
+// bundle too (EnvConfig); a test replaces it. A file that cannot be read is an empty config: no order.
-// unit that cannot be read is unhealthy: failure is loud.
+var keeperConfig = func() KeeperConfig {
-func (m *Machine) UpgradeHealth() Health {
+	var c KeeperConfig
-	p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp", "ExecMainStartTimestamp")
+	if raw, err := os.ReadFile(os.Getenv(EnvConfig)); err == nil {
-	if err != nil {
+		_ = json.Unmarshal(raw, &c)
-		return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()}
 	}
-	switch {
+	return c
-	case p["LoadState"] == "not-found":
-		return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"}
-	case p["ActiveState"] == "activating" || p["ActiveState"] == "active":
-		return m.runningHealth(p["ExecMainStartTimestamp"])
-	case p["ActiveState"] != "failed" && p["Result"] == "success":
-		if at := p["ExecMainExitTimestamp"]; at != "" {
-			return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"}
-		}
-		return Health{Healthy: true, Why: "no weekly upgrade has run yet"}
-	}
-	why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"])
-	if at := p["ExecMainExitTimestamp"]; at != "" {
-		why += " at " + at
-	}
-	found := []string{}
-	if said := lastError(m.journal(WeeklyUnit, 40)); said != "" {
-		why += ": " + said
-		found = append(found, said)
-	}
-	return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found}
 }
 
-// runningHealth judges a run in progress: healthy within RunLimit of its start, unhealthy past it, naming the
+// readOnly is a state the tools bundle reads, as the round's readers take it; it writes nothing.
-// start and the last line the journal holds.
+type readOnly struct{ StateReader }
-func (m *Machine) runningHealth(started string) Health {
+
-	at, err := time.ParseInLocation(systemdTime, started, time.Local)
+var errReadOnly = errors.New("the health writes nothing")
+
+func (readOnly) Create(string, any) (uint64, error)         { return 0, errReadOnly }
+func (readOnly) Update(string, any, uint64) (uint64, error) { return 0, errReadOnly }
+func (readOnly) Put(string, any) (uint64, error)            { return 0, errReadOnly }
+func (readOnly) Delete(string) error                        { return errReadOnly }
+
+// UpgradeHealth is the health of this node's upgrades, from the module's states.
+func (m *Machine) UpgradeHealth() Health {
+	if h, said := m.InstalledHealth(); said {
+		return h
+	}
+	if m.States == nil || m.Node == "" {
+		return Health{Why: "the tools bundle was started without its node (MESH_NODE) or the bus: the upgrades cannot be read"}
+	}
+	cfg := keeperConfig()
+	if len(cfg.UpgradeOrder) == 0 {
+		return Health{Why: "no upgrade-order is set: no node upgrades, since no round starts until the mesh-level key " +
+			"upgrade-order names the nodes in the order they upgrade (ADR 0508 §5)"}
+	}
+	runs := readOnly{m.States(StateRuns)}
+	latest, err := latestOwnRun(runs, m.Node)
 	if err != nil {
-		return Health{Why: fmt.Sprintf("the weekly upgrade is running and when it started could not be read (%q)", started)}
+		return Health{Why: "the node's runs could not be read: " + err.Error()}
 	}
-	if m.Now().Sub(at) <= RunLimit {
+	if latest != nil {
-		return Health{Healthy: true, Why: "the weekly upgrade is running, since " + started}
+		if h, said := m.runHealth(*latest); said {
+			return h
+		}
 	}
-	why := fmt.Sprintf("the weekly upgrade has been running since %s, longer than %s: it is hung, and holds pacman's database lock so no later upgrade can start", started, RunLimit)
+	if h, said := m.roundHealth(readOnly{m.States(StateRounds)}, runs, cfg.RoundStartMinutes); said {
-	found := []string{}
+		return h
-	if ls := m.journal(WeeklyUnit, 1); len(ls) > 0 && !strings.HasPrefix(ls[0], "(the journal could not be read") {
-		why += "; its last line: " + strings.TrimSpace(ls[0])
-		found = append(found, strings.TrimSpace(ls[0]))
 	}
-	return Health{Why: why, Found: found}
+	if latest == nil {
+		return Health{Healthy: true, Why: "no upgrade has run on this node through the mesh yet"}
+	}
+	return Health{Healthy: true, Why: fmt.Sprintf("the last run, %s, passed its after-upgrade check", latest.Run)}
+}
+
+// latestOwnRun is the node's newest run that the mesh ran, or nil; runs read from pacman's log are history.
+func latestOwnRun(runs Bucket, node string) (*Run, error) {
+	keys, err := runs.Keys()
+	if err != nil {
+		return nil, err
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(keys)))
+	for _, k := range keys {
+		if !strings.HasPrefix(k, node+".") {
+			continue
+		}
+		e, err := runs.Get(k)
+		if err != nil {
+			return nil, err
+		}
+		if e == nil {
+			continue
+		}
+		var r Run
+		if err := json.Unmarshal(e.Value, &r); err != nil {
+			return nil, fmt.Errorf("run %s: %w", k, err)
+		}
+		if r.From == FromLog || len(r.Steps) == 0 {
+			continue
+		}
+		return &r, nil
+	}
+	return nil, nil
+}
+
+// runHealth judges the node's latest run; said is false when the run has nothing to say against the node.
+func (m *Machine) runHealth(r Run) (Health, bool) {
+	var held []HeldKernel
+	var started *Step
+	for i, s := range r.Steps {
+		switch s.Step {
+		case StepCheckedBefore:
+			held = s.Held
+		case StepStarted:
+			started = &r.Steps[i]
+		}
+	}
+	what := r.Kind + " run " + r.Run
+	last := r.Steps[len(r.Steps)-1]
+	switch lastStep(r) {
+	case StepRefused:
+		return Health{Why: fmt.Sprintf("the %s did not start (%s): %s", what, last.At.Local().Format(healthTime), last.Why)}, true
+	case StepCheckedAfter:
+		if last.Check == nil || !last.Check.Passed {
+			why := fmt.Sprintf("the %s did not pass its after-upgrade check (%s)", what, last.At.Local().Format(healthTime))
+			for _, s := range r.Steps {
+				if s.Step == StepFailed {
+					why += ": its transaction failed: " + s.Error
+				}
+			}
+			if last.Check != nil {
+				for _, u := range last.Check.Unbuilt {
+					why += fmt.Sprintf("; %s %s is not built for kernel %s", u.Module, u.Version, u.Kernel)
+				}
+			}
+			return Health{Why: why + "; pacman_upgrades with the node and the run says it whole", Found: []string{r.Run}}, true
+		}
+	case StepStarted:
+		if started != nil && m.Now().Sub(started.At) > RunLimit {
+			return Health{Why: fmt.Sprintf("the %s's transaction (%s) has gone on since %s, longer than %s: it is hung, and holds pacman's database lock so no later run can start",
+				what, started.Unit, started.At.Local().Format(healthTime), RunLimit)}, true
+		}
+	}
+	if len(held) > 0 {
+		var said []string
+		for _, h := range held {
+			said = append(said, fmt.Sprintf("%s %s (the DKMS module %s allows %s)", h.Package, h.Version, h.Module, h.Bound))
+		}
+		return Health{Why: "the " + what + " held back " + strings.Join(said, ", ") + ": the kernel waits until its DKMS module follows (ADR 0508 §3)"}, true
+	}
+	return Health{}, false
+}
+
+// roundHealth judges this node's place in the newest round; said is false when the round has nothing to say
+// against it.
+func (m *Machine) roundHealth(rounds, runs Bucket, startMinutes int) (Health, bool) {
+	id, err := latestRound(rounds)
+	if err != nil {
+		return Health{Why: "the rounds could not be read: " + err.Error()}, true
+	}
+	if id == "" {
+		return Health{}, false
+	}
+	v, err := readRound(rounds, runs, id)
+	if err != nil {
+		return Health{Why: "round " + id + " could not be read: " + err.Error()}, true
+	}
+	if v == nil || !contains(v.Round.Order, m.Node) {
+		return Health{}, false
+	}
+	if startMinutes < 1 {
+		startMinutes = defaultRoundStartMinutes
+	}
+	byNode := map[string]NodeInRound{}
+	for _, s := range RoundState(v, m.Now(), startMinutes) {
+		byNode[s.Node] = s
+	}
+	since := func(s NodeInRound) string {
+		if s.Since == nil {
+			return "an unknown time"
+		}
+		return s.Since.Local().Format(healthTime)
+	}
+	me := byNode[m.Node]
+	switch me.State {
+	case InRoundStalled:
+		return Health{Why: fmt.Sprintf("round %s: this node's turn came at %s and it has not started its run within %d minutes", id, since(me), startMinutes)}, true
+	case InRoundWaiting:
+		if w, ok := byNode[me.WaitsFor]; ok {
+			switch w.State {
+			case InRoundStalled:
+				return Health{Why: fmt.Sprintf("round %s stands still: this node waits for %s, whose turn came at %s and which has not started (since %s)", id, w.Node, since(w), since(w))}, true
+			case InRoundFailed:
+				return Health{Why: fmt.Sprintf("round %s stopped at %s, whose run or check failed at %s: this node waits on it since then; only the operator passes it (pacman_round with pass and a reason)", id, w.Node, since(w))}, true
+			}
+		}
+	}
+	return Health{}, false
 }
 
 // lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that

modules/pacman/cmd/pacman-tools/upgrade_health_test.go

@@ -0,0 +1,147 @@
+package main
+
+// The module's health read from the bus (novox/hq ADR 0508 §3, §5, §6, §11; acceptance 1841/AC6, AC7, AC11, and
+// issue 1579): the node's latest run, its round and any held kernel are read from the module's states, never a
+// unit's age, so a laptop that slept through a run is judged by what the bus holds.
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+// the test machine's clock is 2026-10-04 12:00 UTC (machine_test.go)
+var healthNow = time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)
+
+func healthBed(t *testing.T, order []string) (*Machine, *fakeStates) {
+	t.Helper()
+	states := &fakeStates{}
+	m := machine(nil, 1000)
+	m.Node, m.States = "node-b", readerOf(states)
+	saved := keeperConfig
+	keeperConfig = func() KeeperConfig { return KeeperConfig{UpgradeOrder: order, RoundStartMinutes: 120, RecordDays: 180} }
+	t.Cleanup(func() { keeperConfig = saved })
+	return m, states
+}
+
+func putRun(states *fakeStates, r Run) { states.store(StateRuns).Put(NodeKey(r.Node, r.Run), r) }
+
+func passedRun(node string, at time.Time) Run {
+	return Run{Node: node, Run: RunID(at), Kind: RunUpgrade, From: FromRound, Steps: []Step{
+		{Step: StepRecorded, At: at}, {Step: StepCheckedBefore, At: at}, {Step: StepStarted, At: at},
+		{Step: StepFinished, At: at.Add(5 * time.Minute)}, {Step: StepCheckedAfter, At: at.Add(6 * time.Minute), Check: &Check{Passed: true}}}}
+}
+
+func TestWithNoOrderEveryNodesHealthSaysWhy(t *testing.T) {
+	m, _ := healthBed(t, nil)
+	if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "upgrade-order") {
+		t.Fatalf("%+v", h)
+	}
+}
+
+func TestANodeWithNoRunYetIsHealthy(t *testing.T) {
+	m, _ := healthBed(t, []string{"node-a", "node-b"})
+	if h := m.UpgradeHealth(); !h.Healthy {
+		t.Fatalf("%+v", h)
+	}
+}
+
+// issue 1579: a run that failed is unhealthy until a later run passes, whenever the node woke; a run read from
+// the log is history and judges nothing.
+func TestTheLatestRunJudgesTheNodeNotAUnitsAge(t *testing.T) {
+	m, states := healthBed(t, []string{"node-a", "node-b"})
+	exit := 1
+	failedAt := healthNow.Add(-7 * 24 * time.Hour)
+	putRun(states, Run{Node: "node-b", Run: RunID(failedAt), Kind: RunUpgrade, From: FromRound, Steps: []Step{
+		{Step: StepStarted, At: failedAt}, {Step: StepFailed, At: failedAt, Exit: &exit, Error: "error: failed to commit transaction (conflicting files)"},
+		{Step: StepCheckedAfter, At: failedAt, Check: &Check{}}}})
+	putRun(states, Run{Node: "node-b", Run: RunID(healthNow.Add(-time.Hour)), Kind: RunUpgrade, From: FromLog,
+		Steps: []Step{{Step: StepStarted, At: healthNow.Add(-time.Hour)}, {Step: StepFinished, At: healthNow.Add(-time.Hour)}}})
+	putRun(states, passedRun("node-a", healthNow.Add(-time.Minute)))
+	h := m.UpgradeHealth()
+	if h.Healthy || !strings.Contains(h.Why, "conflicting files") || !strings.Contains(h.Why, RunID(failedAt)) {
+		t.Fatalf("a failed run: %+v", h)
+	}
+	putRun(states, passedRun("node-b", healthNow.Add(-30*time.Minute)))
+	if h := m.UpgradeHealth(); !h.Healthy {
+		t.Fatalf("a later run passed: %+v", h)
+	}
+}
+
+func TestAFailedCheckIsUnhealthy(t *testing.T) {
+	m, states := healthBed(t, []string{"node-b"})
+	r := passedRun("node-b", healthNow.Add(-time.Hour))
+	r.Steps[len(r.Steps)-1].Check = &Check{Passed: false, Unbuilt: []DKMSBuild{{Module: "made-up-dkms", Version: "1.0", Kernel: "7.3.1-arch1-1"}}}
+	putRun(states, r)
+	if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "after-upgrade check") {
+		t.Fatalf("%+v", h)
+	}
+}
+
+func TestARunPastTheLimitIsHungAndOneWithinItIsHealthy(t *testing.T) {
+	m, states := healthBed(t, []string{"node-b"})
+	started := healthNow.Add(-3 * time.Hour)
+	putRun(states, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepRecorded, At: started}, {Step: StepStarted, At: started, Unit: TransactionUnit(RunUpgrade, RunID(started))}}})
+	if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, started.Local().Format(healthTime)) {
+		t.Fatalf("3 h: %+v", h)
+	}
+	m2, states2 := healthBed(t, []string{"node-b"})
+	started = healthNow.Add(-10 * time.Minute)
+	putRun(states2, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepStarted, At: started}}})
+	if h := m2.UpgradeHealth(); !h.Healthy {
+		t.Fatalf("10 min: %+v", h)
+	}
+}
+
+// 1841/AC4: a held kernel keeps the health unhealthy, naming it, until a later run upgrades it.
+func TestAHeldKernelIsSaidUntilARunUpgradesIt(t *testing.T) {
+	m, states := healthBed(t, []string{"node-b"})
+	r := passedRun("node-b", healthNow.Add(-2*time.Hour))
+	r.Steps[1].Held = []HeldKernel{{Package: "linux", Version: "7.3.1.arch1-1", Module: "zfs", Bound: "Linux-Maximum 7.2"}}
+	putRun(states, r)
+	h := m.UpgradeHealth()
+	if h.Healthy || !strings.Contains(h.Why, "linux 7.3.1.arch1-1") || !strings.Contains(h.Why, "zfs") || !strings.Contains(h.Why, "Linux-Maximum 7.2") {
+		t.Fatalf("%+v", h)
+	}
+	putRun(states, passedRun("node-b", healthNow.Add(-time.Hour)))
+	if h := m.UpgradeHealth(); !h.Healthy {
+		t.Fatalf("a later run held nothing: %+v", h)
+	}
+}
+
+// 1841/AC7: a turn not started within round-start-minutes makes the waiting node's health name the node it
+// waits for and since when; the stalled node's own health says it too.
+func TestAStalledRoundIsSaidByTheNodesWaitingOnIt(t *testing.T) {
+	order := []string{"node-a", "node-b", "node-c"}
+	m, states := healthBed(t, order)
+	round := RoundID(healthNow)
+	opened := healthNow.Add(-3 * time.Hour)
+	states.store(StateRounds).Put(round, Round{Round: round, Opened: opened, OpenedBy: "node-a", Order: order})
+	putRun(states, passedRun("node-a", opened))
+	states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: RunID(opened)})
+	// node-a passed 2 h 54 min ago; node-b never started.
+	h := m.UpgradeHealth()
+	if h.Healthy || !strings.Contains(h.Why, "has not started") {
+		t.Fatalf("the stalled node: %+v", h)
+	}
+	m.Node = "node-c"
+	h = m.UpgradeHealth()
+	if h.Healthy || !strings.Contains(h.Why, "node-b") || !strings.Contains(h.Why, "since") {
+		t.Fatalf("the node waiting on it: %+v", h)
+	}
+}
+
+func TestARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt(t *testing.T) {
+	order := []string{"node-a", "node-b"}
+	m, states := healthBed(t, order)
+	round := RoundID(healthNow)
+	at := healthNow.Add(-time.Hour)
+	states.store(StateRounds).Put(round, Round{Round: round, Opened: at, OpenedBy: "node-a", Order: order})
+	r := passedRun("node-a", at)
+	r.Steps[len(r.Steps)-1].Check = &Check{Passed: false}
+	putRun(states, r)
+	states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: r.Run})
+	if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "node-a") || !strings.Contains(h.Why, "stopped") {
+		t.Fatalf("%+v", h)
+	}
+}