package main // The module's health read from the bus (novox/hq ADR 0508 §3, §5, §6, §11; acceptance 1841/AC6, AC7, AC11, and // issue 1579): the node's latest run, its round and any held kernel are read from the module's states, never a // unit's age, so a laptop that slept through a run is judged by what the bus holds. import ( "strings" "testing" "time" ) // the test machine's clock is 2026-10-04 12:00 UTC (machine_test.go) var healthNow = time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC) func healthBed(t *testing.T, order []string) (*Machine, *fakeStates) { t.Helper() states := &fakeStates{} m := machine(nil, 1000) m.Node, m.States = "node-b", readerOf(states) saved := keeperConfig keeperConfig = func() KeeperConfig { return KeeperConfig{UpgradeOrder: order, RoundStartMinutes: 120, RecordDays: 180} } t.Cleanup(func() { keeperConfig = saved }) return m, states } func putRun(states *fakeStates, r Run) { states.store(StateRuns).Put(NodeKey(r.Node, r.Run), r) } func passedRun(node string, at time.Time) Run { return Run{Node: node, Run: RunID(at), Kind: RunUpgrade, From: FromRound, Steps: []Step{ {Step: StepRecorded, At: at}, {Step: StepCheckedBefore, At: at}, {Step: StepStarted, At: at}, {Step: StepFinished, At: at.Add(5 * time.Minute)}, {Step: StepCheckedAfter, At: at.Add(6 * time.Minute), Check: &Check{Passed: true}}}} } func TestWithNoOrderEveryNodesHealthSaysWhy(t *testing.T) { m, _ := healthBed(t, nil) if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "upgrade-order") { t.Fatalf("%+v", h) } } func TestANodeWithNoRunYetIsHealthy(t *testing.T) { m, _ := healthBed(t, []string{"node-a", "node-b"}) if h := m.UpgradeHealth(); !h.Healthy { t.Fatalf("%+v", h) } } // issue 1579: a run that failed is unhealthy until a later run passes, whenever the node woke; a run read from // the log is history and judges nothing. func TestTheLatestRunJudgesTheNodeNotAUnitsAge(t *testing.T) { m, states := healthBed(t, []string{"node-a", "node-b"}) exit := 1 failedAt := healthNow.Add(-7 * 24 * time.Hour) putRun(states, Run{Node: "node-b", Run: RunID(failedAt), Kind: RunUpgrade, From: FromRound, Steps: []Step{ {Step: StepStarted, At: failedAt}, {Step: StepFailed, At: failedAt, Exit: &exit, Error: "error: failed to commit transaction (conflicting files)"}, {Step: StepCheckedAfter, At: failedAt, Check: &Check{}}}}) putRun(states, Run{Node: "node-b", Run: RunID(healthNow.Add(-time.Hour)), Kind: RunUpgrade, From: FromLog, Steps: []Step{{Step: StepStarted, At: healthNow.Add(-time.Hour)}, {Step: StepFinished, At: healthNow.Add(-time.Hour)}}}) putRun(states, passedRun("node-a", healthNow.Add(-time.Minute))) h := m.UpgradeHealth() if h.Healthy || !strings.Contains(h.Why, "conflicting files") || !strings.Contains(h.Why, RunID(failedAt)) { t.Fatalf("a failed run: %+v", h) } putRun(states, passedRun("node-b", healthNow.Add(-30*time.Minute))) if h := m.UpgradeHealth(); !h.Healthy { t.Fatalf("a later run passed: %+v", h) } } func TestAFailedCheckIsUnhealthy(t *testing.T) { m, states := healthBed(t, []string{"node-b"}) r := passedRun("node-b", healthNow.Add(-time.Hour)) r.Steps[len(r.Steps)-1].Check = &Check{Passed: false, Unbuilt: []DKMSBuild{{Module: "made-up-dkms", Version: "1.0", Kernel: "7.3.1-arch1-1"}}} putRun(states, r) if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "after-upgrade check") { t.Fatalf("%+v", h) } } func TestARunPastTheLimitIsHungAndOneWithinItIsHealthy(t *testing.T) { m, states := healthBed(t, []string{"node-b"}) started := healthNow.Add(-3 * time.Hour) putRun(states, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepRecorded, At: started}, {Step: StepStarted, At: started, Unit: TransactionUnit(RunUpgrade, RunID(started))}}}) if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, started.Local().Format(healthTime)) { t.Fatalf("3 h: %+v", h) } m2, states2 := healthBed(t, []string{"node-b"}) started = healthNow.Add(-10 * time.Minute) putRun(states2, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepStarted, At: started}}}) if h := m2.UpgradeHealth(); !h.Healthy { t.Fatalf("10 min: %+v", h) } } // 1841/AC4: a held kernel keeps the health unhealthy, naming it, until a later run upgrades it. func TestAHeldKernelIsSaidUntilARunUpgradesIt(t *testing.T) { m, states := healthBed(t, []string{"node-b"}) r := passedRun("node-b", healthNow.Add(-2*time.Hour)) r.Steps[1].Held = []HeldKernel{{Package: "linux", Version: "7.3.1.arch1-1", Module: "zfs", Bound: "Linux-Maximum 7.2"}} putRun(states, r) h := m.UpgradeHealth() if h.Healthy || !strings.Contains(h.Why, "linux 7.3.1.arch1-1") || !strings.Contains(h.Why, "zfs") || !strings.Contains(h.Why, "Linux-Maximum 7.2") { t.Fatalf("%+v", h) } putRun(states, passedRun("node-b", healthNow.Add(-time.Hour))) if h := m.UpgradeHealth(); !h.Healthy { t.Fatalf("a later run held nothing: %+v", h) } } // 1841/AC7: a turn not started within round-start-minutes makes the waiting node's health name the node it // waits for and since when; the stalled node's own health says it too. func TestAStalledRoundIsSaidByTheNodesWaitingOnIt(t *testing.T) { order := []string{"node-a", "node-b", "node-c"} m, states := healthBed(t, order) round := RoundID(healthNow) opened := healthNow.Add(-3 * time.Hour) states.store(StateRounds).Put(round, Round{Round: round, Opened: opened, OpenedBy: "node-a", Order: order}) putRun(states, passedRun("node-a", opened)) states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: RunID(opened)}) // node-a passed 2 h 54 min ago; node-b never started. h := m.UpgradeHealth() if h.Healthy || !strings.Contains(h.Why, "has not started") { t.Fatalf("the stalled node: %+v", h) } m.Node = "node-c" h = m.UpgradeHealth() if h.Healthy || !strings.Contains(h.Why, "node-b") || !strings.Contains(h.Why, "since") { t.Fatalf("the node waiting on it: %+v", h) } } func TestARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt(t *testing.T) { order := []string{"node-a", "node-b"} m, states := healthBed(t, order) round := RoundID(healthNow) at := healthNow.Add(-time.Hour) states.store(StateRounds).Put(round, Round{Round: round, Opened: at, OpenedBy: "node-a", Order: order}) r := passedRun("node-a", at) r.Steps[len(r.Steps)-1].Check = &Check{Passed: false} putRun(states, r) states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: r.Run}) if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "node-a") || !strings.Contains(h.Why, "stopped") { t.Fatalf("%+v", h) } }