148 lines
6.6 KiB
Go
148 lines
6.6 KiB
Go
package main
|
|
|
|
// The module's health read from the bus (novox/hq ADR 0508 §3, §5, §6, §11; acceptance 1841/AC6, AC7, AC11, and
|
|
// issue 1579): the node's latest run, its round and any held kernel are read from the module's states, never a
|
|
// unit's age, so a laptop that slept through a run is judged by what the bus holds.
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// the test machine's clock is 2026-10-04 12:00 UTC (machine_test.go)
|
|
var healthNow = time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)
|
|
|
|
func healthBed(t *testing.T, order []string) (*Machine, *fakeStates) {
|
|
t.Helper()
|
|
states := &fakeStates{}
|
|
m := machine(nil, 1000)
|
|
m.Node, m.States = "node-b", readerOf(states)
|
|
saved := keeperConfig
|
|
keeperConfig = func() KeeperConfig { return KeeperConfig{UpgradeOrder: order, RoundStartMinutes: 120, RecordDays: 180} }
|
|
t.Cleanup(func() { keeperConfig = saved })
|
|
return m, states
|
|
}
|
|
|
|
func putRun(states *fakeStates, r Run) { states.store(StateRuns).Put(NodeKey(r.Node, r.Run), r) }
|
|
|
|
func passedRun(node string, at time.Time) Run {
|
|
return Run{Node: node, Run: RunID(at), Kind: RunUpgrade, From: FromRound, Steps: []Step{
|
|
{Step: StepRecorded, At: at}, {Step: StepCheckedBefore, At: at}, {Step: StepStarted, At: at},
|
|
{Step: StepFinished, At: at.Add(5 * time.Minute)}, {Step: StepCheckedAfter, At: at.Add(6 * time.Minute), Check: &Check{Passed: true}}}}
|
|
}
|
|
|
|
func TestWithNoOrderEveryNodesHealthSaysWhy(t *testing.T) {
|
|
m, _ := healthBed(t, nil)
|
|
if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "upgrade-order") {
|
|
t.Fatalf("%+v", h)
|
|
}
|
|
}
|
|
|
|
func TestANodeWithNoRunYetIsHealthy(t *testing.T) {
|
|
m, _ := healthBed(t, []string{"node-a", "node-b"})
|
|
if h := m.UpgradeHealth(); !h.Healthy {
|
|
t.Fatalf("%+v", h)
|
|
}
|
|
}
|
|
|
|
// issue 1579: a run that failed is unhealthy until a later run passes, whenever the node woke; a run read from
|
|
// the log is history and judges nothing.
|
|
func TestTheLatestRunJudgesTheNodeNotAUnitsAge(t *testing.T) {
|
|
m, states := healthBed(t, []string{"node-a", "node-b"})
|
|
exit := 1
|
|
failedAt := healthNow.Add(-7 * 24 * time.Hour)
|
|
putRun(states, Run{Node: "node-b", Run: RunID(failedAt), Kind: RunUpgrade, From: FromRound, Steps: []Step{
|
|
{Step: StepStarted, At: failedAt}, {Step: StepFailed, At: failedAt, Exit: &exit, Error: "error: failed to commit transaction (conflicting files)"},
|
|
{Step: StepCheckedAfter, At: failedAt, Check: &Check{}}}})
|
|
putRun(states, Run{Node: "node-b", Run: RunID(healthNow.Add(-time.Hour)), Kind: RunUpgrade, From: FromLog,
|
|
Steps: []Step{{Step: StepStarted, At: healthNow.Add(-time.Hour)}, {Step: StepFinished, At: healthNow.Add(-time.Hour)}}})
|
|
putRun(states, passedRun("node-a", healthNow.Add(-time.Minute)))
|
|
h := m.UpgradeHealth()
|
|
if h.Healthy || !strings.Contains(h.Why, "conflicting files") || !strings.Contains(h.Why, RunID(failedAt)) {
|
|
t.Fatalf("a failed run: %+v", h)
|
|
}
|
|
putRun(states, passedRun("node-b", healthNow.Add(-30*time.Minute)))
|
|
if h := m.UpgradeHealth(); !h.Healthy {
|
|
t.Fatalf("a later run passed: %+v", h)
|
|
}
|
|
}
|
|
|
|
func TestAFailedCheckIsUnhealthy(t *testing.T) {
|
|
m, states := healthBed(t, []string{"node-b"})
|
|
r := passedRun("node-b", healthNow.Add(-time.Hour))
|
|
r.Steps[len(r.Steps)-1].Check = &Check{Passed: false, Unbuilt: []DKMSBuild{{Module: "made-up-dkms", Version: "1.0", Kernel: "7.3.1-arch1-1"}}}
|
|
putRun(states, r)
|
|
if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "after-upgrade check") {
|
|
t.Fatalf("%+v", h)
|
|
}
|
|
}
|
|
|
|
func TestARunPastTheLimitIsHungAndOneWithinItIsHealthy(t *testing.T) {
|
|
m, states := healthBed(t, []string{"node-b"})
|
|
started := healthNow.Add(-3 * time.Hour)
|
|
putRun(states, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepRecorded, At: started}, {Step: StepStarted, At: started, Unit: TransactionUnit(RunUpgrade, RunID(started))}}})
|
|
if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, started.Local().Format(healthTime)) {
|
|
t.Fatalf("3 h: %+v", h)
|
|
}
|
|
m2, states2 := healthBed(t, []string{"node-b"})
|
|
started = healthNow.Add(-10 * time.Minute)
|
|
putRun(states2, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepStarted, At: started}}})
|
|
if h := m2.UpgradeHealth(); !h.Healthy {
|
|
t.Fatalf("10 min: %+v", h)
|
|
}
|
|
}
|
|
|
|
// 1841/AC4: a held kernel keeps the health unhealthy, naming it, until a later run upgrades it.
|
|
func TestAHeldKernelIsSaidUntilARunUpgradesIt(t *testing.T) {
|
|
m, states := healthBed(t, []string{"node-b"})
|
|
r := passedRun("node-b", healthNow.Add(-2*time.Hour))
|
|
r.Steps[1].Held = []HeldKernel{{Package: "linux", Version: "7.3.1.arch1-1", Module: "zfs", Bound: "Linux-Maximum 7.2"}}
|
|
putRun(states, r)
|
|
h := m.UpgradeHealth()
|
|
if h.Healthy || !strings.Contains(h.Why, "linux 7.3.1.arch1-1") || !strings.Contains(h.Why, "zfs") || !strings.Contains(h.Why, "Linux-Maximum 7.2") {
|
|
t.Fatalf("%+v", h)
|
|
}
|
|
putRun(states, passedRun("node-b", healthNow.Add(-time.Hour)))
|
|
if h := m.UpgradeHealth(); !h.Healthy {
|
|
t.Fatalf("a later run held nothing: %+v", h)
|
|
}
|
|
}
|
|
|
|
// 1841/AC7: a turn not started within round-start-minutes makes the waiting node's health name the node it
|
|
// waits for and since when; the stalled node's own health says it too.
|
|
func TestAStalledRoundIsSaidByTheNodesWaitingOnIt(t *testing.T) {
|
|
order := []string{"node-a", "node-b", "node-c"}
|
|
m, states := healthBed(t, order)
|
|
round := RoundID(healthNow)
|
|
opened := healthNow.Add(-3 * time.Hour)
|
|
states.store(StateRounds).Put(round, Round{Round: round, Opened: opened, OpenedBy: "node-a", Order: order})
|
|
putRun(states, passedRun("node-a", opened))
|
|
states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: RunID(opened)})
|
|
// node-a passed 2 h 54 min ago; node-b never started.
|
|
h := m.UpgradeHealth()
|
|
if h.Healthy || !strings.Contains(h.Why, "has not started") {
|
|
t.Fatalf("the stalled node: %+v", h)
|
|
}
|
|
m.Node = "node-c"
|
|
h = m.UpgradeHealth()
|
|
if h.Healthy || !strings.Contains(h.Why, "node-b") || !strings.Contains(h.Why, "since") {
|
|
t.Fatalf("the node waiting on it: %+v", h)
|
|
}
|
|
}
|
|
|
|
func TestARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt(t *testing.T) {
|
|
order := []string{"node-a", "node-b"}
|
|
m, states := healthBed(t, order)
|
|
round := RoundID(healthNow)
|
|
at := healthNow.Add(-time.Hour)
|
|
states.store(StateRounds).Put(round, Round{Round: round, Opened: at, OpenedBy: "node-a", Order: order})
|
|
r := passedRun("node-a", at)
|
|
r.Steps[len(r.Steps)-1].Check = &Check{Passed: false}
|
|
putRun(states, r)
|
|
states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: r.Run})
|
|
if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "node-a") || !strings.Contains(h.Why, "stopped") {
|
|
t.Fatalf("%+v", h)
|
|
}
|
|
}
|