From f8ac498d6435ca16e72231aa2cdc3992ee4a13dc Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:38:08 +0200 Subject: [PATCH] issue 1841/S11: judge the upgrades' health from the runs, rounds and held kernels on the bus, so a node that slept through a run is judged by what happened, not by a unit's age (folds issue 1579) --- modules/pacman/cmd/pacman-tools/main.go | 8 +- .../pacman/cmd/pacman-tools/pacman_test.go | 54 ---- .../pacman/cmd/pacman-tools/upgrade_health.go | 250 ++++++++++++++---- .../cmd/pacman-tools/upgrade_health_test.go | 147 ++++++++++ 4 files changed, 343 insertions(+), 116 deletions(-) create mode 100644 modules/pacman/cmd/pacman-tools/upgrade_health_test.go diff --git a/modules/pacman/cmd/pacman-tools/main.go b/modules/pacman/cmd/pacman-tools/main.go index 65099734..1df5256b 100644 --- a/modules/pacman/cmd/pacman-tools/main.go +++ b/modules/pacman/cmd/pacman-tools/main.go @@ -236,13 +236,11 @@ func tools(m *Machine) []stdio.Tool { }, { Name: "pacman_upgrade_health", - Description: "Whether the weekly full upgrade (Saturday 04:00 local time, ADR 0477) last passed, as the module's health check: unhealthy " + - "with the exit and the last error of the journal when it failed, healthy while it runs, after it passed and before it ever ran. (r)", + Description: "This node's upgrades, read from the bus, as the module's health check (ADR 0508 §11): unhealthy while no upgrade-order " + + "is set, while the node's latest run is refused, failed or did not pass its after-upgrade check, hung past two hours, or held a " + + "kernel, and while the week's round stands still at this node or at the node it waits for, naming it and since when. (r)", Input: schema(map[string]any{}), Run: func(map[string]any) (any, error) { - if h, said := m.InstalledHealth(); said { - return h, nil - } return m.UpgradeHealth(), nil }, }, diff --git a/modules/pacman/cmd/pacman-tools/pacman_test.go b/modules/pacman/cmd/pacman-tools/pacman_test.go index f8b34b4d..0c7f1d43 100644 --- a/modules/pacman/cmd/pacman-tools/pacman_test.go +++ b/modules/pacman/cmd/pacman-tools/pacman_test.go @@ -319,57 +319,3 @@ func TestConfigIsReadAsPacmanConfResolvesIt(t *testing.T) { t.Fatalf("%+v", c) } } - -// The weekly upgrade's outcome is the module's health (issue 1574, ADR 0477 clause 5). -func upgradeHealthMachine(props, journal string) *Machine { - return machine(byLine(map[string]Ran{ - "systemctl show " + WeeklyUnit + " --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecMainExitTimestamp --property=ExecMainStartTimestamp": {Stdout: props}, - "sudo -n journalctl --no-pager -o cat -n 40 -u " + WeeklyUnit: {Stdout: journal}, - "sudo -n journalctl --no-pager -o cat -n 1 -u " + WeeklyUnit: {Stdout: journal}, - }, nil), 1000) -} - -func TestAFailedWeeklyUpgradeIsUnhealthyNamingTheError(t *testing.T) { - h := upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=Sat 2026-10-24 04:03:11 CEST\n", - ":: Synchronizing package databases...\nerror: failed to commit transaction (conflicting files)\nErrors occurred, no packages were upgraded.\n").UpgradeHealth() - if h.Healthy || !strings.Contains(h.Why, "failed to commit transaction (conflicting files)") || !strings.Contains(h.Why, "exit-code") || !strings.Contains(h.Why, "Sat 2026-10-24 04:03:11 CEST") { - t.Fatalf("%+v", h) - } - h = upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=\n", "").UpgradeHealth() - if h.Healthy || !strings.Contains(h.Why, "status 1") { - t.Fatalf("a failure without a journal is still unhealthy: %+v", h) - } -} - -func TestAPassedRunningOrNeverRunWeeklyUpgradeIsHealthy(t *testing.T) { - for name, props := range map[string]string{ - "passed": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=Sat 2026-10-24 04:09:00 CEST\n", - "running": "LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", - "never run": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n", - } { - if h := upgradeHealthMachine(props, "").UpgradeHealth(); !h.Healthy { - t.Errorf("%s: %+v", name, h) - } - } - if h := machine(fake(func(call) Ran { return Ran{Status: 1, Stderr: "boom"} }, nil), 1000).UpgradeHealth(); h.Healthy { - t.Errorf("a unit that cannot be read is not healthy: %+v", h) - } -} - -// A run that goes on past RunLimit is hung and holds pacman's lock: unhealthy, naming its start (issue 1574). -func TestAHungWeeklyUpgradeIsUnhealthyNamingItsStart(t *testing.T) { - // the test machine's clock is 2026-10-04 12:00 UTC - h := upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 09:00:00 UTC\n", - "downloading linux-6.99...\n").UpgradeHealth() - if h.Healthy || !strings.Contains(h.Why, "Wed 2026-10-04 09:00:00 UTC") || !strings.Contains(h.Why, "downloading linux-6.99") { - t.Fatalf("started 3 h ago: %+v", h) - } - h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", "").UpgradeHealth() - if !h.Healthy { - t.Fatalf("started 10 min ago: %+v", h) - } - h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=\n", "").UpgradeHealth() - if h.Healthy { - t.Fatalf("a start that cannot be read is not healthy: %+v", h) - } -} diff --git a/modules/pacman/cmd/pacman-tools/upgrade_health.go b/modules/pacman/cmd/pacman-tools/upgrade_health.go index 90180130..f73f1095 100644 --- a/modules/pacman/cmd/pacman-tools/upgrade_health.go +++ b/modules/pacman/cmd/pacman-tools/upgrade_health.go @@ -1,18 +1,30 @@ package main +// The module's health, read from the bus (novox/hq ADR 0508 §11; issue 1579). The health check of the keeper's +// process reads this node's latest run, the week's round and any held kernel from the module's states, never a +// unit's age, so a laptop that slept through a run, or a node that was off, is judged by what the bus holds. It +// is unhealthy, naming why, when: +// +// - `installed.` is marked not current (ADR 0507 §2); +// - no `upgrade-order` is set: no round starts, and every node says so (ADR 0508 §5); +// - the node's latest run was refused, failed, or its after-upgrade check did not pass, until a later run passes; +// - the latest run's transaction goes on past RunLimit: hung, holding pacman's lock; +// - the latest run held a kernel, naming the package, the DKMS module and its bound (ADR 0508 §3); +// - the week's round stands still: this node's turn came more than round-start-minutes ago and it has not +// started, or the node it waits for stalled or failed, named with since when (ADR 0508 §6). +// +// A state that cannot be read is unhealthy: failure is loud. + import ( + "encoding/json" + "errors" "fmt" + "os" + "sort" "strings" "time" ) -// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node -// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager -// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of -// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and, -// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an -// upgrade passes. - // Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why. type Health struct { Healthy bool `json:"healthy"` @@ -20,66 +32,190 @@ type Health struct { Found []string `json:"found,omitempty"` } -// WeeklyUnit is the service the weekly timer starts. -const WeeklyUnit = "mesh-pacman-weekly-upgrade.service" - -// RunLimit is how long a weekly upgrade may go on before it is unhealthy. The service has no start timeout on -// purpose: killing pacman in the middle of a transaction leaves a half-upgraded machine. A run past the limit -// is a hung one (a stuck hook, a stalled transaction), which holds pacman's database lock so that no later run -// can start; it is said, not killed. +// RunLimit is how long a run's transaction may go on before it is unhealthy. A transaction has no timeout on +// purpose: killing pacman in the middle of one leaves a half-upgraded machine. One past the limit is a hung one +// (a stuck hook, a stalled download), which holds pacman's database lock so no later run can start; it is said, +// not killed. const RunLimit = 2 * time.Hour -// systemdTime is how systemctl shows a timestamp, in the machine's local time. -const systemdTime = "Mon 2006-01-02 15:04:05 MST" +// healthTime is how the health writes a moment: the machine's local time. +const healthTime = "2006-01-02 15:04 MST" -// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit -// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A -// unit that cannot be read is unhealthy: failure is loud. -func (m *Machine) UpgradeHealth() Health { - p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp", "ExecMainStartTimestamp") - if err != nil { - return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()} +// keeperConfig is the keeper's config as the tools bundle reads it, from the file the manifest names to the +// bundle too (EnvConfig); a test replaces it. A file that cannot be read is an empty config: no order. +var keeperConfig = func() KeeperConfig { + var c KeeperConfig + if raw, err := os.ReadFile(os.Getenv(EnvConfig)); err == nil { + _ = json.Unmarshal(raw, &c) } - switch { - case p["LoadState"] == "not-found": - return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"} - case p["ActiveState"] == "activating" || p["ActiveState"] == "active": - return m.runningHealth(p["ExecMainStartTimestamp"]) - case p["ActiveState"] != "failed" && p["Result"] == "success": - if at := p["ExecMainExitTimestamp"]; at != "" { - return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"} - } - return Health{Healthy: true, Why: "no weekly upgrade has run yet"} - } - why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"]) - if at := p["ExecMainExitTimestamp"]; at != "" { - why += " at " + at - } - found := []string{} - if said := lastError(m.journal(WeeklyUnit, 40)); said != "" { - why += ": " + said - found = append(found, said) - } - return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found} + return c } -// runningHealth judges a run in progress: healthy within RunLimit of its start, unhealthy past it, naming the -// start and the last line the journal holds. -func (m *Machine) runningHealth(started string) Health { - at, err := time.ParseInLocation(systemdTime, started, time.Local) +// readOnly is a state the tools bundle reads, as the round's readers take it; it writes nothing. +type readOnly struct{ StateReader } + +var errReadOnly = errors.New("the health writes nothing") + +func (readOnly) Create(string, any) (uint64, error) { return 0, errReadOnly } +func (readOnly) Update(string, any, uint64) (uint64, error) { return 0, errReadOnly } +func (readOnly) Put(string, any) (uint64, error) { return 0, errReadOnly } +func (readOnly) Delete(string) error { return errReadOnly } + +// UpgradeHealth is the health of this node's upgrades, from the module's states. +func (m *Machine) UpgradeHealth() Health { + if h, said := m.InstalledHealth(); said { + return h + } + if m.States == nil || m.Node == "" { + return Health{Why: "the tools bundle was started without its node (MESH_NODE) or the bus: the upgrades cannot be read"} + } + cfg := keeperConfig() + if len(cfg.UpgradeOrder) == 0 { + return Health{Why: "no upgrade-order is set: no node upgrades, since no round starts until the mesh-level key " + + "upgrade-order names the nodes in the order they upgrade (ADR 0508 §5)"} + } + runs := readOnly{m.States(StateRuns)} + latest, err := latestOwnRun(runs, m.Node) if err != nil { - return Health{Why: fmt.Sprintf("the weekly upgrade is running and when it started could not be read (%q)", started)} + return Health{Why: "the node's runs could not be read: " + err.Error()} } - if m.Now().Sub(at) <= RunLimit { - return Health{Healthy: true, Why: "the weekly upgrade is running, since " + started} + if latest != nil { + if h, said := m.runHealth(*latest); said { + return h + } } - why := fmt.Sprintf("the weekly upgrade has been running since %s, longer than %s: it is hung, and holds pacman's database lock so no later upgrade can start", started, RunLimit) - found := []string{} - if ls := m.journal(WeeklyUnit, 1); len(ls) > 0 && !strings.HasPrefix(ls[0], "(the journal could not be read") { - why += "; its last line: " + strings.TrimSpace(ls[0]) - found = append(found, strings.TrimSpace(ls[0])) + if h, said := m.roundHealth(readOnly{m.States(StateRounds)}, runs, cfg.RoundStartMinutes); said { + return h } - return Health{Why: why, Found: found} + if latest == nil { + return Health{Healthy: true, Why: "no upgrade has run on this node through the mesh yet"} + } + return Health{Healthy: true, Why: fmt.Sprintf("the last run, %s, passed its after-upgrade check", latest.Run)} +} + +// latestOwnRun is the node's newest run that the mesh ran, or nil; runs read from pacman's log are history. +func latestOwnRun(runs Bucket, node string) (*Run, error) { + keys, err := runs.Keys() + if err != nil { + return nil, err + } + sort.Sort(sort.Reverse(sort.StringSlice(keys))) + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := runs.Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + var r Run + if err := json.Unmarshal(e.Value, &r); err != nil { + return nil, fmt.Errorf("run %s: %w", k, err) + } + if r.From == FromLog || len(r.Steps) == 0 { + continue + } + return &r, nil + } + return nil, nil +} + +// runHealth judges the node's latest run; said is false when the run has nothing to say against the node. +func (m *Machine) runHealth(r Run) (Health, bool) { + var held []HeldKernel + var started *Step + for i, s := range r.Steps { + switch s.Step { + case StepCheckedBefore: + held = s.Held + case StepStarted: + started = &r.Steps[i] + } + } + what := r.Kind + " run " + r.Run + last := r.Steps[len(r.Steps)-1] + switch lastStep(r) { + case StepRefused: + return Health{Why: fmt.Sprintf("the %s did not start (%s): %s", what, last.At.Local().Format(healthTime), last.Why)}, true + case StepCheckedAfter: + if last.Check == nil || !last.Check.Passed { + why := fmt.Sprintf("the %s did not pass its after-upgrade check (%s)", what, last.At.Local().Format(healthTime)) + for _, s := range r.Steps { + if s.Step == StepFailed { + why += ": its transaction failed: " + s.Error + } + } + if last.Check != nil { + for _, u := range last.Check.Unbuilt { + why += fmt.Sprintf("; %s %s is not built for kernel %s", u.Module, u.Version, u.Kernel) + } + } + return Health{Why: why + "; pacman_upgrades with the node and the run says it whole", Found: []string{r.Run}}, true + } + case StepStarted: + if started != nil && m.Now().Sub(started.At) > RunLimit { + return Health{Why: fmt.Sprintf("the %s's transaction (%s) has gone on since %s, longer than %s: it is hung, and holds pacman's database lock so no later run can start", + what, started.Unit, started.At.Local().Format(healthTime), RunLimit)}, true + } + } + if len(held) > 0 { + var said []string + for _, h := range held { + said = append(said, fmt.Sprintf("%s %s (the DKMS module %s allows %s)", h.Package, h.Version, h.Module, h.Bound)) + } + return Health{Why: "the " + what + " held back " + strings.Join(said, ", ") + ": the kernel waits until its DKMS module follows (ADR 0508 §3)"}, true + } + return Health{}, false +} + +// roundHealth judges this node's place in the newest round; said is false when the round has nothing to say +// against it. +func (m *Machine) roundHealth(rounds, runs Bucket, startMinutes int) (Health, bool) { + id, err := latestRound(rounds) + if err != nil { + return Health{Why: "the rounds could not be read: " + err.Error()}, true + } + if id == "" { + return Health{}, false + } + v, err := readRound(rounds, runs, id) + if err != nil { + return Health{Why: "round " + id + " could not be read: " + err.Error()}, true + } + if v == nil || !contains(v.Round.Order, m.Node) { + return Health{}, false + } + if startMinutes < 1 { + startMinutes = defaultRoundStartMinutes + } + byNode := map[string]NodeInRound{} + for _, s := range RoundState(v, m.Now(), startMinutes) { + byNode[s.Node] = s + } + since := func(s NodeInRound) string { + if s.Since == nil { + return "an unknown time" + } + return s.Since.Local().Format(healthTime) + } + me := byNode[m.Node] + switch me.State { + case InRoundStalled: + return Health{Why: fmt.Sprintf("round %s: this node's turn came at %s and it has not started its run within %d minutes", id, since(me), startMinutes)}, true + case InRoundWaiting: + if w, ok := byNode[me.WaitsFor]; ok { + switch w.State { + case InRoundStalled: + return Health{Why: fmt.Sprintf("round %s stands still: this node waits for %s, whose turn came at %s and which has not started (since %s)", id, w.Node, since(w), since(w))}, true + case InRoundFailed: + return Health{Why: fmt.Sprintf("round %s stopped at %s, whose run or check failed at %s: this node waits on it since then; only the operator passes it (pacman_round with pass and a reason)", id, w.Node, since(w))}, true + } + } + } + return Health{}, false } // lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that diff --git a/modules/pacman/cmd/pacman-tools/upgrade_health_test.go b/modules/pacman/cmd/pacman-tools/upgrade_health_test.go new file mode 100644 index 00000000..86690236 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/upgrade_health_test.go @@ -0,0 +1,147 @@ +package main + +// The module's health read from the bus (novox/hq ADR 0508 §3, §5, §6, §11; acceptance 1841/AC6, AC7, AC11, and +// issue 1579): the node's latest run, its round and any held kernel are read from the module's states, never a +// unit's age, so a laptop that slept through a run is judged by what the bus holds. + +import ( + "strings" + "testing" + "time" +) + +// the test machine's clock is 2026-10-04 12:00 UTC (machine_test.go) +var healthNow = time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC) + +func healthBed(t *testing.T, order []string) (*Machine, *fakeStates) { + t.Helper() + states := &fakeStates{} + m := machine(nil, 1000) + m.Node, m.States = "node-b", readerOf(states) + saved := keeperConfig + keeperConfig = func() KeeperConfig { return KeeperConfig{UpgradeOrder: order, RoundStartMinutes: 120, RecordDays: 180} } + t.Cleanup(func() { keeperConfig = saved }) + return m, states +} + +func putRun(states *fakeStates, r Run) { states.store(StateRuns).Put(NodeKey(r.Node, r.Run), r) } + +func passedRun(node string, at time.Time) Run { + return Run{Node: node, Run: RunID(at), Kind: RunUpgrade, From: FromRound, Steps: []Step{ + {Step: StepRecorded, At: at}, {Step: StepCheckedBefore, At: at}, {Step: StepStarted, At: at}, + {Step: StepFinished, At: at.Add(5 * time.Minute)}, {Step: StepCheckedAfter, At: at.Add(6 * time.Minute), Check: &Check{Passed: true}}}} +} + +func TestWithNoOrderEveryNodesHealthSaysWhy(t *testing.T) { + m, _ := healthBed(t, nil) + if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "upgrade-order") { + t.Fatalf("%+v", h) + } +} + +func TestANodeWithNoRunYetIsHealthy(t *testing.T) { + m, _ := healthBed(t, []string{"node-a", "node-b"}) + if h := m.UpgradeHealth(); !h.Healthy { + t.Fatalf("%+v", h) + } +} + +// issue 1579: a run that failed is unhealthy until a later run passes, whenever the node woke; a run read from +// the log is history and judges nothing. +func TestTheLatestRunJudgesTheNodeNotAUnitsAge(t *testing.T) { + m, states := healthBed(t, []string{"node-a", "node-b"}) + exit := 1 + failedAt := healthNow.Add(-7 * 24 * time.Hour) + putRun(states, Run{Node: "node-b", Run: RunID(failedAt), Kind: RunUpgrade, From: FromRound, Steps: []Step{ + {Step: StepStarted, At: failedAt}, {Step: StepFailed, At: failedAt, Exit: &exit, Error: "error: failed to commit transaction (conflicting files)"}, + {Step: StepCheckedAfter, At: failedAt, Check: &Check{}}}}) + putRun(states, Run{Node: "node-b", Run: RunID(healthNow.Add(-time.Hour)), Kind: RunUpgrade, From: FromLog, + Steps: []Step{{Step: StepStarted, At: healthNow.Add(-time.Hour)}, {Step: StepFinished, At: healthNow.Add(-time.Hour)}}}) + putRun(states, passedRun("node-a", healthNow.Add(-time.Minute))) + h := m.UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "conflicting files") || !strings.Contains(h.Why, RunID(failedAt)) { + t.Fatalf("a failed run: %+v", h) + } + putRun(states, passedRun("node-b", healthNow.Add(-30*time.Minute))) + if h := m.UpgradeHealth(); !h.Healthy { + t.Fatalf("a later run passed: %+v", h) + } +} + +func TestAFailedCheckIsUnhealthy(t *testing.T) { + m, states := healthBed(t, []string{"node-b"}) + r := passedRun("node-b", healthNow.Add(-time.Hour)) + r.Steps[len(r.Steps)-1].Check = &Check{Passed: false, Unbuilt: []DKMSBuild{{Module: "made-up-dkms", Version: "1.0", Kernel: "7.3.1-arch1-1"}}} + putRun(states, r) + if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "after-upgrade check") { + t.Fatalf("%+v", h) + } +} + +func TestARunPastTheLimitIsHungAndOneWithinItIsHealthy(t *testing.T) { + m, states := healthBed(t, []string{"node-b"}) + started := healthNow.Add(-3 * time.Hour) + putRun(states, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepRecorded, At: started}, {Step: StepStarted, At: started, Unit: TransactionUnit(RunUpgrade, RunID(started))}}}) + if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, started.Local().Format(healthTime)) { + t.Fatalf("3 h: %+v", h) + } + m2, states2 := healthBed(t, []string{"node-b"}) + started = healthNow.Add(-10 * time.Minute) + putRun(states2, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepStarted, At: started}}}) + if h := m2.UpgradeHealth(); !h.Healthy { + t.Fatalf("10 min: %+v", h) + } +} + +// 1841/AC4: a held kernel keeps the health unhealthy, naming it, until a later run upgrades it. +func TestAHeldKernelIsSaidUntilARunUpgradesIt(t *testing.T) { + m, states := healthBed(t, []string{"node-b"}) + r := passedRun("node-b", healthNow.Add(-2*time.Hour)) + r.Steps[1].Held = []HeldKernel{{Package: "linux", Version: "7.3.1.arch1-1", Module: "zfs", Bound: "Linux-Maximum 7.2"}} + putRun(states, r) + h := m.UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "linux 7.3.1.arch1-1") || !strings.Contains(h.Why, "zfs") || !strings.Contains(h.Why, "Linux-Maximum 7.2") { + t.Fatalf("%+v", h) + } + putRun(states, passedRun("node-b", healthNow.Add(-time.Hour))) + if h := m.UpgradeHealth(); !h.Healthy { + t.Fatalf("a later run held nothing: %+v", h) + } +} + +// 1841/AC7: a turn not started within round-start-minutes makes the waiting node's health name the node it +// waits for and since when; the stalled node's own health says it too. +func TestAStalledRoundIsSaidByTheNodesWaitingOnIt(t *testing.T) { + order := []string{"node-a", "node-b", "node-c"} + m, states := healthBed(t, order) + round := RoundID(healthNow) + opened := healthNow.Add(-3 * time.Hour) + states.store(StateRounds).Put(round, Round{Round: round, Opened: opened, OpenedBy: "node-a", Order: order}) + putRun(states, passedRun("node-a", opened)) + states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: RunID(opened)}) + // node-a passed 2 h 54 min ago; node-b never started. + h := m.UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "has not started") { + t.Fatalf("the stalled node: %+v", h) + } + m.Node = "node-c" + h = m.UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "node-b") || !strings.Contains(h.Why, "since") { + t.Fatalf("the node waiting on it: %+v", h) + } +} + +func TestARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt(t *testing.T) { + order := []string{"node-a", "node-b"} + m, states := healthBed(t, order) + round := RoundID(healthNow) + at := healthNow.Add(-time.Hour) + states.store(StateRounds).Put(round, Round{Round: round, Opened: at, OpenedBy: "node-a", Order: order}) + r := passedRun("node-a", at) + r.Steps[len(r.Steps)-1].Check = &Check{Passed: false} + putRun(states, r) + states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: r.Run}) + if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "node-a") || !strings.Contains(h.Why, "stopped") { + t.Fatalf("%+v", h) + } +}