issue 1841/S11: judge the upgrades' health from the runs, rounds and held kernels on the bus, so a node that slept through a run is judged by what happened, not by a unit's age (folds issue 1579)
This commit is contained in:
@@ -236,13 +236,11 @@ func tools(m *Machine) []stdio.Tool {
|
||||
},
|
||||
{
|
||||
Name: "pacman_upgrade_health",
|
||||
Description: "Whether the weekly full upgrade (Saturday 04:00 local time, ADR 0477) last passed, as the module's health check: unhealthy " +
|
||||
"with the exit and the last error of the journal when it failed, healthy while it runs, after it passed and before it ever ran. (r)",
|
||||
Description: "This node's upgrades, read from the bus, as the module's health check (ADR 0508 §11): unhealthy while no upgrade-order " +
|
||||
"is set, while the node's latest run is refused, failed or did not pass its after-upgrade check, hung past two hours, or held a " +
|
||||
"kernel, and while the week's round stands still at this node or at the node it waits for, naming it and since when. (r)",
|
||||
Input: schema(map[string]any{}),
|
||||
Run: func(map[string]any) (any, error) {
|
||||
if h, said := m.InstalledHealth(); said {
|
||||
return h, nil
|
||||
}
|
||||
return m.UpgradeHealth(), nil
|
||||
},
|
||||
},
|
||||
|
||||
@@ -319,57 +319,3 @@ func TestConfigIsReadAsPacmanConfResolvesIt(t *testing.T) {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// The weekly upgrade's outcome is the module's health (issue 1574, ADR 0477 clause 5).
|
||||
func upgradeHealthMachine(props, journal string) *Machine {
|
||||
return machine(byLine(map[string]Ran{
|
||||
"systemctl show " + WeeklyUnit + " --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecMainExitTimestamp --property=ExecMainStartTimestamp": {Stdout: props},
|
||||
"sudo -n journalctl --no-pager -o cat -n 40 -u " + WeeklyUnit: {Stdout: journal},
|
||||
"sudo -n journalctl --no-pager -o cat -n 1 -u " + WeeklyUnit: {Stdout: journal},
|
||||
}, nil), 1000)
|
||||
}
|
||||
|
||||
func TestAFailedWeeklyUpgradeIsUnhealthyNamingTheError(t *testing.T) {
|
||||
h := upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=Sat 2026-10-24 04:03:11 CEST\n",
|
||||
":: Synchronizing package databases...\nerror: failed to commit transaction (conflicting files)\nErrors occurred, no packages were upgraded.\n").UpgradeHealth()
|
||||
if h.Healthy || !strings.Contains(h.Why, "failed to commit transaction (conflicting files)") || !strings.Contains(h.Why, "exit-code") || !strings.Contains(h.Why, "Sat 2026-10-24 04:03:11 CEST") {
|
||||
t.Fatalf("%+v", h)
|
||||
}
|
||||
h = upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=\n", "").UpgradeHealth()
|
||||
if h.Healthy || !strings.Contains(h.Why, "status 1") {
|
||||
t.Fatalf("a failure without a journal is still unhealthy: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPassedRunningOrNeverRunWeeklyUpgradeIsHealthy(t *testing.T) {
|
||||
for name, props := range map[string]string{
|
||||
"passed": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=Sat 2026-10-24 04:09:00 CEST\n",
|
||||
"running": "LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n",
|
||||
"never run": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n",
|
||||
} {
|
||||
if h := upgradeHealthMachine(props, "").UpgradeHealth(); !h.Healthy {
|
||||
t.Errorf("%s: %+v", name, h)
|
||||
}
|
||||
}
|
||||
if h := machine(fake(func(call) Ran { return Ran{Status: 1, Stderr: "boom"} }, nil), 1000).UpgradeHealth(); h.Healthy {
|
||||
t.Errorf("a unit that cannot be read is not healthy: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// A run that goes on past RunLimit is hung and holds pacman's lock: unhealthy, naming its start (issue 1574).
|
||||
func TestAHungWeeklyUpgradeIsUnhealthyNamingItsStart(t *testing.T) {
|
||||
// the test machine's clock is 2026-10-04 12:00 UTC
|
||||
h := upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 09:00:00 UTC\n",
|
||||
"downloading linux-6.99...\n").UpgradeHealth()
|
||||
if h.Healthy || !strings.Contains(h.Why, "Wed 2026-10-04 09:00:00 UTC") || !strings.Contains(h.Why, "downloading linux-6.99") {
|
||||
t.Fatalf("started 3 h ago: %+v", h)
|
||||
}
|
||||
h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", "").UpgradeHealth()
|
||||
if !h.Healthy {
|
||||
t.Fatalf("started 10 min ago: %+v", h)
|
||||
}
|
||||
h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=\n", "").UpgradeHealth()
|
||||
if h.Healthy {
|
||||
t.Fatalf("a start that cannot be read is not healthy: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,30 @@
|
||||
package main
|
||||
|
||||
// The module's health, read from the bus (novox/hq ADR 0508 §11; issue 1579). The health check of the keeper's
|
||||
// process reads this node's latest run, the week's round and any held kernel from the module's states, never a
|
||||
// unit's age, so a laptop that slept through a run, or a node that was off, is judged by what the bus holds. It
|
||||
// is unhealthy, naming why, when:
|
||||
//
|
||||
// - `installed.<node>` is marked not current (ADR 0507 §2);
|
||||
// - no `upgrade-order` is set: no round starts, and every node says so (ADR 0508 §5);
|
||||
// - the node's latest run was refused, failed, or its after-upgrade check did not pass, until a later run passes;
|
||||
// - the latest run's transaction goes on past RunLimit: hung, holding pacman's lock;
|
||||
// - the latest run held a kernel, naming the package, the DKMS module and its bound (ADR 0508 §3);
|
||||
// - the week's round stands still: this node's turn came more than round-start-minutes ago and it has not
|
||||
// started, or the node it waits for stalled or failed, named with since when (ADR 0508 §6).
|
||||
//
|
||||
// A state that cannot be read is unhealthy: failure is loud.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node
|
||||
// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager
|
||||
// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of
|
||||
// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and,
|
||||
// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an
|
||||
// upgrade passes.
|
||||
|
||||
// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why.
|
||||
type Health struct {
|
||||
Healthy bool `json:"healthy"`
|
||||
@@ -20,66 +32,190 @@ type Health struct {
|
||||
Found []string `json:"found,omitempty"`
|
||||
}
|
||||
|
||||
// WeeklyUnit is the service the weekly timer starts.
|
||||
const WeeklyUnit = "mesh-pacman-weekly-upgrade.service"
|
||||
|
||||
// RunLimit is how long a weekly upgrade may go on before it is unhealthy. The service has no start timeout on
|
||||
// purpose: killing pacman in the middle of a transaction leaves a half-upgraded machine. A run past the limit
|
||||
// is a hung one (a stuck hook, a stalled transaction), which holds pacman's database lock so that no later run
|
||||
// can start; it is said, not killed.
|
||||
// RunLimit is how long a run's transaction may go on before it is unhealthy. A transaction has no timeout on
|
||||
// purpose: killing pacman in the middle of one leaves a half-upgraded machine. One past the limit is a hung one
|
||||
// (a stuck hook, a stalled download), which holds pacman's database lock so no later run can start; it is said,
|
||||
// not killed.
|
||||
const RunLimit = 2 * time.Hour
|
||||
|
||||
// systemdTime is how systemctl shows a timestamp, in the machine's local time.
|
||||
const systemdTime = "Mon 2006-01-02 15:04:05 MST"
|
||||
// healthTime is how the health writes a moment: the machine's local time.
|
||||
const healthTime = "2006-01-02 15:04 MST"
|
||||
|
||||
// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit
|
||||
// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A
|
||||
// unit that cannot be read is unhealthy: failure is loud.
|
||||
func (m *Machine) UpgradeHealth() Health {
|
||||
p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp", "ExecMainStartTimestamp")
|
||||
if err != nil {
|
||||
return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()}
|
||||
// keeperConfig is the keeper's config as the tools bundle reads it, from the file the manifest names to the
|
||||
// bundle too (EnvConfig); a test replaces it. A file that cannot be read is an empty config: no order.
|
||||
var keeperConfig = func() KeeperConfig {
|
||||
var c KeeperConfig
|
||||
if raw, err := os.ReadFile(os.Getenv(EnvConfig)); err == nil {
|
||||
_ = json.Unmarshal(raw, &c)
|
||||
}
|
||||
switch {
|
||||
case p["LoadState"] == "not-found":
|
||||
return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"}
|
||||
case p["ActiveState"] == "activating" || p["ActiveState"] == "active":
|
||||
return m.runningHealth(p["ExecMainStartTimestamp"])
|
||||
case p["ActiveState"] != "failed" && p["Result"] == "success":
|
||||
if at := p["ExecMainExitTimestamp"]; at != "" {
|
||||
return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"}
|
||||
}
|
||||
return Health{Healthy: true, Why: "no weekly upgrade has run yet"}
|
||||
}
|
||||
why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"])
|
||||
if at := p["ExecMainExitTimestamp"]; at != "" {
|
||||
why += " at " + at
|
||||
}
|
||||
found := []string{}
|
||||
if said := lastError(m.journal(WeeklyUnit, 40)); said != "" {
|
||||
why += ": " + said
|
||||
found = append(found, said)
|
||||
}
|
||||
return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found}
|
||||
return c
|
||||
}
|
||||
|
||||
// runningHealth judges a run in progress: healthy within RunLimit of its start, unhealthy past it, naming the
|
||||
// start and the last line the journal holds.
|
||||
func (m *Machine) runningHealth(started string) Health {
|
||||
at, err := time.ParseInLocation(systemdTime, started, time.Local)
|
||||
// readOnly is a state the tools bundle reads, as the round's readers take it; it writes nothing.
|
||||
type readOnly struct{ StateReader }
|
||||
|
||||
var errReadOnly = errors.New("the health writes nothing")
|
||||
|
||||
func (readOnly) Create(string, any) (uint64, error) { return 0, errReadOnly }
|
||||
func (readOnly) Update(string, any, uint64) (uint64, error) { return 0, errReadOnly }
|
||||
func (readOnly) Put(string, any) (uint64, error) { return 0, errReadOnly }
|
||||
func (readOnly) Delete(string) error { return errReadOnly }
|
||||
|
||||
// UpgradeHealth is the health of this node's upgrades, from the module's states.
|
||||
func (m *Machine) UpgradeHealth() Health {
|
||||
if h, said := m.InstalledHealth(); said {
|
||||
return h
|
||||
}
|
||||
if m.States == nil || m.Node == "" {
|
||||
return Health{Why: "the tools bundle was started without its node (MESH_NODE) or the bus: the upgrades cannot be read"}
|
||||
}
|
||||
cfg := keeperConfig()
|
||||
if len(cfg.UpgradeOrder) == 0 {
|
||||
return Health{Why: "no upgrade-order is set: no node upgrades, since no round starts until the mesh-level key " +
|
||||
"upgrade-order names the nodes in the order they upgrade (ADR 0508 §5)"}
|
||||
}
|
||||
runs := readOnly{m.States(StateRuns)}
|
||||
latest, err := latestOwnRun(runs, m.Node)
|
||||
if err != nil {
|
||||
return Health{Why: fmt.Sprintf("the weekly upgrade is running and when it started could not be read (%q)", started)}
|
||||
return Health{Why: "the node's runs could not be read: " + err.Error()}
|
||||
}
|
||||
if m.Now().Sub(at) <= RunLimit {
|
||||
return Health{Healthy: true, Why: "the weekly upgrade is running, since " + started}
|
||||
if latest != nil {
|
||||
if h, said := m.runHealth(*latest); said {
|
||||
return h
|
||||
}
|
||||
}
|
||||
why := fmt.Sprintf("the weekly upgrade has been running since %s, longer than %s: it is hung, and holds pacman's database lock so no later upgrade can start", started, RunLimit)
|
||||
found := []string{}
|
||||
if ls := m.journal(WeeklyUnit, 1); len(ls) > 0 && !strings.HasPrefix(ls[0], "(the journal could not be read") {
|
||||
why += "; its last line: " + strings.TrimSpace(ls[0])
|
||||
found = append(found, strings.TrimSpace(ls[0]))
|
||||
if h, said := m.roundHealth(readOnly{m.States(StateRounds)}, runs, cfg.RoundStartMinutes); said {
|
||||
return h
|
||||
}
|
||||
return Health{Why: why, Found: found}
|
||||
if latest == nil {
|
||||
return Health{Healthy: true, Why: "no upgrade has run on this node through the mesh yet"}
|
||||
}
|
||||
return Health{Healthy: true, Why: fmt.Sprintf("the last run, %s, passed its after-upgrade check", latest.Run)}
|
||||
}
|
||||
|
||||
// latestOwnRun is the node's newest run that the mesh ran, or nil; runs read from pacman's log are history.
|
||||
func latestOwnRun(runs Bucket, node string) (*Run, error) {
|
||||
keys, err := runs.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(keys)))
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, node+".") {
|
||||
continue
|
||||
}
|
||||
e, err := runs.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
var r Run
|
||||
if err := json.Unmarshal(e.Value, &r); err != nil {
|
||||
return nil, fmt.Errorf("run %s: %w", k, err)
|
||||
}
|
||||
if r.From == FromLog || len(r.Steps) == 0 {
|
||||
continue
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// runHealth judges the node's latest run; said is false when the run has nothing to say against the node.
|
||||
func (m *Machine) runHealth(r Run) (Health, bool) {
|
||||
var held []HeldKernel
|
||||
var started *Step
|
||||
for i, s := range r.Steps {
|
||||
switch s.Step {
|
||||
case StepCheckedBefore:
|
||||
held = s.Held
|
||||
case StepStarted:
|
||||
started = &r.Steps[i]
|
||||
}
|
||||
}
|
||||
what := r.Kind + " run " + r.Run
|
||||
last := r.Steps[len(r.Steps)-1]
|
||||
switch lastStep(r) {
|
||||
case StepRefused:
|
||||
return Health{Why: fmt.Sprintf("the %s did not start (%s): %s", what, last.At.Local().Format(healthTime), last.Why)}, true
|
||||
case StepCheckedAfter:
|
||||
if last.Check == nil || !last.Check.Passed {
|
||||
why := fmt.Sprintf("the %s did not pass its after-upgrade check (%s)", what, last.At.Local().Format(healthTime))
|
||||
for _, s := range r.Steps {
|
||||
if s.Step == StepFailed {
|
||||
why += ": its transaction failed: " + s.Error
|
||||
}
|
||||
}
|
||||
if last.Check != nil {
|
||||
for _, u := range last.Check.Unbuilt {
|
||||
why += fmt.Sprintf("; %s %s is not built for kernel %s", u.Module, u.Version, u.Kernel)
|
||||
}
|
||||
}
|
||||
return Health{Why: why + "; pacman_upgrades with the node and the run says it whole", Found: []string{r.Run}}, true
|
||||
}
|
||||
case StepStarted:
|
||||
if started != nil && m.Now().Sub(started.At) > RunLimit {
|
||||
return Health{Why: fmt.Sprintf("the %s's transaction (%s) has gone on since %s, longer than %s: it is hung, and holds pacman's database lock so no later run can start",
|
||||
what, started.Unit, started.At.Local().Format(healthTime), RunLimit)}, true
|
||||
}
|
||||
}
|
||||
if len(held) > 0 {
|
||||
var said []string
|
||||
for _, h := range held {
|
||||
said = append(said, fmt.Sprintf("%s %s (the DKMS module %s allows %s)", h.Package, h.Version, h.Module, h.Bound))
|
||||
}
|
||||
return Health{Why: "the " + what + " held back " + strings.Join(said, ", ") + ": the kernel waits until its DKMS module follows (ADR 0508 §3)"}, true
|
||||
}
|
||||
return Health{}, false
|
||||
}
|
||||
|
||||
// roundHealth judges this node's place in the newest round; said is false when the round has nothing to say
|
||||
// against it.
|
||||
func (m *Machine) roundHealth(rounds, runs Bucket, startMinutes int) (Health, bool) {
|
||||
id, err := latestRound(rounds)
|
||||
if err != nil {
|
||||
return Health{Why: "the rounds could not be read: " + err.Error()}, true
|
||||
}
|
||||
if id == "" {
|
||||
return Health{}, false
|
||||
}
|
||||
v, err := readRound(rounds, runs, id)
|
||||
if err != nil {
|
||||
return Health{Why: "round " + id + " could not be read: " + err.Error()}, true
|
||||
}
|
||||
if v == nil || !contains(v.Round.Order, m.Node) {
|
||||
return Health{}, false
|
||||
}
|
||||
if startMinutes < 1 {
|
||||
startMinutes = defaultRoundStartMinutes
|
||||
}
|
||||
byNode := map[string]NodeInRound{}
|
||||
for _, s := range RoundState(v, m.Now(), startMinutes) {
|
||||
byNode[s.Node] = s
|
||||
}
|
||||
since := func(s NodeInRound) string {
|
||||
if s.Since == nil {
|
||||
return "an unknown time"
|
||||
}
|
||||
return s.Since.Local().Format(healthTime)
|
||||
}
|
||||
me := byNode[m.Node]
|
||||
switch me.State {
|
||||
case InRoundStalled:
|
||||
return Health{Why: fmt.Sprintf("round %s: this node's turn came at %s and it has not started its run within %d minutes", id, since(me), startMinutes)}, true
|
||||
case InRoundWaiting:
|
||||
if w, ok := byNode[me.WaitsFor]; ok {
|
||||
switch w.State {
|
||||
case InRoundStalled:
|
||||
return Health{Why: fmt.Sprintf("round %s stands still: this node waits for %s, whose turn came at %s and which has not started (since %s)", id, w.Node, since(w), since(w))}, true
|
||||
case InRoundFailed:
|
||||
return Health{Why: fmt.Sprintf("round %s stopped at %s, whose run or check failed at %s: this node waits on it since then; only the operator passes it (pacman_round with pass and a reason)", id, w.Node, since(w))}, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return Health{}, false
|
||||
}
|
||||
|
||||
// lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
package main
|
||||
|
||||
// The module's health read from the bus (novox/hq ADR 0508 §3, §5, §6, §11; acceptance 1841/AC6, AC7, AC11, and
|
||||
// issue 1579): the node's latest run, its round and any held kernel are read from the module's states, never a
|
||||
// unit's age, so a laptop that slept through a run is judged by what the bus holds.
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// the test machine's clock is 2026-10-04 12:00 UTC (machine_test.go)
|
||||
var healthNow = time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func healthBed(t *testing.T, order []string) (*Machine, *fakeStates) {
|
||||
t.Helper()
|
||||
states := &fakeStates{}
|
||||
m := machine(nil, 1000)
|
||||
m.Node, m.States = "node-b", readerOf(states)
|
||||
saved := keeperConfig
|
||||
keeperConfig = func() KeeperConfig { return KeeperConfig{UpgradeOrder: order, RoundStartMinutes: 120, RecordDays: 180} }
|
||||
t.Cleanup(func() { keeperConfig = saved })
|
||||
return m, states
|
||||
}
|
||||
|
||||
func putRun(states *fakeStates, r Run) { states.store(StateRuns).Put(NodeKey(r.Node, r.Run), r) }
|
||||
|
||||
func passedRun(node string, at time.Time) Run {
|
||||
return Run{Node: node, Run: RunID(at), Kind: RunUpgrade, From: FromRound, Steps: []Step{
|
||||
{Step: StepRecorded, At: at}, {Step: StepCheckedBefore, At: at}, {Step: StepStarted, At: at},
|
||||
{Step: StepFinished, At: at.Add(5 * time.Minute)}, {Step: StepCheckedAfter, At: at.Add(6 * time.Minute), Check: &Check{Passed: true}}}}
|
||||
}
|
||||
|
||||
func TestWithNoOrderEveryNodesHealthSaysWhy(t *testing.T) {
|
||||
m, _ := healthBed(t, nil)
|
||||
if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "upgrade-order") {
|
||||
t.Fatalf("%+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeWithNoRunYetIsHealthy(t *testing.T) {
|
||||
m, _ := healthBed(t, []string{"node-a", "node-b"})
|
||||
if h := m.UpgradeHealth(); !h.Healthy {
|
||||
t.Fatalf("%+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// issue 1579: a run that failed is unhealthy until a later run passes, whenever the node woke; a run read from
|
||||
// the log is history and judges nothing.
|
||||
func TestTheLatestRunJudgesTheNodeNotAUnitsAge(t *testing.T) {
|
||||
m, states := healthBed(t, []string{"node-a", "node-b"})
|
||||
exit := 1
|
||||
failedAt := healthNow.Add(-7 * 24 * time.Hour)
|
||||
putRun(states, Run{Node: "node-b", Run: RunID(failedAt), Kind: RunUpgrade, From: FromRound, Steps: []Step{
|
||||
{Step: StepStarted, At: failedAt}, {Step: StepFailed, At: failedAt, Exit: &exit, Error: "error: failed to commit transaction (conflicting files)"},
|
||||
{Step: StepCheckedAfter, At: failedAt, Check: &Check{}}}})
|
||||
putRun(states, Run{Node: "node-b", Run: RunID(healthNow.Add(-time.Hour)), Kind: RunUpgrade, From: FromLog,
|
||||
Steps: []Step{{Step: StepStarted, At: healthNow.Add(-time.Hour)}, {Step: StepFinished, At: healthNow.Add(-time.Hour)}}})
|
||||
putRun(states, passedRun("node-a", healthNow.Add(-time.Minute)))
|
||||
h := m.UpgradeHealth()
|
||||
if h.Healthy || !strings.Contains(h.Why, "conflicting files") || !strings.Contains(h.Why, RunID(failedAt)) {
|
||||
t.Fatalf("a failed run: %+v", h)
|
||||
}
|
||||
putRun(states, passedRun("node-b", healthNow.Add(-30*time.Minute)))
|
||||
if h := m.UpgradeHealth(); !h.Healthy {
|
||||
t.Fatalf("a later run passed: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedCheckIsUnhealthy(t *testing.T) {
|
||||
m, states := healthBed(t, []string{"node-b"})
|
||||
r := passedRun("node-b", healthNow.Add(-time.Hour))
|
||||
r.Steps[len(r.Steps)-1].Check = &Check{Passed: false, Unbuilt: []DKMSBuild{{Module: "made-up-dkms", Version: "1.0", Kernel: "7.3.1-arch1-1"}}}
|
||||
putRun(states, r)
|
||||
if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "after-upgrade check") {
|
||||
t.Fatalf("%+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARunPastTheLimitIsHungAndOneWithinItIsHealthy(t *testing.T) {
|
||||
m, states := healthBed(t, []string{"node-b"})
|
||||
started := healthNow.Add(-3 * time.Hour)
|
||||
putRun(states, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepRecorded, At: started}, {Step: StepStarted, At: started, Unit: TransactionUnit(RunUpgrade, RunID(started))}}})
|
||||
if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, started.Local().Format(healthTime)) {
|
||||
t.Fatalf("3 h: %+v", h)
|
||||
}
|
||||
m2, states2 := healthBed(t, []string{"node-b"})
|
||||
started = healthNow.Add(-10 * time.Minute)
|
||||
putRun(states2, Run{Node: "node-b", Run: RunID(started), Kind: RunUpgrade, From: FromAsk, Steps: []Step{{Step: StepStarted, At: started}}})
|
||||
if h := m2.UpgradeHealth(); !h.Healthy {
|
||||
t.Fatalf("10 min: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// 1841/AC4: a held kernel keeps the health unhealthy, naming it, until a later run upgrades it.
|
||||
func TestAHeldKernelIsSaidUntilARunUpgradesIt(t *testing.T) {
|
||||
m, states := healthBed(t, []string{"node-b"})
|
||||
r := passedRun("node-b", healthNow.Add(-2*time.Hour))
|
||||
r.Steps[1].Held = []HeldKernel{{Package: "linux", Version: "7.3.1.arch1-1", Module: "zfs", Bound: "Linux-Maximum 7.2"}}
|
||||
putRun(states, r)
|
||||
h := m.UpgradeHealth()
|
||||
if h.Healthy || !strings.Contains(h.Why, "linux 7.3.1.arch1-1") || !strings.Contains(h.Why, "zfs") || !strings.Contains(h.Why, "Linux-Maximum 7.2") {
|
||||
t.Fatalf("%+v", h)
|
||||
}
|
||||
putRun(states, passedRun("node-b", healthNow.Add(-time.Hour)))
|
||||
if h := m.UpgradeHealth(); !h.Healthy {
|
||||
t.Fatalf("a later run held nothing: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// 1841/AC7: a turn not started within round-start-minutes makes the waiting node's health name the node it
|
||||
// waits for and since when; the stalled node's own health says it too.
|
||||
func TestAStalledRoundIsSaidByTheNodesWaitingOnIt(t *testing.T) {
|
||||
order := []string{"node-a", "node-b", "node-c"}
|
||||
m, states := healthBed(t, order)
|
||||
round := RoundID(healthNow)
|
||||
opened := healthNow.Add(-3 * time.Hour)
|
||||
states.store(StateRounds).Put(round, Round{Round: round, Opened: opened, OpenedBy: "node-a", Order: order})
|
||||
putRun(states, passedRun("node-a", opened))
|
||||
states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: RunID(opened)})
|
||||
// node-a passed 2 h 54 min ago; node-b never started.
|
||||
h := m.UpgradeHealth()
|
||||
if h.Healthy || !strings.Contains(h.Why, "has not started") {
|
||||
t.Fatalf("the stalled node: %+v", h)
|
||||
}
|
||||
m.Node = "node-c"
|
||||
h = m.UpgradeHealth()
|
||||
if h.Healthy || !strings.Contains(h.Why, "node-b") || !strings.Contains(h.Why, "since") {
|
||||
t.Fatalf("the node waiting on it: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARoundStoppedByAFailureIsSaidByTheNodesWaitingOnIt(t *testing.T) {
|
||||
order := []string{"node-a", "node-b"}
|
||||
m, states := healthBed(t, order)
|
||||
round := RoundID(healthNow)
|
||||
at := healthNow.Add(-time.Hour)
|
||||
states.store(StateRounds).Put(round, Round{Round: round, Opened: at, OpenedBy: "node-a", Order: order})
|
||||
r := passedRun("node-a", at)
|
||||
r.Steps[len(r.Steps)-1].Check = &Check{Passed: false}
|
||||
putRun(states, r)
|
||||
states.store(StateRounds).Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: r.Run})
|
||||
if h := m.UpgradeHealth(); h.Healthy || !strings.Contains(h.Why, "node-a") || !strings.Contains(h.Why, "stopped") {
|
||||
t.Fatalf("%+v", h)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user