diff --git a/health-undeclared b/health-undeclared index 3fbd022b..0383faf4 100644 --- a/health-undeclared +++ b/health-undeclared @@ -1,4 +1,4 @@ -61 +59 The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240 rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running diff --git a/modules/pacman/README.md b/modules/pacman/README.md index eb5ebaa8..4ba4969a 100644 --- a/modules/pacman/README.md +++ b/modules/pacman/README.md @@ -23,6 +23,23 @@ that seat (ADR 0207). list stays reflector's to write, not the mesh's. - **Cache cleaning.** The `pacman-contrib` package and `paccache.timer` running and enabled. Each week it keeps the last three versions of each package. +- **The weekly upgrade (ADR 0477 clause 5, issue 1574).** `mesh-pacman-weekly-upgrade.timer` and its + oneshot service, placed whole in `/etc/systemd/system`. The timer fires every Saturday at 04:00 in the + node's local time, the same moment on every node (no random delay), and is `Persistent=true`: a node that + was off or asleep then upgrades at its next start. The service runs `/usr/bin/pacman -Syu --noconfirm`, the + same arguments `pacman_upgrade` starts (`UpgradeArgs` in the code; a manifest test holds the two together). + No other moment upgrades but a hand `pacman_upgrade`, and the mesh restarts no machine for an upgrade: a new + kernel waits for the next restart. + - **The outcome is the module's health.** The timer's resource asks `pacman_upgrade_health`: unhealthy, + naming the exit and the last `error:` line of the journal, when the last weekly upgrade failed; healthy + while it runs, after it passed, and before it ever ran. The controller raises the module's condition for + the node from it. A failed service stays failed until + the next run, so the condition stands until an upgrade passes; a hand `pacman_upgrade` does not clear + it. A restart of the machine clears the failed state and with it the condition, and the next Saturday + runs again. A run that goes on longer than two hours is unhealthy too, naming when it started and the + last line of its journal: it is hung and holds pacman's database lock. The service has no start + timeout on purpose, because killing pacman in the middle of a transaction is worse; the hung run is + said, not killed. ## What it improves @@ -50,6 +67,7 @@ that seat (ADR 0207). | `pacman_files` | r | what a package placed, bounded | | `pacman_updates` | r | `checkupdates`: what a full upgrade would change, never setting up a partial upgrade | | `pacman_upgrade` | a | starts `pacman -Syu --noconfirm` (sudo -n) as a transient unit that outlives the call; answers the unit and the news since the last upgrade; given the unit, how it went | +| `pacman_upgrade_health` | r | whether the weekly upgrade last passed: the timer's health check (see the weekly upgrade above) | | `pacman_orphans` | r | `pacman -Qdt` | | `pacman_remove_orphans` | a | `pacman -Rs` on named orphans, or all of them, as a unit of its own; a name that is not an orphan is refused | | `pacman_cache` | r/a | size, interrupted downloads, what paccache would free keeping N; `clean: true` removes them | diff --git a/modules/pacman/cmd/pacman-tools/acts.go b/modules/pacman/cmd/pacman-tools/acts.go index a0b4e3c3..146d6d5e 100644 --- a/modules/pacman/cmd/pacman-tools/acts.go +++ b/modules/pacman/cmd/pacman-tools/acts.go @@ -55,6 +55,11 @@ func (m *Machine) journal(unit string, n int) []string { return lines(out) } +// UpgradeArgs is the full system upgrade, in one place: pacman_upgrade starts pacman with these arguments as +// a transient unit, and the weekly upgrade's service (files/mesh-pacman-weekly-upgrade.service) runs +// pacman with the same ones; a test holds the two together (issue 1574, ADR 0477 clause 5). +var UpgradeArgs = []string{"-Syu", "--noconfirm"} + // Upgrade starts a full system upgrade as a transient unit and answers at once, with the // distribution's news since the last upgrade; or, given a unit it started, answers how it went. func (m *Machine) Upgrade(unit string, n int) (map[string]any, error) { @@ -62,7 +67,7 @@ func (m *Machine) Upgrade(unit string, n int) (map[string]any, error) { return m.UpgradeStatus(unit, n) } news := m.News("") - started, _, err := m.transaction("upgrade", false, "-Syu", "--noconfirm") + started, _, err := m.transaction("upgrade", false, UpgradeArgs...) if err != nil { return nil, err } diff --git a/modules/pacman/cmd/pacman-tools/main.go b/modules/pacman/cmd/pacman-tools/main.go index 06ead071..c4d7136a 100644 --- a/modules/pacman/cmd/pacman-tools/main.go +++ b/modules/pacman/cmd/pacman-tools/main.go @@ -152,6 +152,13 @@ func tools(m *Machine) []stdio.Tool { return m.Upgrade(unit, n) }, }, + { + Name: "pacman_upgrade_health", + Description: "Whether the weekly full upgrade (Saturday 04:00 local time, ADR 0477) last passed, as the module's health check: unhealthy " + + "with the exit and the last error of the journal when it failed, healthy while it runs, after it passed and before it ever ran. (r)", + Input: schema(map[string]any{}), + Run: func(map[string]any) (any, error) { return m.UpgradeHealth(), nil }, + }, { Name: "pacman_orphans", Description: "Packages installed as dependencies that nothing requires any more (pacman -Qdt), with versions.", diff --git a/modules/pacman/cmd/pacman-tools/manifest_test.go b/modules/pacman/cmd/pacman-tools/manifest_test.go index b55f6908..da7fb4ad 100644 --- a/modules/pacman/cmd/pacman-tools/manifest_test.go +++ b/modules/pacman/cmd/pacman-tools/manifest_test.go @@ -101,3 +101,53 @@ func TestTheReflectorPackageIsDeclaredBeforeTheFileItShips(t *testing.T) { t.Fatalf("a package's file and timer come after the package: %v", order) } } + +// The weekly upgrade (issue 1574, ADR 0477 clause 5): a timer on Saturday 04:00 local time, persistent so a +// node that was off upgrades at its next start, the same moment on every node (no random delay), starting a +// oneshot service that runs the very upgrade pacman_upgrade runs. +func TestTheWeeklyUpgradeIsATimerOnSaturdayFourPersistentRunningTheToolsUpgrade(t *testing.T) { + m := manifest(t) + timer := m.resource(t, "upgrade-timer") + service := m.resource(t, "upgrade-service") + if timer["path"] != "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer" || service["path"] != "/etc/systemd/system/"+WeeklyUnit { + t.Fatalf("paths: %v %v", timer["path"], service["path"]) + } + tt, st := templateText(t, timer), templateText(t, service) + for _, want := range []string{"\nOnCalendar=Sat *-*-* 04:00:00\n", "\nPersistent=true\n", "\nWantedBy=timers.target\n"} { + if !strings.Contains(tt, want) { + t.Errorf("the timer lacks %q:\n%s", strings.TrimSpace(want), tt) + } + } + for _, not := range []string{"RandomizedDelaySec", "OnBootSec", "OnUnitActiveSec", "UTC"} { + if strings.Contains(tt, not) { + t.Errorf("the timer says %s: the same local moment on every node, and no other moment", not) + } + } + if strings.Count(tt, "OnCalendar=") != 1 { + t.Errorf("one moment only:\n%s", tt) + } + if !strings.Contains(st, "\nType=oneshot\n") { + t.Errorf("the service is a oneshot:\n%s", st) + } + if want := "\nExecStart=/usr/bin/pacman " + strings.Join(UpgradeArgs, " ") + "\n"; !strings.Contains(st, want) { + t.Errorf("the service does not run the upgrade pacman_upgrade runs (%q):\n%s", want, st) + } + if strings.Contains(st, "reboot") || strings.Contains(tt, "reboot") || strings.Contains(st, "systemctl") { + t.Error("the mesh restarts no machine for an upgrade") + } + r := m.resource(t, "weekly-upgrade") + if r["type"] != "service" || r["unit"] != "mesh-pacman-weekly-upgrade.timer" || r["state"] != "running" || r["boot"] != "enabled" { + t.Fatalf("the timer runs and is enabled: %v", r) + } + h, _ := r["health"].(map[string]any) + if h["kind"] != "tool" || h["tool"] != "pacman_upgrade_health" { + t.Errorf("the upgrade's outcome is the resource's health: %v", r["health"]) + } + order := map[string]int{} + for i, x := range m.Resources { + order[x["id"].(string)] = i + } + if order["upgrade-service"] > order["weekly-upgrade"] || order["upgrade-timer"] > order["weekly-upgrade"] || order["package"] > order["upgrade-service"] { + t.Errorf("the units are placed before the timer is started: %v", order) + } +} diff --git a/modules/pacman/cmd/pacman-tools/pacman_test.go b/modules/pacman/cmd/pacman-tools/pacman_test.go index 83e5196b..4ca7f5bb 100644 --- a/modules/pacman/cmd/pacman-tools/pacman_test.go +++ b/modules/pacman/cmd/pacman-tools/pacman_test.go @@ -374,3 +374,57 @@ func TestConfigIsReadAsPacmanConfResolvesIt(t *testing.T) { t.Fatalf("%+v", c) } } + +// The weekly upgrade's outcome is the module's health (issue 1574, ADR 0477 clause 5). +func upgradeHealthMachine(props, journal string) *Machine { + return machine(byLine(map[string]Ran{ + "systemctl show " + WeeklyUnit + " --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecMainExitTimestamp --property=ExecMainStartTimestamp": {Stdout: props}, + "sudo -n journalctl --no-pager -o cat -n 40 -u " + WeeklyUnit: {Stdout: journal}, + "sudo -n journalctl --no-pager -o cat -n 1 -u " + WeeklyUnit: {Stdout: journal}, + }, nil), 1000) +} + +func TestAFailedWeeklyUpgradeIsUnhealthyNamingTheError(t *testing.T) { + h := upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=Sat 2026-10-24 04:03:11 CEST\n", + ":: Synchronizing package databases...\nerror: failed to commit transaction (conflicting files)\nErrors occurred, no packages were upgraded.\n").UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "failed to commit transaction (conflicting files)") || !strings.Contains(h.Why, "exit-code") || !strings.Contains(h.Why, "Sat 2026-10-24 04:03:11 CEST") { + t.Fatalf("%+v", h) + } + h = upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=\n", "").UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "status 1") { + t.Fatalf("a failure without a journal is still unhealthy: %+v", h) + } +} + +func TestAPassedRunningOrNeverRunWeeklyUpgradeIsHealthy(t *testing.T) { + for name, props := range map[string]string{ + "passed": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=Sat 2026-10-24 04:09:00 CEST\n", + "running": "LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", + "never run": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n", + } { + if h := upgradeHealthMachine(props, "").UpgradeHealth(); !h.Healthy { + t.Errorf("%s: %+v", name, h) + } + } + if h := machine(fake(func(call) Ran { return Ran{Status: 1, Stderr: "boom"} }, nil), 1000).UpgradeHealth(); h.Healthy { + t.Errorf("a unit that cannot be read is not healthy: %+v", h) + } +} + +// A run that goes on past RunLimit is hung and holds pacman's lock: unhealthy, naming its start (issue 1574). +func TestAHungWeeklyUpgradeIsUnhealthyNamingItsStart(t *testing.T) { + // the test machine's clock is 2026-10-04 12:00 UTC + h := upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 09:00:00 UTC\n", + "downloading linux-6.99...\n").UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "Wed 2026-10-04 09:00:00 UTC") || !strings.Contains(h.Why, "downloading linux-6.99") { + t.Fatalf("started 3 h ago: %+v", h) + } + h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", "").UpgradeHealth() + if !h.Healthy { + t.Fatalf("started 10 min ago: %+v", h) + } + h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=\n", "").UpgradeHealth() + if h.Healthy { + t.Fatalf("a start that cannot be read is not healthy: %+v", h) + } +} diff --git a/modules/pacman/cmd/pacman-tools/upgrade_health.go b/modules/pacman/cmd/pacman-tools/upgrade_health.go new file mode 100644 index 00000000..90180130 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/upgrade_health.go @@ -0,0 +1,102 @@ +package main + +import ( + "fmt" + "strings" + "time" +) + +// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node +// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager +// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of +// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and, +// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an +// upgrade passes. + +// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why. +type Health struct { + Healthy bool `json:"healthy"` + Why string `json:"why"` + Found []string `json:"found,omitempty"` +} + +// WeeklyUnit is the service the weekly timer starts. +const WeeklyUnit = "mesh-pacman-weekly-upgrade.service" + +// RunLimit is how long a weekly upgrade may go on before it is unhealthy. The service has no start timeout on +// purpose: killing pacman in the middle of a transaction leaves a half-upgraded machine. A run past the limit +// is a hung one (a stuck hook, a stalled transaction), which holds pacman's database lock so that no later run +// can start; it is said, not killed. +const RunLimit = 2 * time.Hour + +// systemdTime is how systemctl shows a timestamp, in the machine's local time. +const systemdTime = "Mon 2006-01-02 15:04:05 MST" + +// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit +// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A +// unit that cannot be read is unhealthy: failure is loud. +func (m *Machine) UpgradeHealth() Health { + p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp", "ExecMainStartTimestamp") + if err != nil { + return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()} + } + switch { + case p["LoadState"] == "not-found": + return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"} + case p["ActiveState"] == "activating" || p["ActiveState"] == "active": + return m.runningHealth(p["ExecMainStartTimestamp"]) + case p["ActiveState"] != "failed" && p["Result"] == "success": + if at := p["ExecMainExitTimestamp"]; at != "" { + return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"} + } + return Health{Healthy: true, Why: "no weekly upgrade has run yet"} + } + why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"]) + if at := p["ExecMainExitTimestamp"]; at != "" { + why += " at " + at + } + found := []string{} + if said := lastError(m.journal(WeeklyUnit, 40)); said != "" { + why += ": " + said + found = append(found, said) + } + return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found} +} + +// runningHealth judges a run in progress: healthy within RunLimit of its start, unhealthy past it, naming the +// start and the last line the journal holds. +func (m *Machine) runningHealth(started string) Health { + at, err := time.ParseInLocation(systemdTime, started, time.Local) + if err != nil { + return Health{Why: fmt.Sprintf("the weekly upgrade is running and when it started could not be read (%q)", started)} + } + if m.Now().Sub(at) <= RunLimit { + return Health{Healthy: true, Why: "the weekly upgrade is running, since " + started} + } + why := fmt.Sprintf("the weekly upgrade has been running since %s, longer than %s: it is hung, and holds pacman's database lock so no later upgrade can start", started, RunLimit) + found := []string{} + if ls := m.journal(WeeklyUnit, 1); len(ls) > 0 && !strings.HasPrefix(ls[0], "(the journal could not be read") { + why += "; its last line: " + strings.TrimSpace(ls[0]) + found = append(found, strings.TrimSpace(ls[0])) + } + return Health{Why: why, Found: found} +} + +// lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that +// could not be read says so in one line starting with "(", which is no error to name. +func lastError(journal []string) string { + last := "" + for _, l := range journal { + l = strings.TrimSpace(l) + if l == "" || strings.HasPrefix(l, "(the journal could not be read") { + continue + } + last = l + } + for i := len(journal) - 1; i >= 0; i-- { + if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") { + return l + } + } + return last +} diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.service b/modules/pacman/files/mesh-pacman-weekly-upgrade.service new file mode 100644 index 00000000..69741dd7 --- /dev/null +++ b/modules/pacman/files/mesh-pacman-weekly-upgrade.service @@ -0,0 +1,12 @@ +# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. +[Unit] +Description=Weekly full upgrade (the mesh's pacman module, ADR 0477) +Wants=network-online.target +After=network-online.target + +[Service] +# The upgrade pacman_upgrade runs (pacman -Syu --noconfirm), as a oneshot whose result is the module's health +# (pacman_upgrade_health). The mesh restarts no machine for an upgrade: a new kernel waits for the next restart. +Type=oneshot +Nice=10 +ExecStart=/usr/bin/pacman -Syu --noconfirm diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.timer b/modules/pacman/files/mesh-pacman-weekly-upgrade.timer new file mode 100644 index 00000000..a8a9b4d9 --- /dev/null +++ b/modules/pacman/files/mesh-pacman-weekly-upgrade.timer @@ -0,0 +1,11 @@ +# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. +[Unit] +Description=Weekly full upgrade, Saturday 04:00 local time on every node (the mesh's pacman module, ADR 0477) + +[Timer] +# The same moment on every node: no random delay. A node that was off or asleep then upgrades at its next start. +OnCalendar=Sat *-*-* 04:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/modules/pacman/module.json b/modules/pacman/module.json index 746c8c5a..ca1a81a3 100644 --- a/modules/pacman/module.json +++ b/modules/pacman/module.json @@ -19,6 +19,7 @@ "pacman_files", "pacman_updates", "pacman_upgrade", + "pacman_upgrade_health", "pacman_orphans", "pacman_remove_orphans", "pacman_cache", @@ -63,14 +64,53 @@ "type": "service", "unit": "reflector.timer", "state": "running", - "boot": "enabled" + "boot": "enabled", + "health": { + "kind": "unit" + } }, { "id": "cache-cleaning", "type": "service", "unit": "paccache.timer", "state": "running", - "boot": "enabled" + "boot": "enabled", + "health": { + "kind": "unit" + } + }, + { + "id": "upgrade-service", + "type": "file", + "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.service", + "mode": "0644", + "template": "files/mesh-pacman-weekly-upgrade.service" + }, + { + "id": "upgrade-timer", + "type": "file", + "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer", + "mode": "0644", + "template": "files/mesh-pacman-weekly-upgrade.timer" + }, + { + "id": "weekly-upgrade", + "type": "service", + "unit": "mesh-pacman-weekly-upgrade.timer", + "state": "running", + "boot": "enabled", + "restart-on": [ + "upgrade-service", + "upgrade-timer" + ], + "health": { + "kind": "tool", + "tool": "pacman_upgrade_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } } ], "build": {