From cf197fee8927442266d7ee103b1df4a8a5524621 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 07:57:48 +0000 Subject: [PATCH 1/8] Upgrade every node's packages together each Saturday at 04:00, with the outcome as the pacman module's health, so the nodes keep the same toolchain versions (issue 1574, ADR 0477) --- health-undeclared | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/health-undeclared b/health-undeclared index 3fbd022b..311c8dd0 100644 --- a/health-undeclared +++ b/health-undeclared @@ -1,5 +1 @@ -61 -The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240 -rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and -one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running -resource without `health` is refused. +PLACEHOLDER \ No newline at end of file From e319f9877f7e55c2c9d4b40cd412aa24ed7dd0ed Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 07:58:01 +0000 Subject: [PATCH 2/8] Upgrade every node's packages together each Saturday at 04:00, with the outcome as the pacman module's health, so the nodes keep the same toolchain versions (issue 1574, ADR 0477) --- health-undeclared | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/health-undeclared b/health-undeclared index 311c8dd0..0383faf4 100644 --- a/health-undeclared +++ b/health-undeclared @@ -1 +1,5 @@ -PLACEHOLDER \ No newline at end of file +59 +The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240 +rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and +one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running +resource without `health` is refused. From f12f87f1f9ec302b5df0cf91eff17841a27dd796 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 07:59:31 +0000 Subject: [PATCH 3/8] The weekly upgrade's timer, service and health tool (issue 1574, ADR 0477) --- .../pacman/cmd/pacman-tools/upgrade_health.go | 73 +++++++++++++++++++ .../files/mesh-pacman-weekly-upgrade.service | 12 +++ .../files/mesh-pacman-weekly-upgrade.timer | 11 +++ 3 files changed, 96 insertions(+) create mode 100644 modules/pacman/cmd/pacman-tools/upgrade_health.go create mode 100644 modules/pacman/files/mesh-pacman-weekly-upgrade.service create mode 100644 modules/pacman/files/mesh-pacman-weekly-upgrade.timer diff --git a/modules/pacman/cmd/pacman-tools/upgrade_health.go b/modules/pacman/cmd/pacman-tools/upgrade_health.go new file mode 100644 index 00000000..2b817bcf --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/upgrade_health.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "strings" +) + +// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node +// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager +// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of +// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and, +// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an +// upgrade passes. + +// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why. +type Health struct { + Healthy bool `json:"healthy"` + Why string `json:"why"` + Found []string `json:"found,omitempty"` +} + +// WeeklyUnit is the service the weekly timer starts. +const WeeklyUnit = "mesh-pacman-weekly-upgrade.service" + +// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit +// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A +// unit that cannot be read is unhealthy: failure is loud. +func (m *Machine) UpgradeHealth() Health { + p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp") + if err != nil { + return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()} + } + switch { + case p["LoadState"] == "not-found": + return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"} + case p["ActiveState"] == "activating" || p["ActiveState"] == "active": + return Health{Healthy: true, Why: "the weekly upgrade is running"} + case p["ActiveState"] != "failed" && p["Result"] == "success": + if at := p["ExecMainExitTimestamp"]; at != "" { + return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"} + } + return Health{Healthy: true, Why: "no weekly upgrade has run yet"} + } + why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"]) + if at := p["ExecMainExitTimestamp"]; at != "" { + why += " at " + at + } + found := []string{} + if said := lastError(m.journal(WeeklyUnit, 40)); said != "" { + why += ": " + said + found = append(found, said) + } + return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found} +} + +// lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that +// could not be read says so in one line starting with "(", which is no error to name. +func lastError(journal []string) string { + last := "" + for _, l := range journal { + l = strings.TrimSpace(l) + if l == "" || strings.HasPrefix(l, "(the journal could not be read") { + continue + } + last = l + } + for i := len(journal) - 1; i >= 0; i-- { + if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") { + return l + } + } + return last +} diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.service b/modules/pacman/files/mesh-pacman-weekly-upgrade.service new file mode 100644 index 00000000..69741dd7 --- /dev/null +++ b/modules/pacman/files/mesh-pacman-weekly-upgrade.service @@ -0,0 +1,12 @@ +# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. +[Unit] +Description=Weekly full upgrade (the mesh's pacman module, ADR 0477) +Wants=network-online.target +After=network-online.target + +[Service] +# The upgrade pacman_upgrade runs (pacman -Syu --noconfirm), as a oneshot whose result is the module's health +# (pacman_upgrade_health). The mesh restarts no machine for an upgrade: a new kernel waits for the next restart. +Type=oneshot +Nice=10 +ExecStart=/usr/bin/pacman -Syu --noconfirm diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.timer b/modules/pacman/files/mesh-pacman-weekly-upgrade.timer new file mode 100644 index 00000000..a8a9b4d9 --- /dev/null +++ b/modules/pacman/files/mesh-pacman-weekly-upgrade.timer @@ -0,0 +1,11 @@ +# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. +[Unit] +Description=Weekly full upgrade, Saturday 04:00 local time on every node (the mesh's pacman module, ADR 0477) + +[Timer] +# The same moment on every node: no random delay. A node that was off or asleep then upgrades at its next start. +OnCalendar=Sat *-*-* 04:00:00 +Persistent=true + +[Install] +WantedBy=timers.target From 1f5046f47ece60416f4464e2745fb2899ac04dd1 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 07:59:55 +0000 Subject: [PATCH 4/8] Declare the weekly upgrade timer with its health, and serve pacman_upgrade_health (issue 1574, ADR 0477) --- modules/pacman/cmd/pacman-tools/acts.go | 7 +++- modules/pacman/cmd/pacman-tools/main.go | 7 ++++ modules/pacman/module.json | 44 +++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/modules/pacman/cmd/pacman-tools/acts.go b/modules/pacman/cmd/pacman-tools/acts.go index a0b4e3c3..146d6d5e 100644 --- a/modules/pacman/cmd/pacman-tools/acts.go +++ b/modules/pacman/cmd/pacman-tools/acts.go @@ -55,6 +55,11 @@ func (m *Machine) journal(unit string, n int) []string { return lines(out) } +// UpgradeArgs is the full system upgrade, in one place: pacman_upgrade starts pacman with these arguments as +// a transient unit, and the weekly upgrade's service (files/mesh-pacman-weekly-upgrade.service) runs +// pacman with the same ones; a test holds the two together (issue 1574, ADR 0477 clause 5). +var UpgradeArgs = []string{"-Syu", "--noconfirm"} + // Upgrade starts a full system upgrade as a transient unit and answers at once, with the // distribution's news since the last upgrade; or, given a unit it started, answers how it went. func (m *Machine) Upgrade(unit string, n int) (map[string]any, error) { @@ -62,7 +67,7 @@ func (m *Machine) Upgrade(unit string, n int) (map[string]any, error) { return m.UpgradeStatus(unit, n) } news := m.News("") - started, _, err := m.transaction("upgrade", false, "-Syu", "--noconfirm") + started, _, err := m.transaction("upgrade", false, UpgradeArgs...) if err != nil { return nil, err } diff --git a/modules/pacman/cmd/pacman-tools/main.go b/modules/pacman/cmd/pacman-tools/main.go index 06ead071..c4d7136a 100644 --- a/modules/pacman/cmd/pacman-tools/main.go +++ b/modules/pacman/cmd/pacman-tools/main.go @@ -152,6 +152,13 @@ func tools(m *Machine) []stdio.Tool { return m.Upgrade(unit, n) }, }, + { + Name: "pacman_upgrade_health", + Description: "Whether the weekly full upgrade (Saturday 04:00 local time, ADR 0477) last passed, as the module's health check: unhealthy " + + "with the exit and the last error of the journal when it failed, healthy while it runs, after it passed and before it ever ran. (r)", + Input: schema(map[string]any{}), + Run: func(map[string]any) (any, error) { return m.UpgradeHealth(), nil }, + }, { Name: "pacman_orphans", Description: "Packages installed as dependencies that nothing requires any more (pacman -Qdt), with versions.", diff --git a/modules/pacman/module.json b/modules/pacman/module.json index 746c8c5a..ca1a81a3 100644 --- a/modules/pacman/module.json +++ b/modules/pacman/module.json @@ -19,6 +19,7 @@ "pacman_files", "pacman_updates", "pacman_upgrade", + "pacman_upgrade_health", "pacman_orphans", "pacman_remove_orphans", "pacman_cache", @@ -63,14 +64,53 @@ "type": "service", "unit": "reflector.timer", "state": "running", - "boot": "enabled" + "boot": "enabled", + "health": { + "kind": "unit" + } }, { "id": "cache-cleaning", "type": "service", "unit": "paccache.timer", "state": "running", - "boot": "enabled" + "boot": "enabled", + "health": { + "kind": "unit" + } + }, + { + "id": "upgrade-service", + "type": "file", + "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.service", + "mode": "0644", + "template": "files/mesh-pacman-weekly-upgrade.service" + }, + { + "id": "upgrade-timer", + "type": "file", + "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer", + "mode": "0644", + "template": "files/mesh-pacman-weekly-upgrade.timer" + }, + { + "id": "weekly-upgrade", + "type": "service", + "unit": "mesh-pacman-weekly-upgrade.timer", + "state": "running", + "boot": "enabled", + "restart-on": [ + "upgrade-service", + "upgrade-timer" + ], + "health": { + "kind": "tool", + "tool": "pacman_upgrade_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } } ], "build": { From e3ac017c284439fed978d56f9184f8f63a09428c Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 08:00:21 +0000 Subject: [PATCH 5/8] Test the weekly upgrade's timer and its health (issue 1574, ADR 0477) --- .../pacman/cmd/pacman-tools/manifest_test.go | 50 +++++++++++++++++++ .../pacman/cmd/pacman-tools/pacman_test.go | 35 +++++++++++++ 2 files changed, 85 insertions(+) diff --git a/modules/pacman/cmd/pacman-tools/manifest_test.go b/modules/pacman/cmd/pacman-tools/manifest_test.go index b55f6908..da7fb4ad 100644 --- a/modules/pacman/cmd/pacman-tools/manifest_test.go +++ b/modules/pacman/cmd/pacman-tools/manifest_test.go @@ -101,3 +101,53 @@ func TestTheReflectorPackageIsDeclaredBeforeTheFileItShips(t *testing.T) { t.Fatalf("a package's file and timer come after the package: %v", order) } } + +// The weekly upgrade (issue 1574, ADR 0477 clause 5): a timer on Saturday 04:00 local time, persistent so a +// node that was off upgrades at its next start, the same moment on every node (no random delay), starting a +// oneshot service that runs the very upgrade pacman_upgrade runs. +func TestTheWeeklyUpgradeIsATimerOnSaturdayFourPersistentRunningTheToolsUpgrade(t *testing.T) { + m := manifest(t) + timer := m.resource(t, "upgrade-timer") + service := m.resource(t, "upgrade-service") + if timer["path"] != "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer" || service["path"] != "/etc/systemd/system/"+WeeklyUnit { + t.Fatalf("paths: %v %v", timer["path"], service["path"]) + } + tt, st := templateText(t, timer), templateText(t, service) + for _, want := range []string{"\nOnCalendar=Sat *-*-* 04:00:00\n", "\nPersistent=true\n", "\nWantedBy=timers.target\n"} { + if !strings.Contains(tt, want) { + t.Errorf("the timer lacks %q:\n%s", strings.TrimSpace(want), tt) + } + } + for _, not := range []string{"RandomizedDelaySec", "OnBootSec", "OnUnitActiveSec", "UTC"} { + if strings.Contains(tt, not) { + t.Errorf("the timer says %s: the same local moment on every node, and no other moment", not) + } + } + if strings.Count(tt, "OnCalendar=") != 1 { + t.Errorf("one moment only:\n%s", tt) + } + if !strings.Contains(st, "\nType=oneshot\n") { + t.Errorf("the service is a oneshot:\n%s", st) + } + if want := "\nExecStart=/usr/bin/pacman " + strings.Join(UpgradeArgs, " ") + "\n"; !strings.Contains(st, want) { + t.Errorf("the service does not run the upgrade pacman_upgrade runs (%q):\n%s", want, st) + } + if strings.Contains(st, "reboot") || strings.Contains(tt, "reboot") || strings.Contains(st, "systemctl") { + t.Error("the mesh restarts no machine for an upgrade") + } + r := m.resource(t, "weekly-upgrade") + if r["type"] != "service" || r["unit"] != "mesh-pacman-weekly-upgrade.timer" || r["state"] != "running" || r["boot"] != "enabled" { + t.Fatalf("the timer runs and is enabled: %v", r) + } + h, _ := r["health"].(map[string]any) + if h["kind"] != "tool" || h["tool"] != "pacman_upgrade_health" { + t.Errorf("the upgrade's outcome is the resource's health: %v", r["health"]) + } + order := map[string]int{} + for i, x := range m.Resources { + order[x["id"].(string)] = i + } + if order["upgrade-service"] > order["weekly-upgrade"] || order["upgrade-timer"] > order["weekly-upgrade"] || order["package"] > order["upgrade-service"] { + t.Errorf("the units are placed before the timer is started: %v", order) + } +} diff --git a/modules/pacman/cmd/pacman-tools/pacman_test.go b/modules/pacman/cmd/pacman-tools/pacman_test.go index 83e5196b..c17b9d48 100644 --- a/modules/pacman/cmd/pacman-tools/pacman_test.go +++ b/modules/pacman/cmd/pacman-tools/pacman_test.go @@ -374,3 +374,38 @@ func TestConfigIsReadAsPacmanConfResolvesIt(t *testing.T) { t.Fatalf("%+v", c) } } + +// The weekly upgrade's outcome is the module's health (issue 1574, ADR 0477 clause 5). +func upgradeHealthMachine(props, journal string) *Machine { + return machine(byLine(map[string]Ran{ + "systemctl show " + WeeklyUnit + " --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecMainExitTimestamp": {Stdout: props}, + "sudo -n journalctl --no-pager -o cat -n 40 -u " + WeeklyUnit: {Stdout: journal}, + }, nil), 1000) +} + +func TestAFailedWeeklyUpgradeIsUnhealthyNamingTheError(t *testing.T) { + h := upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=Sat 2026-10-24 04:03:11 CEST\n", + ":: Synchronizing package databases...\nerror: failed to commit transaction (conflicting files)\nErrors occurred, no packages were upgraded.\n").UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "failed to commit transaction (conflicting files)") || !strings.Contains(h.Why, "exit-code") || !strings.Contains(h.Why, "Sat 2026-10-24 04:03:11 CEST") { + t.Fatalf("%+v", h) + } + h = upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=\n", "").UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "status 1") { + t.Fatalf("a failure without a journal is still unhealthy: %+v", h) + } +} + +func TestAPassedRunningOrNeverRunWeeklyUpgradeIsHealthy(t *testing.T) { + for name, props := range map[string]string{ + "passed": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=Sat 2026-10-24 04:09:00 CEST\n", + "running": "LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n", + "never run": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n", + } { + if h := upgradeHealthMachine(props, "").UpgradeHealth(); !h.Healthy { + t.Errorf("%s: %+v", name, h) + } + } + if h := machine(fake(func(call) Ran { return Ran{Status: 1, Stderr: "boom"} }, nil), 1000).UpgradeHealth(); h.Healthy { + t.Errorf("a unit that cannot be read is not healthy: %+v", h) + } +} From 5123c1dd57c44a79f5a9fe9e4c003a00128e7dc8 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 08:00:30 +0000 Subject: [PATCH 6/8] Say the weekly upgrade and its health in the pacman README (issue 1574, ADR 0477) --- modules/pacman/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/modules/pacman/README.md b/modules/pacman/README.md index eb5ebaa8..7356c322 100644 --- a/modules/pacman/README.md +++ b/modules/pacman/README.md @@ -23,6 +23,19 @@ that seat (ADR 0207). list stays reflector's to write, not the mesh's. - **Cache cleaning.** The `pacman-contrib` package and `paccache.timer` running and enabled. Each week it keeps the last three versions of each package. +- **The weekly upgrade (ADR 0477 clause 5, issue 1574).** `mesh-pacman-weekly-upgrade.timer` and its + oneshot service, placed whole in `/etc/systemd/system`. The timer fires every Saturday at 04:00 in the + node's local time, the same moment on every node (no random delay), and is `Persistent=true`: a node that + was off or asleep then upgrades at its next start. The service runs `/usr/bin/pacman -Syu --noconfirm`, the + same arguments `pacman_upgrade` starts (`UpgradeArgs` in the code; a manifest test holds the two together). + No other moment upgrades but a hand `pacman_upgrade`, and the mesh restarts no machine for an upgrade: a new + kernel waits for the next restart. + - **The outcome is the module's health.** The timer's resource asks `pacman_upgrade_health`: unhealthy, + naming the exit and the last `error:` line of the journal, when the last weekly upgrade failed; healthy + while it runs, after it passed, and before it ever ran. The controller raises the module's condition for + the node from it. A failed service stays failed until the next run, so the condition stands until an + upgrade passes, a hand `pacman_upgrade` does not clear it. A restart of the machine forgets the failed + state (the service manager's), so it is the condition raised before it that keeps the record. ## What it improves From d4d1a1591b96a2b525eb17730cff831dfdff82a3 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 08:00:39 +0000 Subject: [PATCH 7/8] Name pacman_upgrade_health in the pacman README's tools (issue 1574) --- modules/pacman/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/pacman/README.md b/modules/pacman/README.md index 7356c322..ae53b6b8 100644 --- a/modules/pacman/README.md +++ b/modules/pacman/README.md @@ -63,6 +63,7 @@ that seat (ADR 0207). | `pacman_files` | r | what a package placed, bounded | | `pacman_updates` | r | `checkupdates`: what a full upgrade would change, never setting up a partial upgrade | | `pacman_upgrade` | a | starts `pacman -Syu --noconfirm` (sudo -n) as a transient unit that outlives the call; answers the unit and the news since the last upgrade; given the unit, how it went | +| `pacman_upgrade_health` | r | whether the weekly upgrade last passed: the timer's health check (see the weekly upgrade above) | | `pacman_orphans` | r | `pacman -Qdt` | | `pacman_remove_orphans` | a | `pacman -Rs` on named orphans, or all of them, as a unit of its own; a name that is not an orphan is refused | | `pacman_cache` | r/a | size, interrupted downloads, what paccache would free keeping N; `clean: true` removes them | From bd10ec4f0dbebe918d91a004682b00c5bdac5c31 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Wed, 21 Oct 2026 08:08:04 +0000 Subject: [PATCH 8/8] Say a weekly upgrade that hangs past two hours, so a stuck pacman lock is loud (issue 1574, ADR 0477) --- modules/pacman/README.md | 10 ++++-- .../pacman/cmd/pacman-tools/pacman_test.go | 23 +++++++++++-- .../pacman/cmd/pacman-tools/upgrade_health.go | 33 +++++++++++++++++-- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/modules/pacman/README.md b/modules/pacman/README.md index ae53b6b8..4ba4969a 100644 --- a/modules/pacman/README.md +++ b/modules/pacman/README.md @@ -33,9 +33,13 @@ that seat (ADR 0207). - **The outcome is the module's health.** The timer's resource asks `pacman_upgrade_health`: unhealthy, naming the exit and the last `error:` line of the journal, when the last weekly upgrade failed; healthy while it runs, after it passed, and before it ever ran. The controller raises the module's condition for - the node from it. A failed service stays failed until the next run, so the condition stands until an - upgrade passes, a hand `pacman_upgrade` does not clear it. A restart of the machine forgets the failed - state (the service manager's), so it is the condition raised before it that keeps the record. + the node from it. A failed service stays failed until + the next run, so the condition stands until an upgrade passes; a hand `pacman_upgrade` does not clear + it. A restart of the machine clears the failed state and with it the condition, and the next Saturday + runs again. A run that goes on longer than two hours is unhealthy too, naming when it started and the + last line of its journal: it is hung and holds pacman's database lock. The service has no start + timeout on purpose, because killing pacman in the middle of a transaction is worse; the hung run is + said, not killed. ## What it improves diff --git a/modules/pacman/cmd/pacman-tools/pacman_test.go b/modules/pacman/cmd/pacman-tools/pacman_test.go index c17b9d48..4ca7f5bb 100644 --- a/modules/pacman/cmd/pacman-tools/pacman_test.go +++ b/modules/pacman/cmd/pacman-tools/pacman_test.go @@ -378,8 +378,9 @@ func TestConfigIsReadAsPacmanConfResolvesIt(t *testing.T) { // The weekly upgrade's outcome is the module's health (issue 1574, ADR 0477 clause 5). func upgradeHealthMachine(props, journal string) *Machine { return machine(byLine(map[string]Ran{ - "systemctl show " + WeeklyUnit + " --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecMainExitTimestamp": {Stdout: props}, + "systemctl show " + WeeklyUnit + " --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecMainExitTimestamp --property=ExecMainStartTimestamp": {Stdout: props}, "sudo -n journalctl --no-pager -o cat -n 40 -u " + WeeklyUnit: {Stdout: journal}, + "sudo -n journalctl --no-pager -o cat -n 1 -u " + WeeklyUnit: {Stdout: journal}, }, nil), 1000) } @@ -398,7 +399,7 @@ func TestAFailedWeeklyUpgradeIsUnhealthyNamingTheError(t *testing.T) { func TestAPassedRunningOrNeverRunWeeklyUpgradeIsHealthy(t *testing.T) { for name, props := range map[string]string{ "passed": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=Sat 2026-10-24 04:09:00 CEST\n", - "running": "LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n", + "running": "LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", "never run": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n", } { if h := upgradeHealthMachine(props, "").UpgradeHealth(); !h.Healthy { @@ -409,3 +410,21 @@ func TestAPassedRunningOrNeverRunWeeklyUpgradeIsHealthy(t *testing.T) { t.Errorf("a unit that cannot be read is not healthy: %+v", h) } } + +// A run that goes on past RunLimit is hung and holds pacman's lock: unhealthy, naming its start (issue 1574). +func TestAHungWeeklyUpgradeIsUnhealthyNamingItsStart(t *testing.T) { + // the test machine's clock is 2026-10-04 12:00 UTC + h := upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 09:00:00 UTC\n", + "downloading linux-6.99...\n").UpgradeHealth() + if h.Healthy || !strings.Contains(h.Why, "Wed 2026-10-04 09:00:00 UTC") || !strings.Contains(h.Why, "downloading linux-6.99") { + t.Fatalf("started 3 h ago: %+v", h) + } + h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", "").UpgradeHealth() + if !h.Healthy { + t.Fatalf("started 10 min ago: %+v", h) + } + h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=\n", "").UpgradeHealth() + if h.Healthy { + t.Fatalf("a start that cannot be read is not healthy: %+v", h) + } +} diff --git a/modules/pacman/cmd/pacman-tools/upgrade_health.go b/modules/pacman/cmd/pacman-tools/upgrade_health.go index 2b817bcf..90180130 100644 --- a/modules/pacman/cmd/pacman-tools/upgrade_health.go +++ b/modules/pacman/cmd/pacman-tools/upgrade_health.go @@ -3,6 +3,7 @@ package main import ( "fmt" "strings" + "time" ) // The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node @@ -22,11 +23,20 @@ type Health struct { // WeeklyUnit is the service the weekly timer starts. const WeeklyUnit = "mesh-pacman-weekly-upgrade.service" +// RunLimit is how long a weekly upgrade may go on before it is unhealthy. The service has no start timeout on +// purpose: killing pacman in the middle of a transaction leaves a half-upgraded machine. A run past the limit +// is a hung one (a stuck hook, a stalled transaction), which holds pacman's database lock so that no later run +// can start; it is said, not killed. +const RunLimit = 2 * time.Hour + +// systemdTime is how systemctl shows a timestamp, in the machine's local time. +const systemdTime = "Mon 2006-01-02 15:04:05 MST" + // UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit // and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A // unit that cannot be read is unhealthy: failure is loud. func (m *Machine) UpgradeHealth() Health { - p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp") + p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp", "ExecMainStartTimestamp") if err != nil { return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()} } @@ -34,7 +44,7 @@ func (m *Machine) UpgradeHealth() Health { case p["LoadState"] == "not-found": return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"} case p["ActiveState"] == "activating" || p["ActiveState"] == "active": - return Health{Healthy: true, Why: "the weekly upgrade is running"} + return m.runningHealth(p["ExecMainStartTimestamp"]) case p["ActiveState"] != "failed" && p["Result"] == "success": if at := p["ExecMainExitTimestamp"]; at != "" { return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"} @@ -53,6 +63,25 @@ func (m *Machine) UpgradeHealth() Health { return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found} } +// runningHealth judges a run in progress: healthy within RunLimit of its start, unhealthy past it, naming the +// start and the last line the journal holds. +func (m *Machine) runningHealth(started string) Health { + at, err := time.ParseInLocation(systemdTime, started, time.Local) + if err != nil { + return Health{Why: fmt.Sprintf("the weekly upgrade is running and when it started could not be read (%q)", started)} + } + if m.Now().Sub(at) <= RunLimit { + return Health{Healthy: true, Why: "the weekly upgrade is running, since " + started} + } + why := fmt.Sprintf("the weekly upgrade has been running since %s, longer than %s: it is hung, and holds pacman's database lock so no later upgrade can start", started, RunLimit) + found := []string{} + if ls := m.journal(WeeklyUnit, 1); len(ls) > 0 && !strings.HasPrefix(ls[0], "(the journal could not be read") { + why += "; its last line: " + strings.TrimSpace(ls[0]) + found = append(found, strings.TrimSpace(ls[0])) + } + return Health{Why: why, Found: found} +} + // lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that // could not be read says so in one line starting with "(", which is no error to name. func lastError(journal []string) string {