Merge pull request 'Upgrade every node's packages on a Saturday 04:00 timer and make its outcome the pacman module's health, so the nodes stay on the same versions (issue 1574, ADR 0477)' (#404) from fix/1574-weekly-upgrade into main

health-undeclared

@@ -1,4 +1,4 @@
-61
+59
 The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
 rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
 one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running

modules/pacman/README.md

@@ -23,6 +23,23 @@ that seat (ADR 0207).
   list stays reflector's to write, not the mesh's.
 - **Cache cleaning.** The `pacman-contrib` package and `paccache.timer` running and enabled. Each
   week it keeps the last three versions of each package.
+- **The weekly upgrade (ADR 0477 clause 5, issue 1574).** `mesh-pacman-weekly-upgrade.timer` and its
+  oneshot service, placed whole in `/etc/systemd/system`. The timer fires every Saturday at 04:00 in the
+  node's local time, the same moment on every node (no random delay), and is `Persistent=true`: a node that
+  was off or asleep then upgrades at its next start. The service runs `/usr/bin/pacman -Syu --noconfirm`, the
+  same arguments `pacman_upgrade` starts (`UpgradeArgs` in the code; a manifest test holds the two together).
+  No other moment upgrades but a hand `pacman_upgrade`, and the mesh restarts no machine for an upgrade: a new
+  kernel waits for the next restart.
+  - **The outcome is the module's health.** The timer's resource asks `pacman_upgrade_health`: unhealthy,
+    naming the exit and the last `error:` line of the journal, when the last weekly upgrade failed; healthy
+    while it runs, after it passed, and before it ever ran. The controller raises the module's condition for
+    the node from it. A failed service stays failed until
+    the next run, so the condition stands until an upgrade passes; a hand `pacman_upgrade` does not clear
+    it. A restart of the machine clears the failed state and with it the condition, and the next Saturday
+    runs again. A run that goes on longer than two hours is unhealthy too, naming when it started and the
+    last line of its journal: it is hung and holds pacman's database lock. The service has no start
+    timeout on purpose, because killing pacman in the middle of a transaction is worse; the hung run is
+    said, not killed.
 
 ## What it improves
 
@@ -50,6 +67,7 @@ that seat (ADR 0207).
 | `pacman_files` | r | what a package placed, bounded |
 | `pacman_updates` | r | `checkupdates`: what a full upgrade would change, never setting up a partial upgrade |
 | `pacman_upgrade` | a | starts `pacman -Syu --noconfirm` (sudo -n) as a transient unit that outlives the call; answers the unit and the news since the last upgrade; given the unit, how it went |
+| `pacman_upgrade_health` | r | whether the weekly upgrade last passed: the timer's health check (see the weekly upgrade above) |
 | `pacman_orphans` | r | `pacman -Qdt` |
 | `pacman_remove_orphans` | a | `pacman -Rs` on named orphans, or all of them, as a unit of its own; a name that is not an orphan is refused |
 | `pacman_cache` | r/a | size, interrupted downloads, what paccache would free keeping N; `clean: true` removes them |

modules/pacman/cmd/pacman-tools/acts.go

@@ -55,6 +55,11 @@ func (m *Machine) journal(unit string, n int) []string {
 	return lines(out)
 }
 
+// UpgradeArgs is the full system upgrade, in one place: pacman_upgrade starts pacman with these arguments as
+// a transient unit, and the weekly upgrade's service (files/mesh-pacman-weekly-upgrade.service) runs
+// pacman with the same ones; a test holds the two together (issue 1574, ADR 0477 clause 5).
+var UpgradeArgs = []string{"-Syu", "--noconfirm"}
+
 // Upgrade starts a full system upgrade as a transient unit and answers at once, with the
 // distribution's news since the last upgrade; or, given a unit it started, answers how it went.
 func (m *Machine) Upgrade(unit string, n int) (map[string]any, error) {
@@ -62,7 +67,7 @@ func (m *Machine) Upgrade(unit string, n int) (map[string]any, error) {
 		return m.UpgradeStatus(unit, n)
 	}
 	news := m.News("")
-	started, _, err := m.transaction("upgrade", false, "-Syu", "--noconfirm")
+	started, _, err := m.transaction("upgrade", false, UpgradeArgs...)
 	if err != nil {
 		return nil, err
 	}

modules/pacman/cmd/pacman-tools/main.go

@@ -152,6 +152,13 @@ func tools(m *Machine) []stdio.Tool {
 				return m.Upgrade(unit, n)
 			},
 		},
+		{
+			Name: "pacman_upgrade_health",
+			Description: "Whether the weekly full upgrade (Saturday 04:00 local time, ADR 0477) last passed, as the module's health check: unhealthy " +
+				"with the exit and the last error of the journal when it failed, healthy while it runs, after it passed and before it ever ran. (r)",
+			Input: schema(map[string]any{}),
+			Run:   func(map[string]any) (any, error) { return m.UpgradeHealth(), nil },
+		},
 		{
 			Name:        "pacman_orphans",
 			Description: "Packages installed as dependencies that nothing requires any more (pacman -Qdt), with versions.",

modules/pacman/cmd/pacman-tools/manifest_test.go

@@ -101,3 +101,53 @@ func TestTheReflectorPackageIsDeclaredBeforeTheFileItShips(t *testing.T) {
 		t.Fatalf("a package's file and timer come after the package: %v", order)
 	}
 }
+
+// The weekly upgrade (issue 1574, ADR 0477 clause 5): a timer on Saturday 04:00 local time, persistent so a
+// node that was off upgrades at its next start, the same moment on every node (no random delay), starting a
+// oneshot service that runs the very upgrade pacman_upgrade runs.
+func TestTheWeeklyUpgradeIsATimerOnSaturdayFourPersistentRunningTheToolsUpgrade(t *testing.T) {
+	m := manifest(t)
+	timer := m.resource(t, "upgrade-timer")
+	service := m.resource(t, "upgrade-service")
+	if timer["path"] != "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer" || service["path"] != "/etc/systemd/system/"+WeeklyUnit {
+		t.Fatalf("paths: %v %v", timer["path"], service["path"])
+	}
+	tt, st := templateText(t, timer), templateText(t, service)
+	for _, want := range []string{"\nOnCalendar=Sat *-*-* 04:00:00\n", "\nPersistent=true\n", "\nWantedBy=timers.target\n"} {
+		if !strings.Contains(tt, want) {
+			t.Errorf("the timer lacks %q:\n%s", strings.TrimSpace(want), tt)
+		}
+	}
+	for _, not := range []string{"RandomizedDelaySec", "OnBootSec", "OnUnitActiveSec", "UTC"} {
+		if strings.Contains(tt, not) {
+			t.Errorf("the timer says %s: the same local moment on every node, and no other moment", not)
+		}
+	}
+	if strings.Count(tt, "OnCalendar=") != 1 {
+		t.Errorf("one moment only:\n%s", tt)
+	}
+	if !strings.Contains(st, "\nType=oneshot\n") {
+		t.Errorf("the service is a oneshot:\n%s", st)
+	}
+	if want := "\nExecStart=/usr/bin/pacman " + strings.Join(UpgradeArgs, " ") + "\n"; !strings.Contains(st, want) {
+		t.Errorf("the service does not run the upgrade pacman_upgrade runs (%q):\n%s", want, st)
+	}
+	if strings.Contains(st, "reboot") || strings.Contains(tt, "reboot") || strings.Contains(st, "systemctl") {
+		t.Error("the mesh restarts no machine for an upgrade")
+	}
+	r := m.resource(t, "weekly-upgrade")
+	if r["type"] != "service" || r["unit"] != "mesh-pacman-weekly-upgrade.timer" || r["state"] != "running" || r["boot"] != "enabled" {
+		t.Fatalf("the timer runs and is enabled: %v", r)
+	}
+	h, _ := r["health"].(map[string]any)
+	if h["kind"] != "tool" || h["tool"] != "pacman_upgrade_health" {
+		t.Errorf("the upgrade's outcome is the resource's health: %v", r["health"])
+	}
+	order := map[string]int{}
+	for i, x := range m.Resources {
+		order[x["id"].(string)] = i
+	}
+	if order["upgrade-service"] > order["weekly-upgrade"] || order["upgrade-timer"] > order["weekly-upgrade"] || order["package"] > order["upgrade-service"] {
+		t.Errorf("the units are placed before the timer is started: %v", order)
+	}
+}

modules/pacman/cmd/pacman-tools/pacman_test.go

@@ -374,3 +374,57 @@ func TestConfigIsReadAsPacmanConfResolvesIt(t *testing.T) {
 		t.Fatalf("%+v", c)
 	}
 }
+
+// The weekly upgrade's outcome is the module's health (issue 1574, ADR 0477 clause 5).
+func upgradeHealthMachine(props, journal string) *Machine {
+	return machine(byLine(map[string]Ran{
+		"systemctl show " + WeeklyUnit + " --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecMainExitTimestamp --property=ExecMainStartTimestamp": {Stdout: props},
+		"sudo -n journalctl --no-pager -o cat -n 40 -u " + WeeklyUnit: {Stdout: journal},
+		"sudo -n journalctl --no-pager -o cat -n 1 -u " + WeeklyUnit:  {Stdout: journal},
+	}, nil), 1000)
+}
+
+func TestAFailedWeeklyUpgradeIsUnhealthyNamingTheError(t *testing.T) {
+	h := upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=Sat 2026-10-24 04:03:11 CEST\n",
+		":: Synchronizing package databases...\nerror: failed to commit transaction (conflicting files)\nErrors occurred, no packages were upgraded.\n").UpgradeHealth()
+	if h.Healthy || !strings.Contains(h.Why, "failed to commit transaction (conflicting files)") || !strings.Contains(h.Why, "exit-code") || !strings.Contains(h.Why, "Sat 2026-10-24 04:03:11 CEST") {
+		t.Fatalf("%+v", h)
+	}
+	h = upgradeHealthMachine("LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\nExecMainExitTimestamp=\n", "").UpgradeHealth()
+	if h.Healthy || !strings.Contains(h.Why, "status 1") {
+		t.Fatalf("a failure without a journal is still unhealthy: %+v", h)
+	}
+}
+
+func TestAPassedRunningOrNeverRunWeeklyUpgradeIsHealthy(t *testing.T) {
+	for name, props := range map[string]string{
+		"passed":    "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=Sat 2026-10-24 04:09:00 CEST\n",
+		"running":   "LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n",
+		"never run": "LoadState=loaded\nActiveState=inactive\nResult=success\nExecMainStatus=0\nExecMainExitTimestamp=\n",
+	} {
+		if h := upgradeHealthMachine(props, "").UpgradeHealth(); !h.Healthy {
+			t.Errorf("%s: %+v", name, h)
+		}
+	}
+	if h := machine(fake(func(call) Ran { return Ran{Status: 1, Stderr: "boom"} }, nil), 1000).UpgradeHealth(); h.Healthy {
+		t.Errorf("a unit that cannot be read is not healthy: %+v", h)
+	}
+}
+
+// A run that goes on past RunLimit is hung and holds pacman's lock: unhealthy, naming its start (issue 1574).
+func TestAHungWeeklyUpgradeIsUnhealthyNamingItsStart(t *testing.T) {
+	// the test machine's clock is 2026-10-04 12:00 UTC
+	h := upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 09:00:00 UTC\n",
+		"downloading linux-6.99...\n").UpgradeHealth()
+	if h.Healthy || !strings.Contains(h.Why, "Wed 2026-10-04 09:00:00 UTC") || !strings.Contains(h.Why, "downloading linux-6.99") {
+		t.Fatalf("started 3 h ago: %+v", h)
+	}
+	h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=Wed 2026-10-04 11:50:00 UTC\n", "").UpgradeHealth()
+	if !h.Healthy {
+		t.Fatalf("started 10 min ago: %+v", h)
+	}
+	h = upgradeHealthMachine("LoadState=loaded\nActiveState=activating\nResult=success\nExecMainStatus=0\nExecMainStartTimestamp=\n", "").UpgradeHealth()
+	if h.Healthy {
+		t.Fatalf("a start that cannot be read is not healthy: %+v", h)
+	}
+}

modules/pacman/cmd/pacman-tools/upgrade_health.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node
+// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager
+// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of
+// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and,
+// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an
+// upgrade passes.
+
+// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why.
+type Health struct {
+	Healthy bool     `json:"healthy"`
+	Why     string   `json:"why"`
+	Found   []string `json:"found,omitempty"`
+}
+
+// WeeklyUnit is the service the weekly timer starts.
+const WeeklyUnit = "mesh-pacman-weekly-upgrade.service"
+
+// RunLimit is how long a weekly upgrade may go on before it is unhealthy. The service has no start timeout on
+// purpose: killing pacman in the middle of a transaction leaves a half-upgraded machine. A run past the limit
+// is a hung one (a stuck hook, a stalled transaction), which holds pacman's database lock so that no later run
+// can start; it is said, not killed.
+const RunLimit = 2 * time.Hour
+
+// systemdTime is how systemctl shows a timestamp, in the machine's local time.
+const systemdTime = "Mon 2006-01-02 15:04:05 MST"
+
+// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit
+// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A
+// unit that cannot be read is unhealthy: failure is loud.
+func (m *Machine) UpgradeHealth() Health {
+	p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp", "ExecMainStartTimestamp")
+	if err != nil {
+		return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()}
+	}
+	switch {
+	case p["LoadState"] == "not-found":
+		return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"}
+	case p["ActiveState"] == "activating" || p["ActiveState"] == "active":
+		return m.runningHealth(p["ExecMainStartTimestamp"])
+	case p["ActiveState"] != "failed" && p["Result"] == "success":
+		if at := p["ExecMainExitTimestamp"]; at != "" {
+			return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"}
+		}
+		return Health{Healthy: true, Why: "no weekly upgrade has run yet"}
+	}
+	why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"])
+	if at := p["ExecMainExitTimestamp"]; at != "" {
+		why += " at " + at
+	}
+	found := []string{}
+	if said := lastError(m.journal(WeeklyUnit, 40)); said != "" {
+		why += ": " + said
+		found = append(found, said)
+	}
+	return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found}
+}
+
+// runningHealth judges a run in progress: healthy within RunLimit of its start, unhealthy past it, naming the
+// start and the last line the journal holds.
+func (m *Machine) runningHealth(started string) Health {
+	at, err := time.ParseInLocation(systemdTime, started, time.Local)
+	if err != nil {
+		return Health{Why: fmt.Sprintf("the weekly upgrade is running and when it started could not be read (%q)", started)}
+	}
+	if m.Now().Sub(at) <= RunLimit {
+		return Health{Healthy: true, Why: "the weekly upgrade is running, since " + started}
+	}
+	why := fmt.Sprintf("the weekly upgrade has been running since %s, longer than %s: it is hung, and holds pacman's database lock so no later upgrade can start", started, RunLimit)
+	found := []string{}
+	if ls := m.journal(WeeklyUnit, 1); len(ls) > 0 && !strings.HasPrefix(ls[0], "(the journal could not be read") {
+		why += "; its last line: " + strings.TrimSpace(ls[0])
+		found = append(found, strings.TrimSpace(ls[0]))
+	}
+	return Health{Why: why, Found: found}
+}
+
+// lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that
+// could not be read says so in one line starting with "(", which is no error to name.
+func lastError(journal []string) string {
+	last := ""
+	for _, l := range journal {
+		l = strings.TrimSpace(l)
+		if l == "" || strings.HasPrefix(l, "(the journal could not be read") {
+			continue
+		}
+		last = l
+	}
+	for i := len(journal) - 1; i >= 0; i-- {
+		if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") {
+			return l
+		}
+	}
+	return last
+}

modules/pacman/files/mesh-pacman-weekly-upgrade.service

@@ -0,0 +1,12 @@
+# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push.
+[Unit]
+Description=Weekly full upgrade (the mesh's pacman module, ADR 0477)
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+# The upgrade pacman_upgrade runs (pacman -Syu --noconfirm), as a oneshot whose result is the module's health
+# (pacman_upgrade_health). The mesh restarts no machine for an upgrade: a new kernel waits for the next restart.
+Type=oneshot
+Nice=10
+ExecStart=/usr/bin/pacman -Syu --noconfirm

modules/pacman/files/mesh-pacman-weekly-upgrade.timer

@@ -0,0 +1,11 @@
+# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push.
+[Unit]
+Description=Weekly full upgrade, Saturday 04:00 local time on every node (the mesh's pacman module, ADR 0477)
+
+[Timer]
+# The same moment on every node: no random delay. A node that was off or asleep then upgrades at its next start.
+OnCalendar=Sat *-*-* 04:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

modules/pacman/module.json

@@ -19,6 +19,7 @@
     "pacman_files",
     "pacman_updates",
     "pacman_upgrade",
+    "pacman_upgrade_health",
     "pacman_orphans",
     "pacman_remove_orphans",
     "pacman_cache",
@@ -63,14 +64,53 @@
       "type": "service",
       "unit": "reflector.timer",
       "state": "running",
-      "boot": "enabled"
+      "boot": "enabled",
+      "health": {
+        "kind": "unit"
+      }
     },
     {
       "id": "cache-cleaning",
       "type": "service",
       "unit": "paccache.timer",
       "state": "running",
-      "boot": "enabled"
+      "boot": "enabled",
+      "health": {
+        "kind": "unit"
+      }
+    },
+    {
+      "id": "upgrade-service",
+      "type": "file",
+      "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.service",
+      "mode": "0644",
+      "template": "files/mesh-pacman-weekly-upgrade.service"
+    },
+    {
+      "id": "upgrade-timer",
+      "type": "file",
+      "path": "/etc/systemd/system/mesh-pacman-weekly-upgrade.timer",
+      "mode": "0644",
+      "template": "files/mesh-pacman-weekly-upgrade.timer"
+    },
+    {
+      "id": "weekly-upgrade",
+      "type": "service",
+      "unit": "mesh-pacman-weekly-upgrade.timer",
+      "state": "running",
+      "boot": "enabled",
+      "restart-on": [
+        "upgrade-service",
+        "upgrade-timer"
+      ],
+      "health": {
+        "kind": "tool",
+        "tool": "pacman_upgrade_health",
+        "interval": "60s",
+        "timeout": "20s",
+        "looks": 2,
+        "grace": "60s"
+      }
     }
   ],
   "build": {