The weekly upgrade's timer, service and health tool (issue 1574, ADR 0477)

modules/pacman/cmd/pacman-tools/upgrade_health.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node
+// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager
+// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of
+// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and,
+// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an
+// upgrade passes.
+
+// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why.
+type Health struct {
+	Healthy bool     `json:"healthy"`
+	Why     string   `json:"why"`
+	Found   []string `json:"found,omitempty"`
+}
+
+// WeeklyUnit is the service the weekly timer starts.
+const WeeklyUnit = "mesh-pacman-weekly-upgrade.service"
+
+// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit
+// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A
+// unit that cannot be read is unhealthy: failure is loud.
+func (m *Machine) UpgradeHealth() Health {
+	p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp")
+	if err != nil {
+		return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()}
+	}
+	switch {
+	case p["LoadState"] == "not-found":
+		return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"}
+	case p["ActiveState"] == "activating" || p["ActiveState"] == "active":
+		return Health{Healthy: true, Why: "the weekly upgrade is running"}
+	case p["ActiveState"] != "failed" && p["Result"] == "success":
+		if at := p["ExecMainExitTimestamp"]; at != "" {
+			return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"}
+		}
+		return Health{Healthy: true, Why: "no weekly upgrade has run yet"}
+	}
+	why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"])
+	if at := p["ExecMainExitTimestamp"]; at != "" {
+		why += " at " + at
+	}
+	found := []string{}
+	if said := lastError(m.journal(WeeklyUnit, 40)); said != "" {
+		why += ": " + said
+		found = append(found, said)
+	}
+	return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found}
+}
+
+// lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that
+// could not be read says so in one line starting with "(", which is no error to name.
+func lastError(journal []string) string {
+	last := ""
+	for _, l := range journal {
+		l = strings.TrimSpace(l)
+		if l == "" || strings.HasPrefix(l, "(the journal could not be read") {
+			continue
+		}
+		last = l
+	}
+	for i := len(journal) - 1; i >= 0; i-- {
+		if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") {
+			return l
+		}
+	}
+	return last
+}

modules/pacman/files/mesh-pacman-weekly-upgrade.service

@@ -0,0 +1,12 @@
+# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push.
+[Unit]
+Description=Weekly full upgrade (the mesh's pacman module, ADR 0477)
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+# The upgrade pacman_upgrade runs (pacman -Syu --noconfirm), as a oneshot whose result is the module's health
+# (pacman_upgrade_health). The mesh restarts no machine for an upgrade: a new kernel waits for the next restart.
+Type=oneshot
+Nice=10
+ExecStart=/usr/bin/pacman -Syu --noconfirm

modules/pacman/files/mesh-pacman-weekly-upgrade.timer

@@ -0,0 +1,11 @@
+# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push.
+[Unit]
+Description=Weekly full upgrade, Saturday 04:00 local time on every node (the mesh's pacman module, ADR 0477)
+
+[Timer]
+# The same moment on every node: no random delay. A node that was off or asleep then upgrades at its next start.
+OnCalendar=Sat *-*-* 04:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target