diff --git a/modules/pacman/cmd/pacman-tools/upgrade_health.go b/modules/pacman/cmd/pacman-tools/upgrade_health.go new file mode 100644 index 00000000..2b817bcf --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/upgrade_health.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "strings" +) + +// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node +// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager +// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of +// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and, +// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an +// upgrade passes. + +// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why. +type Health struct { + Healthy bool `json:"healthy"` + Why string `json:"why"` + Found []string `json:"found,omitempty"` +} + +// WeeklyUnit is the service the weekly timer starts. +const WeeklyUnit = "mesh-pacman-weekly-upgrade.service" + +// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit +// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A +// unit that cannot be read is unhealthy: failure is loud. +func (m *Machine) UpgradeHealth() Health { + p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp") + if err != nil { + return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()} + } + switch { + case p["LoadState"] == "not-found": + return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"} + case p["ActiveState"] == "activating" || p["ActiveState"] == "active": + return Health{Healthy: true, Why: "the weekly upgrade is running"} + case p["ActiveState"] != "failed" && p["Result"] == "success": + if at := p["ExecMainExitTimestamp"]; at != "" { + return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"} + } + return Health{Healthy: true, Why: "no weekly upgrade has run yet"} + } + why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"]) + if at := p["ExecMainExitTimestamp"]; at != "" { + why += " at " + at + } + found := []string{} + if said := lastError(m.journal(WeeklyUnit, 40)); said != "" { + why += ": " + said + found = append(found, said) + } + return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found} +} + +// lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that +// could not be read says so in one line starting with "(", which is no error to name. +func lastError(journal []string) string { + last := "" + for _, l := range journal { + l = strings.TrimSpace(l) + if l == "" || strings.HasPrefix(l, "(the journal could not be read") { + continue + } + last = l + } + for i := len(journal) - 1; i >= 0; i-- { + if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") { + return l + } + } + return last +} diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.service b/modules/pacman/files/mesh-pacman-weekly-upgrade.service new file mode 100644 index 00000000..69741dd7 --- /dev/null +++ b/modules/pacman/files/mesh-pacman-weekly-upgrade.service @@ -0,0 +1,12 @@ +# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. +[Unit] +Description=Weekly full upgrade (the mesh's pacman module, ADR 0477) +Wants=network-online.target +After=network-online.target + +[Service] +# The upgrade pacman_upgrade runs (pacman -Syu --noconfirm), as a oneshot whose result is the module's health +# (pacman_upgrade_health). The mesh restarts no machine for an upgrade: a new kernel waits for the next restart. +Type=oneshot +Nice=10 +ExecStart=/usr/bin/pacman -Syu --noconfirm diff --git a/modules/pacman/files/mesh-pacman-weekly-upgrade.timer b/modules/pacman/files/mesh-pacman-weekly-upgrade.timer new file mode 100644 index 00000000..a8a9b4d9 --- /dev/null +++ b/modules/pacman/files/mesh-pacman-weekly-upgrade.timer @@ -0,0 +1,11 @@ +# Generated by the mesh. Do not edit — module pacman writes this file and replaces it at every push. +[Unit] +Description=Weekly full upgrade, Saturday 04:00 local time on every node (the mesh's pacman module, ADR 0477) + +[Timer] +# The same moment on every node: no random delay. A node that was off or asleep then upgrades at its next start. +OnCalendar=Sat *-*-* 04:00:00 +Persistent=true + +[Install] +WantedBy=timers.target