74 lines
3.0 KiB
Go
74 lines
3.0 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node
|
|
// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager
|
|
// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of
|
|
// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and,
|
|
// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an
|
|
// upgrade passes.
|
|
|
|
// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why.
|
|
type Health struct {
|
|
Healthy bool `json:"healthy"`
|
|
Why string `json:"why"`
|
|
Found []string `json:"found,omitempty"`
|
|
}
|
|
|
|
// WeeklyUnit is the service the weekly timer starts.
|
|
const WeeklyUnit = "mesh-pacman-weekly-upgrade.service"
|
|
|
|
// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit
|
|
// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A
|
|
// unit that cannot be read is unhealthy: failure is loud.
|
|
func (m *Machine) UpgradeHealth() Health {
|
|
p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp")
|
|
if err != nil {
|
|
return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()}
|
|
}
|
|
switch {
|
|
case p["LoadState"] == "not-found":
|
|
return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"}
|
|
case p["ActiveState"] == "activating" || p["ActiveState"] == "active":
|
|
return Health{Healthy: true, Why: "the weekly upgrade is running"}
|
|
case p["ActiveState"] != "failed" && p["Result"] == "success":
|
|
if at := p["ExecMainExitTimestamp"]; at != "" {
|
|
return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"}
|
|
}
|
|
return Health{Healthy: true, Why: "no weekly upgrade has run yet"}
|
|
}
|
|
why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"])
|
|
if at := p["ExecMainExitTimestamp"]; at != "" {
|
|
why += " at " + at
|
|
}
|
|
found := []string{}
|
|
if said := lastError(m.journal(WeeklyUnit, 40)); said != "" {
|
|
why += ": " + said
|
|
found = append(found, said)
|
|
}
|
|
return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found}
|
|
}
|
|
|
|
// lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that
|
|
// could not be read says so in one line starting with "(", which is no error to name.
|
|
func lastError(journal []string) string {
|
|
last := ""
|
|
for _, l := range journal {
|
|
l = strings.TrimSpace(l)
|
|
if l == "" || strings.HasPrefix(l, "(the journal could not be read") {
|
|
continue
|
|
}
|
|
last = l
|
|
}
|
|
for i := len(journal) - 1; i >= 0; i-- {
|
|
if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") {
|
|
return l
|
|
}
|
|
}
|
|
return last
|
|
}
|