package main import ( "fmt" "strings" ) // The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node // each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager // (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of // the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and, // here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an // upgrade passes. // Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why. type Health struct { Healthy bool `json:"healthy"` Why string `json:"why"` Found []string `json:"found,omitempty"` } // WeeklyUnit is the service the weekly timer starts. const WeeklyUnit = "mesh-pacman-weekly-upgrade.service" // UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit // and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A // unit that cannot be read is unhealthy: failure is loud. func (m *Machine) UpgradeHealth() Health { p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp") if err != nil { return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()} } switch { case p["LoadState"] == "not-found": return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"} case p["ActiveState"] == "activating" || p["ActiveState"] == "active": return Health{Healthy: true, Why: "the weekly upgrade is running"} case p["ActiveState"] != "failed" && p["Result"] == "success": if at := p["ExecMainExitTimestamp"]; at != "" { return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"} } return Health{Healthy: true, Why: "no weekly upgrade has run yet"} } why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"]) if at := p["ExecMainExitTimestamp"]; at != "" { why += " at " + at } found := []string{} if said := lastError(m.journal(WeeklyUnit, 40)); said != "" { why += ": " + said found = append(found, said) } return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found} } // lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that // could not be read says so in one line starting with "(", which is no error to name. func lastError(journal []string) string { last := "" for _, l := range journal { l = strings.TrimSpace(l) if l == "" || strings.HasPrefix(l, "(the journal could not be read") { continue } last = l } for i := len(journal) - 1; i >= 0; i-- { if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") { return l } } return last }