mesh-catalog/modules/pacman/cmd/pacman-tools/upgrade_health.go

package main

import (
	"fmt"
	"strings"
	"time"
)

// The weekly upgrade's outcome (issue 1574, ADR 0477 clause 5). The mesh runs a full upgrade on every node
// each Saturday at 04:00 local time, from a timer and a oneshot service of the service manager
// (files/mesh-pacman-weekly-upgrade.*). Its outcome is the module's health: this tool is the health check of
// the timer's resource, so a failed upgrade raises the module's condition on the node, naming the node and,
// here, the error. A unit that failed stays failed until the next run, so the health stays unhealthy until an
// upgrade passes.

// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why.
type Health struct {
	Healthy bool     `json:"healthy"`
	Why     string   `json:"why"`
	Found   []string `json:"found,omitempty"`
}

// WeeklyUnit is the service the weekly timer starts.
const WeeklyUnit = "mesh-pacman-weekly-upgrade.service"

// RunLimit is how long a weekly upgrade may go on before it is unhealthy. The service has no start timeout on
// purpose: killing pacman in the middle of a transaction leaves a half-upgraded machine. A run past the limit
// is a hung one (a stuck hook, a stalled transaction), which holds pacman's database lock so that no later run
// can start; it is said, not killed.
const RunLimit = 2 * time.Hour

// systemdTime is how systemctl shows a timestamp, in the machine's local time.
const systemdTime = "Mon 2006-01-02 15:04:05 MST"

// UpgradeHealth judges the last weekly upgrade from its service: unhealthy when it failed, naming the exit
// and the last error the journal holds; healthy while it runs, after it passed, and before it ever ran. A
// unit that cannot be read is unhealthy: failure is loud.
func (m *Machine) UpgradeHealth() Health {
	p, err := m.unitProps(WeeklyUnit, "LoadState", "ActiveState", "Result", "ExecMainStatus", "ExecMainExitTimestamp", "ExecMainStartTimestamp")
	if err != nil {
		return Health{Why: "the weekly upgrade's service could not be read: " + err.Error()}
	}
	switch {
	case p["LoadState"] == "not-found":
		return Health{Healthy: true, Why: "the weekly upgrade's service is not placed yet; no upgrade has run"}
	case p["ActiveState"] == "activating" || p["ActiveState"] == "active":
		return m.runningHealth(p["ExecMainStartTimestamp"])
	case p["ActiveState"] != "failed" && p["Result"] == "success":
		if at := p["ExecMainExitTimestamp"]; at != "" {
			return Health{Healthy: true, Why: "the last weekly upgrade passed (" + at + ")"}
		}
		return Health{Healthy: true, Why: "no weekly upgrade has run yet"}
	}
	why := fmt.Sprintf("the weekly upgrade failed (%s, status %s)", p["Result"], p["ExecMainStatus"])
	if at := p["ExecMainExitTimestamp"]; at != "" {
		why += " at " + at
	}
	found := []string{}
	if said := lastError(m.journal(WeeklyUnit, 40)); said != "" {
		why += ": " + said
		found = append(found, said)
	}
	return Health{Why: why + "; the next upgrade is the next Saturday 04:00, or a hand pacman_upgrade", Found: found}
}

// runningHealth judges a run in progress: healthy within RunLimit of its start, unhealthy past it, naming the
// start and the last line the journal holds.
func (m *Machine) runningHealth(started string) Health {
	at, err := time.ParseInLocation(systemdTime, started, time.Local)
	if err != nil {
		return Health{Why: fmt.Sprintf("the weekly upgrade is running and when it started could not be read (%q)", started)}
	}
	if m.Now().Sub(at) <= RunLimit {
		return Health{Healthy: true, Why: "the weekly upgrade is running, since " + started}
	}
	why := fmt.Sprintf("the weekly upgrade has been running since %s, longer than %s: it is hung, and holds pacman's database lock so no later upgrade can start", started, RunLimit)
	found := []string{}
	if ls := m.journal(WeeklyUnit, 1); len(ls) > 0 && !strings.HasPrefix(ls[0], "(the journal could not be read") {
		why += "; its last line: " + strings.TrimSpace(ls[0])
		found = append(found, strings.TrimSpace(ls[0]))
	}
	return Health{Why: why, Found: found}
}

// lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that
// could not be read says so in one line starting with "(", which is no error to name.
func lastError(journal []string) string {
	last := ""
	for _, l := range journal {
		l = strings.TrimSpace(l)
		if l == "" || strings.HasPrefix(l, "(the journal could not be read") {
			continue
		}
		last = l
	}
	for i := len(journal) - 1; i >= 0; i-- {
		if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") {
			return l
		}
	}
	return last
}