239 lines
8.7 KiB
Go
239 lines
8.7 KiB
Go
package main
|
|
|
|
// The module's health, read from the bus (novox/hq ADR 0508 §11; issue 1579). The health check of the keeper's
|
|
// process reads this node's latest run, the week's round and any held kernel from the module's states, never a
|
|
// unit's age, so a laptop that slept through a run, or a node that was off, is judged by what the bus holds. It
|
|
// is unhealthy, naming why, when:
|
|
//
|
|
// - `installed.<node>` is marked not current (ADR 0507 §2);
|
|
// - no `upgrade-order` is set: no round starts, and every node says so (ADR 0508 §5);
|
|
// - the node's latest run was refused, failed, or its after-upgrade check did not pass, until a later run passes;
|
|
// - the latest run's transaction goes on past RunLimit: hung, holding pacman's lock;
|
|
// - the latest run held a kernel, naming the package, the DKMS module and its bound (ADR 0508 §3);
|
|
// - the week's round stands still: this node's turn came more than round-start-minutes ago and it has not
|
|
// started, or the node it waits for stalled or failed, named with since when (ADR 0508 §6).
|
|
//
|
|
// A state that cannot be read is unhealthy: failure is loud.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why.
|
|
type Health struct {
|
|
Healthy bool `json:"healthy"`
|
|
Why string `json:"why"`
|
|
Found []string `json:"found,omitempty"`
|
|
}
|
|
|
|
// RunLimit is how long a run's transaction may go on before it is unhealthy. A transaction has no timeout on
|
|
// purpose: killing pacman in the middle of one leaves a half-upgraded machine. One past the limit is a hung one
|
|
// (a stuck hook, a stalled download), which holds pacman's database lock so no later run can start; it is said,
|
|
// not killed.
|
|
const RunLimit = 2 * time.Hour
|
|
|
|
// healthTime is how the health writes a moment: the machine's local time.
|
|
const healthTime = "2006-01-02 15:04 MST"
|
|
|
|
// keeperConfig is the keeper's config as the tools bundle reads it, from the file the manifest names to the
|
|
// bundle too (EnvConfig); a test replaces it. A file that cannot be read is an empty config: no order.
|
|
var keeperConfig = func() KeeperConfig {
|
|
var c KeeperConfig
|
|
if raw, err := os.ReadFile(os.Getenv(EnvConfig)); err == nil {
|
|
_ = json.Unmarshal(raw, &c)
|
|
}
|
|
return c
|
|
}
|
|
|
|
// readOnly is a state the tools bundle reads, as the round's readers take it; it writes nothing.
|
|
type readOnly struct{ StateReader }
|
|
|
|
var errReadOnly = errors.New("the health writes nothing")
|
|
|
|
func (readOnly) Create(string, any) (uint64, error) { return 0, errReadOnly }
|
|
func (readOnly) Update(string, any, uint64) (uint64, error) { return 0, errReadOnly }
|
|
func (readOnly) Put(string, any) (uint64, error) { return 0, errReadOnly }
|
|
func (readOnly) Delete(string) error { return errReadOnly }
|
|
|
|
// UpgradeHealth is the health of this node's upgrades, from the module's states.
|
|
func (m *Machine) UpgradeHealth() Health {
|
|
if h, said := m.InstalledHealth(); said {
|
|
return h
|
|
}
|
|
if m.States == nil || m.Node == "" {
|
|
return Health{Why: "the tools bundle was started without its node (MESH_NODE) or the bus: the upgrades cannot be read"}
|
|
}
|
|
cfg := keeperConfig()
|
|
if len(cfg.UpgradeOrder) == 0 {
|
|
return Health{Why: "no upgrade-order is set: no node upgrades, since no round starts until the mesh-level key " +
|
|
"upgrade-order names the nodes in the order they upgrade (ADR 0508 §5)"}
|
|
}
|
|
runs := readOnly{m.States(StateRuns)}
|
|
latest, err := latestOwnRun(runs, m.Node)
|
|
if err != nil {
|
|
return Health{Why: "the node's runs could not be read: " + err.Error()}
|
|
}
|
|
if latest != nil {
|
|
if h, said := m.runHealth(*latest); said {
|
|
return h
|
|
}
|
|
}
|
|
if h, said := m.roundHealth(readOnly{m.States(StateRounds)}, runs, cfg.RoundStartMinutes); said {
|
|
return h
|
|
}
|
|
if latest == nil {
|
|
return Health{Healthy: true, Why: "no upgrade has run on this node through the mesh yet"}
|
|
}
|
|
return Health{Healthy: true, Why: fmt.Sprintf("the last run, %s, passed its after-upgrade check", latest.Run)}
|
|
}
|
|
|
|
// latestOwnRun is the node's newest run that the mesh ran, or nil; runs read from pacman's log are history.
|
|
func latestOwnRun(runs Bucket, node string) (*Run, error) {
|
|
keys, err := runs.Keys()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Sort(sort.Reverse(sort.StringSlice(keys)))
|
|
for _, k := range keys {
|
|
if !strings.HasPrefix(k, node+".") {
|
|
continue
|
|
}
|
|
e, err := runs.Get(k)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if e == nil {
|
|
continue
|
|
}
|
|
var r Run
|
|
if err := json.Unmarshal(e.Value, &r); err != nil {
|
|
return nil, fmt.Errorf("run %s: %w", k, err)
|
|
}
|
|
if r.From == FromLog || len(r.Steps) == 0 {
|
|
continue
|
|
}
|
|
return &r, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// runHealth judges the node's latest run; said is false when the run has nothing to say against the node.
|
|
func (m *Machine) runHealth(r Run) (Health, bool) {
|
|
var held []HeldKernel
|
|
var started *Step
|
|
for i, s := range r.Steps {
|
|
switch s.Step {
|
|
case StepCheckedBefore:
|
|
held = s.Held
|
|
case StepStarted:
|
|
started = &r.Steps[i]
|
|
}
|
|
}
|
|
what := r.Kind + " run " + r.Run
|
|
last := r.Steps[len(r.Steps)-1]
|
|
switch lastStep(r) {
|
|
case StepRefused:
|
|
return Health{Why: fmt.Sprintf("the %s did not start (%s): %s", what, last.At.Local().Format(healthTime), last.Why)}, true
|
|
case StepCheckedAfter:
|
|
if last.Check == nil || !last.Check.Passed {
|
|
why := fmt.Sprintf("the %s did not pass its after-upgrade check (%s)", what, last.At.Local().Format(healthTime))
|
|
for _, s := range r.Steps {
|
|
if s.Step == StepFailed {
|
|
why += ": its transaction failed: " + s.Error
|
|
}
|
|
}
|
|
if last.Check != nil {
|
|
for _, u := range last.Check.Unbuilt {
|
|
why += fmt.Sprintf("; %s %s is not built for kernel %s", u.Module, u.Version, u.Kernel)
|
|
}
|
|
}
|
|
return Health{Why: why + "; pacman_upgrades with the node and the run says it whole", Found: []string{r.Run}}, true
|
|
}
|
|
case StepStarted:
|
|
if started != nil && m.Now().Sub(started.At) > RunLimit {
|
|
return Health{Why: fmt.Sprintf("the %s's transaction (%s) has gone on since %s, longer than %s: it is hung, and holds pacman's database lock so no later run can start",
|
|
what, started.Unit, started.At.Local().Format(healthTime), RunLimit)}, true
|
|
}
|
|
}
|
|
if len(held) > 0 {
|
|
var said []string
|
|
for _, h := range held {
|
|
said = append(said, fmt.Sprintf("%s %s (the DKMS module %s allows %s)", h.Package, h.Version, h.Module, h.Bound))
|
|
}
|
|
return Health{Why: "the " + what + " held back " + strings.Join(said, ", ") + ": the kernel waits until its DKMS module follows (ADR 0508 §3)"}, true
|
|
}
|
|
return Health{}, false
|
|
}
|
|
|
|
// roundHealth judges this node's place in the newest round; said is false when the round has nothing to say
|
|
// against it.
|
|
func (m *Machine) roundHealth(rounds, runs Bucket, startMinutes int) (Health, bool) {
|
|
id, err := latestRound(rounds)
|
|
if err != nil {
|
|
return Health{Why: "the rounds could not be read: " + err.Error()}, true
|
|
}
|
|
if id == "" {
|
|
return Health{}, false
|
|
}
|
|
v, err := readRound(rounds, runs, id)
|
|
if err != nil {
|
|
return Health{Why: "round " + id + " could not be read: " + err.Error()}, true
|
|
}
|
|
if v == nil || !contains(v.Round.Order, m.Node) {
|
|
return Health{}, false
|
|
}
|
|
if startMinutes < 1 {
|
|
startMinutes = defaultRoundStartMinutes
|
|
}
|
|
byNode := map[string]NodeInRound{}
|
|
for _, s := range RoundState(v, m.Now(), startMinutes) {
|
|
byNode[s.Node] = s
|
|
}
|
|
since := func(s NodeInRound) string {
|
|
if s.Since == nil {
|
|
return "an unknown time"
|
|
}
|
|
return s.Since.Local().Format(healthTime)
|
|
}
|
|
me := byNode[m.Node]
|
|
switch me.State {
|
|
case InRoundStalled:
|
|
return Health{Why: fmt.Sprintf("round %s: this node's turn came at %s and it has not started its run within %d minutes", id, since(me), startMinutes)}, true
|
|
case InRoundWaiting:
|
|
if w, ok := byNode[me.WaitsFor]; ok {
|
|
switch w.State {
|
|
case InRoundStalled:
|
|
return Health{Why: fmt.Sprintf("round %s stands still: this node waits for %s, whose turn came at %s and which has not started (since %s)", id, w.Node, since(w), since(w))}, true
|
|
case InRoundFailed:
|
|
return Health{Why: fmt.Sprintf("round %s stopped at %s, whose run or check failed at %s: this node waits on it since then; only the operator passes it (pacman_round with pass and a reason)", id, w.Node, since(w))}, true
|
|
}
|
|
}
|
|
}
|
|
return Health{}, false
|
|
}
|
|
|
|
// lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that
|
|
// could not be read says so in one line starting with "(", which is no error to name.
|
|
func lastError(journal []string) string {
|
|
last := ""
|
|
for _, l := range journal {
|
|
l = strings.TrimSpace(l)
|
|
if l == "" || strings.HasPrefix(l, "(the journal could not be read") {
|
|
continue
|
|
}
|
|
last = l
|
|
}
|
|
for i := len(journal) - 1; i >= 0; i-- {
|
|
if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") {
|
|
return l
|
|
}
|
|
}
|
|
return last
|
|
}
|