package main // The module's health, read from the bus (novox/hq ADR 0508 §11; issue 1579). The health check of the keeper's // process reads this node's latest run, the week's round and any held kernel from the module's states, never a // unit's age, so a laptop that slept through a run, or a node that was off, is judged by what the bus holds. It // is unhealthy, naming why, when: // // - `installed.` is marked not current (ADR 0507 §2); // - no `upgrade-order` is set: no round starts, and every node says so (ADR 0508 §5); // - the node's latest run was refused, failed, or its after-upgrade check did not pass, until a later run passes; // - the latest run's transaction goes on past RunLimit: hung, holding pacman's lock; // - the latest run held a kernel, naming the package, the DKMS module and its bound (ADR 0508 §3); // - the week's round stands still: this node's turn came more than round-start-minutes ago and it has not // started, or the node it waits for stalled or failed, named with since when (ADR 0508 §6). // // A state that cannot be read is unhealthy: failure is loud. import ( "encoding/json" "errors" "fmt" "os" "sort" "strings" "time" ) // Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why. type Health struct { Healthy bool `json:"healthy"` Why string `json:"why"` Found []string `json:"found,omitempty"` } // RunLimit is how long a run's transaction may go on before it is unhealthy. A transaction has no timeout on // purpose: killing pacman in the middle of one leaves a half-upgraded machine. One past the limit is a hung one // (a stuck hook, a stalled download), which holds pacman's database lock so no later run can start; it is said, // not killed. const RunLimit = 2 * time.Hour // healthTime is how the health writes a moment: the machine's local time. const healthTime = "2006-01-02 15:04 MST" // keeperConfig is the keeper's config as the tools bundle reads it, from the file the manifest names to the // bundle too (EnvConfig); a test replaces it. A file that cannot be read is an empty config: no order. var keeperConfig = func() KeeperConfig { var c KeeperConfig if raw, err := os.ReadFile(os.Getenv(EnvConfig)); err == nil { _ = json.Unmarshal(raw, &c) } return c } // readOnly is a state the tools bundle reads, as the round's readers take it; it writes nothing. type readOnly struct{ StateReader } var errReadOnly = errors.New("the health writes nothing") func (readOnly) Create(string, any) (uint64, error) { return 0, errReadOnly } func (readOnly) Update(string, any, uint64) (uint64, error) { return 0, errReadOnly } func (readOnly) Put(string, any) (uint64, error) { return 0, errReadOnly } func (readOnly) Delete(string) error { return errReadOnly } // UpgradeHealth is the health of this node's upgrades, from the module's states. func (m *Machine) UpgradeHealth() Health { if h, said := m.InstalledHealth(); said { return h } if m.States == nil || m.Node == "" { return Health{Why: "the tools bundle was started without its node (MESH_NODE) or the bus: the upgrades cannot be read"} } cfg := keeperConfig() if len(cfg.UpgradeOrder) == 0 { return Health{Why: "no upgrade-order is set: no node upgrades, since no round starts until the mesh-level key " + "upgrade-order names the nodes in the order they upgrade (ADR 0508 §5)"} } runs := readOnly{m.States(StateRuns)} latest, err := latestOwnRun(runs, m.Node) if err != nil { return Health{Why: "the node's runs could not be read: " + err.Error()} } if latest != nil { if h, said := m.runHealth(*latest); said { return h } } if h, said := m.roundHealth(readOnly{m.States(StateRounds)}, runs, cfg.RoundStartMinutes); said { return h } if latest == nil { return Health{Healthy: true, Why: "no upgrade has run on this node through the mesh yet"} } return Health{Healthy: true, Why: fmt.Sprintf("the last run, %s, passed its after-upgrade check", latest.Run)} } // latestOwnRun is the node's newest run that the mesh ran, or nil; runs read from pacman's log are history. func latestOwnRun(runs Bucket, node string) (*Run, error) { keys, err := runs.Keys() if err != nil { return nil, err } sort.Sort(sort.Reverse(sort.StringSlice(keys))) for _, k := range keys { if !strings.HasPrefix(k, node+".") { continue } e, err := runs.Get(k) if err != nil { return nil, err } if e == nil { continue } var r Run if err := json.Unmarshal(e.Value, &r); err != nil { return nil, fmt.Errorf("run %s: %w", k, err) } if r.From == FromLog || len(r.Steps) == 0 { continue } return &r, nil } return nil, nil } // runHealth judges the node's latest run; said is false when the run has nothing to say against the node. func (m *Machine) runHealth(r Run) (Health, bool) { var held []HeldKernel var started *Step for i, s := range r.Steps { switch s.Step { case StepCheckedBefore: held = s.Held case StepStarted: started = &r.Steps[i] } } what := r.Kind + " run " + r.Run last := r.Steps[len(r.Steps)-1] switch lastStep(r) { case StepRefused: return Health{Why: fmt.Sprintf("the %s did not start (%s): %s", what, last.At.Local().Format(healthTime), last.Why)}, true case StepCheckedAfter: if last.Check == nil || !last.Check.Passed { why := fmt.Sprintf("the %s did not pass its after-upgrade check (%s)", what, last.At.Local().Format(healthTime)) for _, s := range r.Steps { if s.Step == StepFailed { why += ": its transaction failed: " + s.Error } } if last.Check != nil { for _, u := range last.Check.Unbuilt { why += fmt.Sprintf("; %s %s is not built for kernel %s", u.Module, u.Version, u.Kernel) } } return Health{Why: why + "; pacman_upgrades with the node and the run says it whole", Found: []string{r.Run}}, true } case StepStarted: if started != nil && m.Now().Sub(started.At) > RunLimit { return Health{Why: fmt.Sprintf("the %s's transaction (%s) has gone on since %s, longer than %s: it is hung, and holds pacman's database lock so no later run can start", what, started.Unit, started.At.Local().Format(healthTime), RunLimit)}, true } } if len(held) > 0 { var said []string for _, h := range held { said = append(said, fmt.Sprintf("%s %s (the DKMS module %s allows %s)", h.Package, h.Version, h.Module, h.Bound)) } return Health{Why: "the " + what + " held back " + strings.Join(said, ", ") + ": the kernel waits until its DKMS module follows (ADR 0508 §3)"}, true } return Health{}, false } // roundHealth judges this node's place in the newest round; said is false when the round has nothing to say // against it. func (m *Machine) roundHealth(rounds, runs Bucket, startMinutes int) (Health, bool) { id, err := latestRound(rounds) if err != nil { return Health{Why: "the rounds could not be read: " + err.Error()}, true } if id == "" { return Health{}, false } v, err := readRound(rounds, runs, id) if err != nil { return Health{Why: "round " + id + " could not be read: " + err.Error()}, true } if v == nil || !contains(v.Round.Order, m.Node) { return Health{}, false } if startMinutes < 1 { startMinutes = defaultRoundStartMinutes } byNode := map[string]NodeInRound{} for _, s := range RoundState(v, m.Now(), startMinutes) { byNode[s.Node] = s } since := func(s NodeInRound) string { if s.Since == nil { return "an unknown time" } return s.Since.Local().Format(healthTime) } me := byNode[m.Node] switch me.State { case InRoundStalled: return Health{Why: fmt.Sprintf("round %s: this node's turn came at %s and it has not started its run within %d minutes", id, since(me), startMinutes)}, true case InRoundWaiting: if w, ok := byNode[me.WaitsFor]; ok { switch w.State { case InRoundStalled: return Health{Why: fmt.Sprintf("round %s stands still: this node waits for %s, whose turn came at %s and which has not started (since %s)", id, w.Node, since(w), since(w))}, true case InRoundFailed: return Health{Why: fmt.Sprintf("round %s stopped at %s, whose run or check failed at %s: this node waits on it since then; only the operator passes it (pacman_round with pass and a reason)", id, w.Node, since(w))}, true } } } return Health{}, false } // lastError is the last line pacman marked as an error, else the last line the journal holds; a journal that // could not be read says so in one line starting with "(", which is no error to name. func lastError(journal []string) string { last := "" for _, l := range journal { l = strings.TrimSpace(l) if l == "" || strings.HasPrefix(l, "(the journal could not be read") { continue } last = l } for i := len(journal) - 1; i >= 0; i-- { if l := strings.TrimSpace(journal[i]); strings.HasPrefix(l, "error:") { return l } } return last }