565 lines
24 KiB
Go
565 lines
24 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
|
|
// Phase A).
|
|
//
|
|
// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
|
|
// machine states, in every report and as an event between reports, the state of every long-running
|
|
// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
|
|
// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
|
|
// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
|
|
// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
|
|
// the stated health: a judging passes a module only when every long-running resource of it on that
|
|
// machine is stated healthy, so a resource still starting is not yet a pass.
|
|
//
|
|
// **An engine older than the judging states nothing**, and its machine's health is not known: never
|
|
// healthy, never a reason to raise anything, and the gate judges it as it did before.
|
|
|
|
// The condition a module's health raises.
|
|
const (
|
|
kindModuleUnhealthy = "module-unhealthy"
|
|
// sourceHealth is what raised it: the machine's own statement.
|
|
sourceHealth = "health"
|
|
// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
|
|
moduleUnhealthyUrgentAfter = 4 * time.Hour
|
|
// moduleUnhealthyAfter is how many statements in a row raise it.
|
|
moduleUnhealthyAfter = 2
|
|
)
|
|
|
|
// healthRefused counts the statements refused as older than the one kept, for the log and a test.
|
|
var healthRefused atomic.Int64
|
|
|
|
// moduleHealth keeps what the machines state, for the link (link.Healths).
|
|
type moduleHealth struct {
|
|
inv *inventory.Inventory
|
|
keeper func() *conditions.Keeper
|
|
}
|
|
|
|
func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
|
|
return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
|
|
}
|
|
|
|
// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
|
|
// older statement than the one kept is refused, by when the engine looked.
|
|
func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
|
|
now time.Time) error {
|
|
if h.Contract == 0 {
|
|
return nil
|
|
}
|
|
prev, had, err := inv.HealthOf(ctx, node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if had && h.At.Before(prev.SaidAt) {
|
|
healthRefused.Add(1)
|
|
return nil
|
|
}
|
|
unhealthy := map[string][]inventory.ResourceHealth{}
|
|
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
|
for _, r := range h.Resources {
|
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
|
Check: r.Check, Needs: r.Needs, Account: r.Account}
|
|
resources = append(resources, kept)
|
|
if r.State == link.StateUnhealthy && r.Module != "" {
|
|
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
|
}
|
|
}
|
|
streaks := map[string]int{}
|
|
for module := range unhealthy {
|
|
streaks[module] = prev.Streaks[module] + 1
|
|
}
|
|
var network *inventory.NetworkHealth
|
|
if h.Network != nil {
|
|
network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}}
|
|
for _, p := range h.Network.Parts {
|
|
network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
|
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak})
|
|
}
|
|
}
|
|
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
|
HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)})
|
|
if err != nil || !stored {
|
|
if err == nil {
|
|
healthRefused.Add(1)
|
|
}
|
|
return err
|
|
}
|
|
if k == nil {
|
|
return nil
|
|
}
|
|
err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
|
// And the machine's own failed units (issue 315): what no module places, one finding for the machine.
|
|
if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil {
|
|
if err == nil {
|
|
err = uerr
|
|
} else {
|
|
err = fmt.Errorf("%w; %v", err, uerr)
|
|
}
|
|
}
|
|
// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
|
|
// machine can hold another's finding, or release it.
|
|
if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
|
|
if err == nil {
|
|
return nerr
|
|
}
|
|
return fmt.Errorf("%w; %v", err, nerr)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
|
// resource of it is unhealthy — or on the first while it is already open — and clears every one this
|
|
// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48
|
|
// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone
|
|
// waits on it.
|
|
func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string,
|
|
unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
|
|
open, err := k.Open(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
standing := map[string]conditions.Condition{}
|
|
for _, c := range open {
|
|
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded) && c.Subject.Machine == node {
|
|
standing[c.Key] = c
|
|
}
|
|
}
|
|
var hold *holding
|
|
if inv != nil {
|
|
if hold, err = readHolding(ctx, inv, open); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
var problems []string
|
|
modules := make([]string, 0, len(unhealthy))
|
|
for m := range unhealthy {
|
|
modules = append(modules, m)
|
|
}
|
|
sort.Strings(modules)
|
|
seen := map[string]bool{}
|
|
// became is, per module, the kind of condition this statement says of it: what a standing one of the
|
|
// other kind turned into.
|
|
became := map[string]string{}
|
|
heldOn := map[string]string{}
|
|
providers := map[catalogue.Chosen]bool{}
|
|
for _, m := range modules {
|
|
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
|
|
// operator, never urgent, cleared on the first statement that no longer says it.
|
|
if said, waits := personWait(m, node, unhealthy[m]); waits {
|
|
o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
|
|
// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
|
|
// consumers are held under it, and its own condition is where they are said.
|
|
if hold != nil {
|
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
}
|
|
seen[o.Key()] = true
|
|
became[m] = kindReloginNeeded
|
|
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
|
if hold != nil {
|
|
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
|
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
|
heldOn[o.Key()] = p.Module + " on " + p.Node
|
|
providers[p] = true
|
|
continue
|
|
}
|
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
}
|
|
seen[o.Key()] = true
|
|
became[m] = kindModuleUnhealthy
|
|
c, isOpen := standing[o.Key()]
|
|
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
|
}
|
|
if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
|
|
o.Severity = conditions.Urgent
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
for key, c := range standing {
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
|
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
|
|
if c.Kind == kindReloginNeeded {
|
|
why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
|
|
}
|
|
if on, held := heldOn[key]; held {
|
|
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
|
}
|
|
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
|
// line says what it became.
|
|
resolved := ""
|
|
switch {
|
|
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
|
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
|
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
|
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
|
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
|
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
|
}
|
|
if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
// And each provider a consumer here now waits on, when its own condition is open: said again with who
|
|
// waits on it, so the wait is listed at the provider whichever machine's statement arrived first.
|
|
for p := range providers {
|
|
if err := sayWaiters(ctx, k, hold, p, now); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
|
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
|
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
|
var raisedAt *conditions.Condition
|
|
for i, c := range hold.open {
|
|
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
|
raisedAt = &hold.open[i]
|
|
}
|
|
}
|
|
if raisedAt == nil {
|
|
return nil
|
|
}
|
|
var rs []inventory.ResourceHealth
|
|
for _, r := range hold.healths[p.Node].Resources {
|
|
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
|
rs = append(rs, r)
|
|
}
|
|
}
|
|
if len(rs) == 0 {
|
|
return nil
|
|
}
|
|
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
|
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
|
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
|
}
|
|
if o.Key() != raisedAt.Key {
|
|
return nil // its own statement says it next
|
|
}
|
|
sayWaitingOn(&o, hold.waitersOn(p))
|
|
_, err := k.Observe(ctx, o)
|
|
return err
|
|
}
|
|
|
|
// reloginKey is a module's relogin-needed condition on a machine.
|
|
func reloginKey(module, node string) string {
|
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
|
}
|
|
|
|
// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
|
|
func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
|
|
if inv == nil {
|
|
return ""
|
|
}
|
|
n, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return n.Account
|
|
}
|
|
|
|
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
|
|
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
|
|
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
|
|
// person in again, and a restart of the module's service would start it in the same session. operator is
|
|
// the machine's operator account, when known: the words say "your account" only for it.
|
|
func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
o := moduleUnhealthyObservation(module, node, rs)
|
|
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
|
|
accounts := waitingAccounts(module, rs)
|
|
yours := operator != "" && len(accounts) > 0
|
|
for _, a := range accounts {
|
|
yours = yours && a == operator
|
|
}
|
|
w := reloginWords(module, node, yours)
|
|
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
|
|
return o
|
|
}
|
|
|
|
// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
|
|
// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
|
|
// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
|
|
func reloginWords(module, node string, yours bool) words {
|
|
if yours {
|
|
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
|
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
|
|
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
|
|
"cannot run until you log in again. Its update is in place and nothing was undone.",
|
|
conditions.Capital(module), module),
|
|
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
|
|
}
|
|
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
|
Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
|
|
Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
|
|
"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
|
|
conditions.Capital(module), node, module),
|
|
Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
|
|
}
|
|
|
|
// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
|
|
func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, r := range rs {
|
|
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
|
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
|
|
seen[accountOf(r)] = true
|
|
out = append(out, accountOf(r))
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
|
// waits on it, whether it is not working or waits for a new login.
|
|
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
|
if len(waiters) == 0 {
|
|
return
|
|
}
|
|
o.Severity = conditions.Urgent
|
|
o.Said += "; " + waitingWords(waiters)
|
|
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
|
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
|
}
|
|
|
|
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
|
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
|
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
var words, said, plain []string
|
|
needs, actions := moduleNeeds(node, rs)
|
|
for _, r := range rs {
|
|
plain = append(plain, resourcePlainWords(r))
|
|
if r.Kind == link.KindUnit {
|
|
// A failed unit is named by the unit, which is what a person looks for (issue 315); the
|
|
// resource that places it — a package, a file — is evidence.
|
|
words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason))
|
|
said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource,
|
|
orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
|
continue
|
|
}
|
|
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
|
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
|
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
|
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
|
}
|
|
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
|
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
|
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
|
Said: strings.Join(said, "; "),
|
|
Headline: fmt.Sprintf("%s not working on %s", module, node),
|
|
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
|
namesWords(plain, 3)),
|
|
Needs: needs,
|
|
Actions: actions,
|
|
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
|
}
|
|
|
|
// reasonWords is why a resource is unhealthy, as a person reads it.
|
|
func reasonWords(r inventory.ResourceHealth) string {
|
|
switch r.Reason {
|
|
case "restarting":
|
|
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
|
case "down":
|
|
return "is not running"
|
|
case "":
|
|
return "is unhealthy"
|
|
}
|
|
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
|
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
|
if r.Check != "" {
|
|
return "fails its " + r.Check + " check"
|
|
}
|
|
return "is unhealthy: " + r.Reason
|
|
}
|
|
|
|
func orNotSaid(s string) string {
|
|
if s == "" {
|
|
return "no reason said"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
|
|
// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
|
|
const kindReloginNeeded = "relogin-needed"
|
|
|
|
// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
|
|
// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
|
|
// excused:
|
|
//
|
|
// - at least one of the module's resources is its account (kind account), unhealthy with a reason that
|
|
// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
|
|
// running began before;
|
|
// - every other unhealthy resource of the module runs in the own service manager of one of those very
|
|
// accounts (Account names it): the manager that began before the group and does not hold it.
|
|
//
|
|
// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
|
|
// another account's manager, a resource whose engine does not say whose manager it is in, an account
|
|
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
|
|
// resource still starting is not unhealthy and does not make a wait either. Pure.
|
|
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
|
|
accounts := waitingAccounts(module, rs)
|
|
if len(accounts) == 0 {
|
|
return "", false
|
|
}
|
|
waiting := map[string]bool{}
|
|
for _, a := range accounts {
|
|
waiting[a] = true
|
|
}
|
|
var units []string
|
|
for _, r := range rs {
|
|
if r.Module != module || r.State != link.StateUnhealthy {
|
|
continue
|
|
}
|
|
switch {
|
|
case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
|
|
case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
|
|
units = append(units, r.Target)
|
|
default:
|
|
return "", false
|
|
}
|
|
}
|
|
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
|
|
"running session began before it was; log out of every session and in again, or reboot", machine, module,
|
|
strings.Join(accounts, ", "))
|
|
if len(units) > 0 {
|
|
sort.Strings(units)
|
|
said += fmt.Sprintf(" (until then %s cannot run in that session)", strings.Join(units, ", "))
|
|
}
|
|
return said, true
|
|
}
|
|
|
|
// accountOf is the account a resource of kind account is: named by the engine, or its target.
|
|
func accountOf(r inventory.ResourceHealth) string {
|
|
if r.Account != "" {
|
|
return r.Account
|
|
}
|
|
return r.Target
|
|
}
|
|
|
|
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
|
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
|
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
|
//
|
|
// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
|
|
// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
|
|
// and not something a machine can meet within a bound.
|
|
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
|
if f.healthErr != nil {
|
|
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
|
}
|
|
h, states := f.health[machine]
|
|
if !states {
|
|
return healthGood, ""
|
|
}
|
|
if h.HeardAt.Before(since) {
|
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
|
}
|
|
wait, waits := personWait(module, machine, h.Resources)
|
|
// **Only the build whose own send put the account into its wait is excused** (issue 318 review): an
|
|
// account already waiting before this send was not brought to it by this build, and a later build of
|
|
// the module sent while the login is still owed is judged as before. Two seconds of skew between the
|
|
// machine's clock and the controller's, as the witness allows.
|
|
if waits {
|
|
for _, r := range h.Resources {
|
|
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
|
strings.HasPrefix(r.Reason, link.ReasonRelogin) && r.Since.Before(since.Add(-2*time.Second)) {
|
|
waits = false
|
|
}
|
|
}
|
|
}
|
|
for _, r := range h.Resources {
|
|
if r.Module != module {
|
|
continue
|
|
}
|
|
if waits && r.State == link.StateUnhealthy {
|
|
continue
|
|
}
|
|
switch r.State {
|
|
case link.StateHealthy:
|
|
case link.StateStarting:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
|
case link.StateUnhealthy:
|
|
if on, held := f.heldOn[module+"@"+machine]; held {
|
|
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
|
r.Resource, machine, on)
|
|
}
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
|
default:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
|
|
reasonAfter(r.Reason))
|
|
}
|
|
}
|
|
if waits {
|
|
return healthPerson, wait
|
|
}
|
|
return healthGood, ""
|
|
}
|
|
|
|
func reasonAfter(s string) string {
|
|
if s == "" {
|
|
return ""
|
|
}
|
|
return ": " + s
|
|
}
|
|
|
|
// healthLines is what `node show` says of a machine's long-running resources: each with its state and
|
|
// since when, an unhealthy one said once marked unconfirmed.
|
|
func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
|
if !had {
|
|
return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
|
|
}
|
|
if len(h.Resources) == 0 {
|
|
return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
|
|
}
|
|
out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
|
|
for _, r := range h.Resources {
|
|
line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
|
|
r.Since.Local().Format("2006-01-02 15:04"))
|
|
if r.Reason != "" {
|
|
line += " — " + r.Reason
|
|
}
|
|
if r.Restarts > 0 {
|
|
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
|
}
|
|
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
|
line += " (unconfirmed: said once)"
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
return out
|
|
}
|