jochen
9006c82393
A module's condition told the operator to run the node-engine's hand-over at the machine, as root (issue 339), and the mesh had no channel for it. Now node hand-over <node> <path> is the controller's terminal's — a node subcommand that is not a read, so every verb and mesh-cli outside the terminal refuse it — and asks that node's engine on mesh.node.<node>.ask.hand-over, a request only the controller may publish and only that node's engine may hear and answer (its grant gains the subject and the right to answer what it was asked). The line's node and path are judged before anything is asked; the engine's answer is printed, a refusal as a refusal. Every text addressed to the operator names the nox line (ADR 0272); the condition's words stay plain.
688 lines
29 KiB
Go
688 lines
29 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
|
|
// Phase A).
|
|
//
|
|
// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
|
|
// machine states, in every report and as an event between reports, the state of every long-running
|
|
// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
|
|
// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
|
|
// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
|
|
// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
|
|
// the stated health: a judging passes a module only when every long-running resource of it on that
|
|
// machine is stated healthy, so a resource still starting is not yet a pass.
|
|
//
|
|
// **An engine older than the judging states nothing**, and its machine's health is not known: never
|
|
// healthy, never a reason to raise anything, and the gate judges it as it did before.
|
|
|
|
// The condition a module's health raises.
|
|
const (
|
|
kindModuleUnhealthy = "module-unhealthy"
|
|
// sourceHealth is what raised it: the machine's own statement.
|
|
sourceHealth = "health"
|
|
// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
|
|
moduleUnhealthyUrgentAfter = 4 * time.Hour
|
|
// moduleUnhealthyAfter is how many statements in a row raise it.
|
|
moduleUnhealthyAfter = 2
|
|
)
|
|
|
|
// healthRefused counts the statements refused as older than the one kept, for the log and a test.
|
|
var healthRefused atomic.Int64
|
|
|
|
// moduleHealth keeps what the machines state, for the link (link.Healths).
|
|
type moduleHealth struct {
|
|
inv *inventory.Inventory
|
|
keeper func() *conditions.Keeper
|
|
}
|
|
|
|
func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
|
|
return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
|
|
}
|
|
|
|
// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
|
|
// older statement than the one kept is refused, by when the engine looked.
|
|
func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
|
|
now time.Time) error {
|
|
if h.Contract == 0 {
|
|
return nil
|
|
}
|
|
prev, had, err := inv.HealthOf(ctx, node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if had && h.At.Before(prev.SaidAt) {
|
|
healthRefused.Add(1)
|
|
return nil
|
|
}
|
|
unhealthy := map[string][]inventory.ResourceHealth{}
|
|
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
|
for _, r := range h.Resources {
|
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
|
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
|
resources = append(resources, kept)
|
|
if r.State == link.StateUnhealthy && r.Module != "" {
|
|
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
|
}
|
|
}
|
|
streaks := map[string]int{}
|
|
for module := range unhealthy {
|
|
streaks[module] = prev.Streaks[module] + 1
|
|
}
|
|
var network *inventory.NetworkHealth
|
|
if h.Network != nil {
|
|
network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}}
|
|
for _, p := range h.Network.Parts {
|
|
network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
|
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak})
|
|
}
|
|
}
|
|
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
|
HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)})
|
|
if err != nil || !stored {
|
|
if err == nil {
|
|
healthRefused.Add(1)
|
|
}
|
|
return err
|
|
}
|
|
if k == nil {
|
|
return nil
|
|
}
|
|
err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
|
// And the machine's own failed units (issue 315): what no module places, one finding for the machine.
|
|
if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil {
|
|
if err == nil {
|
|
err = uerr
|
|
} else {
|
|
err = fmt.Errorf("%w; %v", err, uerr)
|
|
}
|
|
}
|
|
// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
|
|
// machine can hold another's finding, or release it.
|
|
if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
|
|
if err == nil {
|
|
return nerr
|
|
}
|
|
return fmt.Errorf("%w; %v", err, nerr)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
|
// resource of it is unhealthy — or on the first while it is already open — and clears every one this
|
|
// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48
|
|
// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone
|
|
// waits on it.
|
|
func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string,
|
|
unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
|
|
open, err := k.Open(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
standing := map[string]conditions.Condition{}
|
|
for _, c := range open {
|
|
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
|
|
c.Subject.Machine == node {
|
|
standing[c.Key] = c
|
|
}
|
|
}
|
|
var hold *holding
|
|
if inv != nil {
|
|
if hold, err = readHolding(ctx, inv, open); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
var problems []string
|
|
modules := make([]string, 0, len(unhealthy))
|
|
for m := range unhealthy {
|
|
modules = append(modules, m)
|
|
}
|
|
sort.Strings(modules)
|
|
seen := map[string]bool{}
|
|
// became is, per module, the kind of condition this statement says of it: what a standing one of the
|
|
// other kind turned into.
|
|
became := map[string]string{}
|
|
heldOn := map[string]string{}
|
|
providers := map[catalogue.Chosen]bool{}
|
|
for _, m := range modules {
|
|
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
|
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
|
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
|
if said, waits := foundWait(m, node, unhealthy[m]); waits {
|
|
o := usedAsFoundObservation(m, node, said, unhealthy[m])
|
|
seen[o.Key()] = true
|
|
became[m] = kindUsedAsFound
|
|
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
|
|
// operator, never urgent, cleared on the first statement that no longer says it.
|
|
if said, waits := personWait(m, node, unhealthy[m]); waits {
|
|
o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
|
|
// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
|
|
// consumers are held under it, and its own condition is where they are said.
|
|
if hold != nil {
|
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
}
|
|
seen[o.Key()] = true
|
|
became[m] = kindReloginNeeded
|
|
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
|
if hold != nil {
|
|
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
|
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
|
heldOn[o.Key()] = p.Module + " on " + p.Node
|
|
providers[p] = true
|
|
continue
|
|
}
|
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
}
|
|
seen[o.Key()] = true
|
|
became[m] = kindModuleUnhealthy
|
|
c, isOpen := standing[o.Key()]
|
|
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
|
}
|
|
if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
|
|
o.Severity = conditions.Urgent
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
for key, c := range standing {
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
|
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
|
|
if c.Kind == kindReloginNeeded {
|
|
why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
|
|
}
|
|
if c.Kind == kindUsedAsFound {
|
|
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
|
}
|
|
if on, held := heldOn[key]; held {
|
|
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
|
}
|
|
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
|
// line says what it became.
|
|
resolved := ""
|
|
switch {
|
|
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
|
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
|
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
|
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
|
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
|
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
|
}
|
|
if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
// And each provider a consumer here now waits on, when its own condition is open: said again with who
|
|
// waits on it, so the wait is listed at the provider whichever machine's statement arrived first.
|
|
for p := range providers {
|
|
if err := sayWaiters(ctx, k, hold, p, now); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
|
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
|
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
|
var raisedAt *conditions.Condition
|
|
for i, c := range hold.open {
|
|
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
|
raisedAt = &hold.open[i]
|
|
}
|
|
}
|
|
if raisedAt == nil {
|
|
return nil
|
|
}
|
|
var rs []inventory.ResourceHealth
|
|
for _, r := range hold.healths[p.Node].Resources {
|
|
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
|
rs = append(rs, r)
|
|
}
|
|
}
|
|
if len(rs) == 0 {
|
|
return nil
|
|
}
|
|
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
|
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
|
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
|
}
|
|
if o.Key() != raisedAt.Key {
|
|
return nil // its own statement says it next
|
|
}
|
|
sayWaitingOn(&o, hold.waitersOn(p))
|
|
_, err := k.Observe(ctx, o)
|
|
return err
|
|
}
|
|
|
|
// reloginKey is a module's relogin-needed condition on a machine.
|
|
func reloginKey(module, node string) string {
|
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
|
}
|
|
|
|
// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
|
|
func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
|
|
if inv == nil {
|
|
return ""
|
|
}
|
|
n, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return n.Account
|
|
}
|
|
|
|
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
|
|
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
|
|
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
|
|
// person in again, and a restart of the module's service would start it in the same session. operator is
|
|
// the machine's operator account, when known: the words say "your account" only for it.
|
|
func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
o := moduleUnhealthyObservation(module, node, rs)
|
|
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
|
|
accounts := waitingAccounts(module, rs)
|
|
yours := operator != "" && len(accounts) > 0
|
|
for _, a := range accounts {
|
|
yours = yours && a == operator
|
|
}
|
|
w := reloginWords(module, node, yours)
|
|
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
|
|
return o
|
|
}
|
|
|
|
// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
|
|
// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
|
|
// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
|
|
func reloginWords(module, node string, yours bool) words {
|
|
if yours {
|
|
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
|
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
|
|
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
|
|
"cannot run until you log in again. Its update is in place and nothing was undone.",
|
|
conditions.Capital(module), module),
|
|
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
|
|
}
|
|
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
|
Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
|
|
Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
|
|
"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
|
|
conditions.Capital(module), node, module),
|
|
Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
|
|
}
|
|
|
|
// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
|
|
func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, r := range rs {
|
|
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
|
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
|
|
seen[accountOf(r)] = true
|
|
out = append(out, accountOf(r))
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
|
// waits on it, whether it is not working or waits for a new login.
|
|
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
|
if len(waiters) == 0 {
|
|
return
|
|
}
|
|
o.Severity = conditions.Urgent
|
|
o.Said += "; " + waitingWords(waiters)
|
|
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
|
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
|
}
|
|
|
|
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
|
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
|
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
var words, said, plain []string
|
|
needs := moduleNeeds(node, rs)
|
|
for _, r := range rs {
|
|
plain = append(plain, resourcePlainWords(r))
|
|
if r.Kind == link.KindUnit {
|
|
// A failed unit is named by the unit, which is what a person looks for (issue 315); the
|
|
// resource that places it — a package, a file — is evidence.
|
|
words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason))
|
|
said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource,
|
|
orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
|
continue
|
|
}
|
|
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
|
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
|
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
|
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
|
}
|
|
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
|
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
|
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
|
Said: strings.Join(said, "; "),
|
|
Headline: fmt.Sprintf("%s not working on %s", module, node),
|
|
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
|
namesWords(plain, 3)),
|
|
Needs: needs,
|
|
Actions: moduleActions(node, rs),
|
|
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
|
}
|
|
|
|
// reasonWords is why a resource is unhealthy, as a person reads it.
|
|
func reasonWords(r inventory.ResourceHealth) string {
|
|
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
|
|
if r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) {
|
|
return fmt.Sprintf("waits for a new login of %s, which is in the group and logged in before it was", accountOf(r))
|
|
}
|
|
switch r.Reason {
|
|
case "restarting":
|
|
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
|
case "down":
|
|
return "is not running"
|
|
case "":
|
|
return "is unhealthy"
|
|
}
|
|
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
|
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
|
if r.Check != "" {
|
|
return "fails its " + r.Check + " check"
|
|
}
|
|
return "is unhealthy: " + r.Reason
|
|
}
|
|
|
|
func orNotSaid(s string) string {
|
|
if s == "" {
|
|
return "no reason said"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
|
|
// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
|
|
const kindReloginNeeded = "relogin-needed"
|
|
|
|
// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
|
|
// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
|
|
// excused:
|
|
//
|
|
// - at least one of the module's resources is its account (kind account), unhealthy with a reason that
|
|
// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
|
|
// running began before;
|
|
// - every other unhealthy resource of the module runs in the own service manager of one of those very
|
|
// accounts (Account names it): the manager that began before the group and does not hold it.
|
|
//
|
|
// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
|
|
// another account's manager, a resource whose engine does not say whose manager it is in, an account
|
|
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
|
|
// resource still starting is not unhealthy and does not make a wait either. Pure.
|
|
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
|
|
accounts := waitingAccounts(module, rs)
|
|
if len(accounts) == 0 {
|
|
return "", false
|
|
}
|
|
waiting := map[string]bool{}
|
|
for _, a := range accounts {
|
|
waiting[a] = true
|
|
}
|
|
var units []string
|
|
for _, r := range rs {
|
|
if r.Module != module || r.State != link.StateUnhealthy {
|
|
continue
|
|
}
|
|
switch {
|
|
case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
|
|
case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
|
|
units = append(units, r.Target)
|
|
default:
|
|
return "", false
|
|
}
|
|
}
|
|
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
|
|
"login began before it was; log out of %[1]s completely and log in again, or restart it", machine, module,
|
|
strings.Join(accounts, ", "))
|
|
if len(units) > 0 {
|
|
sort.Strings(units)
|
|
said += fmt.Sprintf(" (until then %s cannot run)", strings.Join(units, ", "))
|
|
}
|
|
return said, true
|
|
}
|
|
|
|
// accountOf is the account a resource of kind account is: named by the engine, or its target.
|
|
func accountOf(r inventory.ResourceHealth) string {
|
|
if r.Account != "" {
|
|
return r.Account
|
|
}
|
|
return r.Target
|
|
}
|
|
|
|
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
|
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
|
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
|
//
|
|
// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
|
|
// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
|
|
// and not something a machine can meet within a bound.
|
|
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
|
if f.healthErr != nil {
|
|
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
|
}
|
|
h, states := f.health[machine]
|
|
if !states {
|
|
return healthGood, ""
|
|
}
|
|
if h.HeardAt.Before(since) {
|
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
|
}
|
|
wait, waits := personWait(module, machine, h.Resources)
|
|
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
|
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
|
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
|
// have brought a wait, so a later build sent while the login is still owed is judged as before.
|
|
if waits && !f.groupsAdded[module] {
|
|
waits = false
|
|
}
|
|
var found []string
|
|
for _, r := range h.Resources {
|
|
if r.Module != module {
|
|
continue
|
|
}
|
|
if waits && r.State == link.StateUnhealthy {
|
|
continue
|
|
}
|
|
// **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine
|
|
// left its owner and mode, and only someone at the machine can hand it over. It is no fault of this
|
|
// build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict
|
|
// carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy.
|
|
if usedAsFound(r) && r.Since.Before(since) {
|
|
found = append(found, r.Resource)
|
|
continue
|
|
}
|
|
switch r.State {
|
|
case link.StateHealthy:
|
|
case link.StateStarting:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
|
case link.StateUnhealthy:
|
|
if on, held := f.heldOn[module+"@"+machine]; held {
|
|
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
|
r.Resource, machine, on)
|
|
}
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
|
default:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
|
|
reasonAfter(r.Reason))
|
|
}
|
|
}
|
|
if waits || len(found) > 0 {
|
|
var said []string
|
|
if waits {
|
|
said = append(said, wait)
|
|
}
|
|
if len(found) > 0 {
|
|
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
|
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
|
}
|
|
return healthPerson, strings.Join(said, "; ")
|
|
}
|
|
return healthGood, ""
|
|
}
|
|
|
|
// usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339).
|
|
func usedAsFound(r inventory.ResourceHealth) bool {
|
|
return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound)
|
|
}
|
|
|
|
func reasonAfter(s string) string {
|
|
if s == "" {
|
|
return ""
|
|
}
|
|
return ": " + s
|
|
}
|
|
|
|
// healthLines is what `node show` says of a machine's long-running resources: each with its state and
|
|
// since when, an unhealthy one said once marked unconfirmed.
|
|
func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
|
if !had {
|
|
return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
|
|
}
|
|
if len(h.Resources) == 0 {
|
|
return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
|
|
}
|
|
out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
|
|
for _, r := range h.Resources {
|
|
line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
|
|
r.Since.Local().Format("2006-01-02 15:04"))
|
|
if r.Reason != "" {
|
|
line += " — " + r.Reason
|
|
}
|
|
if r.Restarts > 0 {
|
|
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
|
}
|
|
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
|
line += " (unconfirmed: said once)"
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// accountGroups is every account group a manifest declares, as "<account>/<group>": a user resource's
|
|
// groups (ADR 0252).
|
|
func accountGroups(m catalogue.Manifest) map[string]bool {
|
|
out := map[string]bool{}
|
|
for _, r := range m.Resources {
|
|
if t, _ := r["type"].(string); t != "user" {
|
|
continue
|
|
}
|
|
name, _ := r["name"].(string)
|
|
groups, _ := r["groups"].([]any)
|
|
for _, g := range groups {
|
|
if group, ok := g.(string); ok && group != "" {
|
|
out[name+"/"+group] = true
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// addsAccountGroups is whether a move from one manifest of a module to another puts an account in a group the
|
|
// earlier one did not (issue 318 review): the only send that can bring a wait for a new login. A module new to
|
|
// the machine (no earlier manifest) adds every group it declares.
|
|
func addsAccountGroups(from catalogue.Manifest, hadFrom bool, to catalogue.Manifest) bool {
|
|
before := map[string]bool{}
|
|
if hadFrom {
|
|
before = accountGroups(from)
|
|
}
|
|
for g := range accountGroups(to) {
|
|
if !before[g] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// kindUsedAsFound is a module's condition while the node-engine uses one of its directories as found (novox/hq
|
|
// issue 339): its own kind, the operator's, never urgent, and never read by the gate as a fault of a build.
|
|
const kindUsedAsFound = "directory-used-as-found"
|
|
|
|
// usedAsFoundKey is a module's used-as-found condition on a machine.
|
|
func usedAsFoundKey(module, node string) string {
|
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindUsedAsFound)
|
|
}
|
|
|
|
// foundWait is whether everything unhealthy of a module on a machine is a directory used as found, and that in
|
|
// one sentence. Anything else unhealthy beside it is judged as a fault, with the directory among its resources.
|
|
func foundWait(module, node string, rs []inventory.ResourceHealth) (string, bool) {
|
|
var ids, why []string
|
|
for _, r := range rs {
|
|
if r.Module != module || r.State != link.StateUnhealthy {
|
|
continue
|
|
}
|
|
if !usedAsFound(r) {
|
|
return "", false
|
|
}
|
|
ids = append(ids, r.Resource)
|
|
why = append(why, strings.TrimSpace(strings.TrimPrefix(r.Reason, link.ReasonUsedAsFound)))
|
|
}
|
|
if len(ids) == 0 {
|
|
return "", false
|
|
}
|
|
return fmt.Sprintf("%s on %s uses %s as found: %s", module, node, strings.Join(ids, ", "),
|
|
strings.Join(why, "; ")), true
|
|
}
|
|
|
|
// usedAsFoundObservation is a module whose directory the node-engine uses as found, in words: the operator's, a
|
|
// warning however long it stays, its summary naming the directories and their owners; the paths are evidence.
|
|
func usedAsFoundObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
o := moduleUnhealthyObservation(module, node, rs)
|
|
o.Token, o.Kind, o.Resolver, o.Severity, o.Summary = kindUsedAsFound, kindUsedAsFound, conditions.ResolverOperator,
|
|
conditions.Warning, said
|
|
o.Headline = fmt.Sprintf("%s waits for a directory on %s", module, node)
|
|
o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+
|
|
"The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.",
|
|
module, node, module, module)
|
|
// Plain words (ADR 0253): the line itself — `nox node hand-over <node> <path>` on the control-node (ADR 0272,
|
|
// issue 356) — is in the summary and the evidence, which name the directory; a path is never in these.
|
|
o.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type."
|
|
o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node)
|
|
o.Actions = nil
|
|
return o
|
|
}
|