jschoubben
1c117d1189
A keep wait on a filesystem that cannot carry ACLs is not excused, and nothing is asked; one whose filesystem must first enable them says in the ask that the take sets it. The tool runner's account is sent with no uid, for the engine to resolve by name.
870 lines
39 KiB
Go
870 lines
39 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
|
|
// Phase A).
|
|
//
|
|
// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
|
|
// machine states, in every report and as an event between reports, the state of every long-running
|
|
// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
|
|
// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
|
|
// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
|
|
// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
|
|
// the stated health: a judging passes a module only when every long-running resource of it on that
|
|
// machine is stated healthy, so a resource still starting is not yet a pass.
|
|
//
|
|
// **An engine older than the judging states nothing**, and its machine's health is not known: never
|
|
// healthy, never a reason to raise anything, and the gate judges it as it did before.
|
|
|
|
// The condition a module's health raises.
|
|
const (
|
|
kindModuleUnhealthy = "module-unhealthy"
|
|
// sourceHealth is what raised it: the machine's own statement.
|
|
sourceHealth = "health"
|
|
// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
|
|
moduleUnhealthyUrgentAfter = 4 * time.Hour
|
|
// moduleUnhealthyAfter is how many statements in a row raise it.
|
|
moduleUnhealthyAfter = 2
|
|
)
|
|
|
|
// healthRefused counts the statements refused as older than the one kept, for the log and a test.
|
|
var healthRefused atomic.Int64
|
|
|
|
// moduleHealth keeps what the machines state, for the link (link.Healths).
|
|
type moduleHealth struct {
|
|
inv *inventory.Inventory
|
|
keeper func() *conditions.Keeper
|
|
}
|
|
|
|
func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
|
|
return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
|
|
}
|
|
|
|
// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
|
|
// older statement than the one kept is refused, by when the engine looked.
|
|
func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
|
|
now time.Time) error {
|
|
if h.Contract == 0 {
|
|
return nil
|
|
}
|
|
prev, had, err := inv.HealthOf(ctx, node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if had && h.At.Before(prev.SaidAt) {
|
|
healthRefused.Add(1)
|
|
return nil
|
|
}
|
|
unhealthy := map[string][]inventory.ResourceHealth{}
|
|
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
|
for _, r := range h.Resources {
|
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
|
Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
|
|
for _, w := range r.Waits {
|
|
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, What: w.What})
|
|
}
|
|
// An access's waits (novox/hq ADR 0325 §7), kept beside the others as one more thing the operator gives: the
|
|
// access by its module's id, and whether it waits for a disk rather than for the operator's word.
|
|
for _, w := range r.AccessWaits {
|
|
kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Access: w.Access, Disk: w.Reason == link.WaitDisk,
|
|
Placed: w.Placed, Reason: w.Reason, ACL: w.ACL, What: w.What})
|
|
}
|
|
for _, e := range r.Entries {
|
|
kept.Entries = append(kept.Entries, inventory.KeptEntry{Identity: e.Identity, UID: e.UID, Use: e.Use, From: e.From,
|
|
Note: e.Note})
|
|
}
|
|
resources = append(resources, kept)
|
|
}
|
|
// **A wait for the operator is checked before it is excused** (novox/hq ADR 0283): a waiting resource whose
|
|
// wait does not check out is judged unhealthy, saying why; one that does is kept beside the unhealthy ones, so
|
|
// judgeModuleHealth can say it as needs-operator. What is stored is what the machine said.
|
|
var wf operatorWaitFacts
|
|
if mods := waitingModules(resources); len(mods) > 0 {
|
|
readWaitFacts(ctx, inv, node, mods, nil, &wf)
|
|
}
|
|
for _, r := range checkWaiting(node, resources, wf) {
|
|
if (r.State == link.StateUnhealthy || r.State == link.StateWaiting) && r.Module != "" {
|
|
unhealthy[r.Module] = append(unhealthy[r.Module], r)
|
|
}
|
|
}
|
|
streaks := map[string]int{}
|
|
for module := range unhealthy {
|
|
streaks[module] = prev.Streaks[module] + 1
|
|
}
|
|
var network *inventory.NetworkHealth
|
|
if h.Network != nil {
|
|
network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}}
|
|
for _, p := range h.Network.Parts {
|
|
network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
|
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak})
|
|
}
|
|
}
|
|
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
|
HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)})
|
|
if err != nil || !stored {
|
|
if err == nil {
|
|
healthRefused.Add(1)
|
|
}
|
|
return err
|
|
}
|
|
if k == nil {
|
|
return nil
|
|
}
|
|
err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
|
// And the machine's own failed units (issue 315): what no module places, one finding for the machine.
|
|
if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil {
|
|
if err == nil {
|
|
err = uerr
|
|
} else {
|
|
err = fmt.Errorf("%w; %v", err, uerr)
|
|
}
|
|
}
|
|
// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
|
|
// machine can hold another's finding, or release it.
|
|
if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
|
|
if err == nil {
|
|
return nerr
|
|
}
|
|
return fmt.Errorf("%w; %v", err, nerr)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
|
// resource of it is unhealthy — or on the first while it is already open — and clears every one this
|
|
// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48
|
|
// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone
|
|
// waits on it.
|
|
func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string,
|
|
unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
|
|
open, err := k.Open(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
standing := map[string]conditions.Condition{}
|
|
for _, c := range open {
|
|
// A module left out of the composition is the self-check's to raise and clear, never a statement's
|
|
// (novox/hq issue 380): its needs-operator is not cleared for not being in what the machine runs.
|
|
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
|
|
c.Kind == kindNeedsOperator) &&
|
|
c.Subject.Machine == node && !isLeftOutCondition(c) {
|
|
standing[c.Key] = c
|
|
}
|
|
}
|
|
hold, err := readHoldingFor(ctx, inv, open)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var problems []string
|
|
modules := make([]string, 0, len(unhealthy))
|
|
for m := range unhealthy {
|
|
modules = append(modules, m)
|
|
}
|
|
sort.Strings(modules)
|
|
seen := map[string]bool{}
|
|
// became is, per module, the kind of condition this statement says of it: what a standing one of the
|
|
// other kind turned into.
|
|
became := map[string]string{}
|
|
heldOn := map[string]string{}
|
|
providers := map[catalogue.Chosen]bool{}
|
|
for _, m := range modules {
|
|
// **A part that waits for the operator is said as that** (novox/hq ADR 0283): its waits already checked,
|
|
// the operator's, never urgent, its words naming the act.
|
|
if waits, waiting := operatorWait(m, unhealthy[m]); waiting {
|
|
o := needsOperatorObservation(m, node, waits, unhealthy[m])
|
|
// **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a
|
|
// login does: its consumers are held under it.
|
|
if hold != nil {
|
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
}
|
|
seen[o.Key()] = true
|
|
became[m] = kindNeedsOperator
|
|
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
// **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked
|
|
// wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the
|
|
// operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator
|
|
// condition stands beside the other, on the same looks; what follows judges the rest without the wait.
|
|
if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 {
|
|
o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m]))
|
|
seen[o.Key()] = true
|
|
if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen {
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
unhealthy[m] = rest
|
|
}
|
|
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
|
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
|
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
|
if said, waits := foundWait(m, node, unhealthy[m]); waits {
|
|
o := usedAsFoundObservation(m, node, said, unhealthy[m])
|
|
seen[o.Key()] = true
|
|
became[m] = kindUsedAsFound
|
|
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
|
|
// operator, never urgent, cleared on the first statement that no longer says it.
|
|
if said, waits := personWait(m, node, unhealthy[m]); waits {
|
|
o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
|
|
// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
|
|
// consumers are held under it, and its own condition is where they are said.
|
|
if hold != nil {
|
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
}
|
|
seen[o.Key()] = true
|
|
became[m] = kindReloginNeeded
|
|
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
|
if hold != nil {
|
|
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
|
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
|
// clearing line says which the provider is: unhealthy, or waiting for the operator (issue 405).
|
|
p := by.provider
|
|
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which is unhealthy"
|
|
if len(by.waits) > 0 {
|
|
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which waits for you"
|
|
}
|
|
providers[p] = true
|
|
continue
|
|
}
|
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
// A provider that waits for the operator for a part this finding cannot be matched to does not hold it
|
|
// (novox/hq issue 405): raised as its own, and saying the provider waits, so neither is hidden.
|
|
if p, waiting := hold.waitingUncovered(node, m, unhealthy[m]); waiting {
|
|
o.Said += fmt.Sprintf("; %s on %s waits for you, but not for anything %s is known to need, so %s's "+
|
|
"fault is said on its own", p.Module, p.Node, m, m)
|
|
}
|
|
}
|
|
seen[o.Key()] = true
|
|
became[m] = kindModuleUnhealthy
|
|
c, isOpen := standing[o.Key()]
|
|
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
|
}
|
|
if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
|
|
o.Severity = conditions.Urgent
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
for key, c := range standing {
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
|
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
|
|
if c.Kind == kindReloginNeeded {
|
|
why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
|
|
}
|
|
if c.Kind == kindUsedAsFound {
|
|
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
|
}
|
|
if c.Kind == kindNeedsOperator {
|
|
why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module)
|
|
}
|
|
if on, held := heldOn[key]; held {
|
|
why = fmt.Sprintf("what %s finds on %s waits on %s: held under its condition", module, node, on)
|
|
}
|
|
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
|
// line says what it became.
|
|
resolved := ""
|
|
switch {
|
|
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
|
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
|
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
|
case c.Kind == kindModuleUnhealthy && became[module] == kindNeedsOperator:
|
|
why = fmt.Sprintf("%s on %s now only waits for the operator", module, node)
|
|
resolved = fmt.Sprintf("%s on %s now only waits for you", module, node)
|
|
case c.Kind == kindNeedsOperator && became[module] == kindModuleUnhealthy:
|
|
why = fmt.Sprintf("%s on %s no longer only waits for the operator, and is not healthy", module, node)
|
|
resolved = fmt.Sprintf("What %s on %s waited for is given, and it still does not work", module, node)
|
|
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
|
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
|
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
|
}
|
|
if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
// And each provider a consumer here now waits on, when its own condition is open: said again with who
|
|
// waits on it, so the wait is listed at the provider whichever machine's statement arrived first.
|
|
for p := range providers {
|
|
if err := sayWaiters(ctx, k, hold, p, now); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
|
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
|
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
|
// Its statement as it was judged, its waits checked (novox/hq issue 450): a wait that failed the check is
|
|
// unhealthy, so the condition built here is the one its own statement raised, and who waits on it is listed.
|
|
var rs []inventory.ResourceHealth
|
|
for _, r := range hold.checked(p.Node) {
|
|
if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) {
|
|
rs = append(rs, r)
|
|
}
|
|
}
|
|
if len(rs) == 0 {
|
|
return nil
|
|
}
|
|
// The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq
|
|
// issue 405), else that for the rest of it, a wait beside it said on its own.
|
|
var o conditions.Observation
|
|
if waits, waiting := operatorWait(p.Module, rs); waiting {
|
|
o = needsOperatorObservation(p.Module, p.Node, waits, rs)
|
|
} else {
|
|
_, rest := besideAWait(p.Module, rs)
|
|
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
|
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
|
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
|
}
|
|
}
|
|
raised := false
|
|
for _, c := range hold.open {
|
|
raised = raised || c.Key == o.Key()
|
|
}
|
|
if !raised {
|
|
return nil // not open yet, or open as another kind: its own statement says it next
|
|
}
|
|
sayWaitingOn(&o, hold.waitersOn(p))
|
|
_, err := k.Observe(ctx, o)
|
|
return err
|
|
}
|
|
|
|
// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked
|
|
// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the
|
|
// module only waits (operatorWait says that whole). Pure.
|
|
func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) {
|
|
if _, only := operatorWait(module, rs); only {
|
|
return nil, rs
|
|
}
|
|
var waits []inventory.Wait
|
|
var rest []inventory.ResourceHealth
|
|
for _, r := range rs {
|
|
if r.Module == module && r.State == link.StateWaiting {
|
|
waits = append(waits, r.Waits...)
|
|
continue
|
|
}
|
|
rest = append(rest, r)
|
|
}
|
|
if len(waits) == 0 {
|
|
return nil, rs
|
|
}
|
|
return waits, rest
|
|
}
|
|
|
|
// waitingOf is a module's waiting resources: the evidence of its wait.
|
|
func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth {
|
|
var out []inventory.ResourceHealth
|
|
for _, r := range rs {
|
|
if r.Module == module && r.State == link.StateWaiting {
|
|
out = append(out, r)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// reloginKey is a module's relogin-needed condition on a machine.
|
|
func reloginKey(module, node string) string {
|
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
|
}
|
|
|
|
// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
|
|
func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
|
|
if inv == nil {
|
|
return ""
|
|
}
|
|
n, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return n.Account
|
|
}
|
|
|
|
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
|
|
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
|
|
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
|
|
// person in again, and a restart of the module's service would start it in the same session. operator is
|
|
// the machine's operator account, when known: the words say "your account" only for it.
|
|
func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
o := moduleUnhealthyObservation(module, node, rs)
|
|
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
|
|
accounts := waitingAccounts(module, rs)
|
|
yours := operator != "" && len(accounts) > 0
|
|
for _, a := range accounts {
|
|
yours = yours && a == operator
|
|
}
|
|
w := reloginWords(module, node, yours)
|
|
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
|
|
return o
|
|
}
|
|
|
|
// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
|
|
// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
|
|
// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
|
|
func reloginWords(module, node string, yours bool) words {
|
|
if yours {
|
|
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
|
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
|
|
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
|
|
"cannot run until you log in again. Its update is in place and nothing was undone.",
|
|
conditions.Capital(module), module),
|
|
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
|
|
}
|
|
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
|
Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
|
|
Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
|
|
"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
|
|
conditions.Capital(module), node, module),
|
|
Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
|
|
}
|
|
|
|
// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
|
|
func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, r := range rs {
|
|
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
|
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
|
|
seen[accountOf(r)] = true
|
|
out = append(out, accountOf(r))
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
|
// waits on it, whether it is not working or waits for a new login. **A wait for the operator's secret or setting
|
|
// stays a warning** (ADR 0283 decision 5, novox/hq issue 405): who waits on it is listed, and nothing escalates it.
|
|
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
|
if len(waiters) == 0 {
|
|
return
|
|
}
|
|
if o.Kind != kindNeedsOperator {
|
|
o.Severity = conditions.Urgent
|
|
}
|
|
o.Said += "; " + waitingWords(waiters)
|
|
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
|
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
|
}
|
|
|
|
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
|
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
|
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
var words, said, plain []string
|
|
needs := moduleNeeds(node, rs)
|
|
for _, r := range rs {
|
|
plain = append(plain, resourcePlainWords(r))
|
|
if r.Kind == link.KindUnit {
|
|
// A failed unit is named by the unit, which is what a person looks for (issue 315); the
|
|
// resource that places it — a package, a file — is evidence.
|
|
words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason))
|
|
said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource,
|
|
orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
|
continue
|
|
}
|
|
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
|
// The restart count runs from when the engine began watching it, not from the state's since (novox/hq
|
|
// issue 458).
|
|
said = append(said, fmt.Sprintf("%s (%s %s): %s, since %s, %d look(s) in a row, %s",
|
|
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"),
|
|
r.Streak, restartsWords(r.Restarts)))
|
|
}
|
|
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
|
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
|
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
|
Said: strings.Join(said, "; "),
|
|
Headline: fmt.Sprintf("%s not working on %s", module, node),
|
|
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
|
namesWords(plain, 3)),
|
|
Needs: needs,
|
|
Actions: moduleActions(node, rs),
|
|
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
|
}
|
|
|
|
// restartsWords is a restart count as what it is (novox/hq issue 458). The engine reports its own count
|
|
// (mesh-host's liveness judge, kept.Counted): every rise of systemd's NRestarts or the runtime's RestartCount
|
|
// after a start's grace, so only restarts by the Restart= setting or the restart policy. It keeps the count
|
|
// across a recreate, a start by hand and its own restarts, and forgets it only when the resource is no longer
|
|
// declared. So it counts automatic restarts since the engine began watching it, never since a state's "since".
|
|
func restartsWords(n int) string {
|
|
return fmt.Sprintf("%d automatic restart(s) since the node-engine began watching it", n)
|
|
}
|
|
|
|
// reasonWords is why a resource is unhealthy, as a person reads it.
|
|
func reasonWords(r inventory.ResourceHealth) string {
|
|
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
|
|
if r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) {
|
|
return fmt.Sprintf("waits for a new login of %s, which is in the group and logged in before it was", accountOf(r))
|
|
}
|
|
switch r.Reason {
|
|
case "restarting":
|
|
return fmt.Sprintf("keeps restarting (%s)", restartsWords(r.Restarts))
|
|
case "down":
|
|
return "is not running"
|
|
case "":
|
|
return "is unhealthy"
|
|
}
|
|
// A wait for the operator that did not check out is said as the controller found it (ADR 0283): names of
|
|
// secrets and settings only, never what the check itself said.
|
|
if strings.HasPrefix(r.Reason, waitRefusedPrefix) {
|
|
return r.Reason
|
|
}
|
|
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
|
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
|
if r.Check != "" {
|
|
return "fails its " + r.Check + " check"
|
|
}
|
|
return "is unhealthy: " + r.Reason
|
|
}
|
|
|
|
func orNotSaid(s string) string {
|
|
if s == "" {
|
|
return "no reason said"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
|
|
// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
|
|
const kindReloginNeeded = "relogin-needed"
|
|
|
|
// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
|
|
// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
|
|
// excused:
|
|
//
|
|
// - at least one of the module's resources is its account (kind account), unhealthy with a reason that
|
|
// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
|
|
// running began before;
|
|
// - every other unhealthy resource of the module runs in the own service manager of one of those very
|
|
// accounts (Account names it): the manager that began before the group and does not hold it.
|
|
//
|
|
// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
|
|
// another account's manager, a resource whose engine does not say whose manager it is in, an account
|
|
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
|
|
// resource still starting is not unhealthy and does not make a wait either. Pure.
|
|
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
|
|
accounts := waitingAccounts(module, rs)
|
|
if len(accounts) == 0 {
|
|
return "", false
|
|
}
|
|
waiting := map[string]bool{}
|
|
for _, a := range accounts {
|
|
waiting[a] = true
|
|
}
|
|
var units []string
|
|
for _, r := range rs {
|
|
if r.Module != module || r.State != link.StateUnhealthy {
|
|
continue
|
|
}
|
|
switch {
|
|
case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
|
|
case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
|
|
units = append(units, r.Target)
|
|
default:
|
|
return "", false
|
|
}
|
|
}
|
|
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
|
|
"login began before it was; log out of %[1]s completely and log in again, or restart it", machine, module,
|
|
strings.Join(accounts, ", "))
|
|
if len(units) > 0 {
|
|
sort.Strings(units)
|
|
said += fmt.Sprintf(" (until then %s cannot run)", strings.Join(units, ", "))
|
|
}
|
|
return said, true
|
|
}
|
|
|
|
// accountOf is the account a resource of kind account is: named by the engine, or its target.
|
|
func accountOf(r inventory.ResourceHealth) string {
|
|
if r.Account != "" {
|
|
return r.Account
|
|
}
|
|
return r.Target
|
|
}
|
|
|
|
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
|
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
|
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
|
//
|
|
// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
|
|
// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
|
|
// and not something a machine can meet within a bound.
|
|
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
|
if f.healthErr != nil {
|
|
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
|
}
|
|
h, states := f.health[machine]
|
|
if !states {
|
|
return healthGood, ""
|
|
}
|
|
if h.HeardAt.Before(since) {
|
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
|
}
|
|
// **A wait for the operator is checked first** (novox/hq ADR 0283): one that does not check out is unhealthy.
|
|
resources := checkWaiting(machine, h.Resources, f.waits)
|
|
wait, waits := personWait(module, machine, resources)
|
|
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
|
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
|
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
|
// have brought a wait, so a later build sent while the login is still owed is judged as before.
|
|
if waits && !f.groupsAdded[module] {
|
|
waits = false
|
|
}
|
|
var found, onWaiting, leftover []string
|
|
var forOperator []inventory.Wait
|
|
for _, r := range resources {
|
|
if r.Module != module {
|
|
continue
|
|
}
|
|
// **Any build is excused while its wait for the operator checks out** (ADR 0283 decision 4): a secret not
|
|
// given is owed by every build alike, so it is no fault of this one, and the verdict carries it.
|
|
if r.State == link.StateWaiting {
|
|
forOperator = append(forOperator, r.Waits...)
|
|
continue
|
|
}
|
|
if waits && r.State == link.StateUnhealthy {
|
|
continue
|
|
}
|
|
// **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine
|
|
// left its owner and mode, and only someone at the machine can hand it over. It is no fault of this
|
|
// build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict
|
|
// carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy.
|
|
if usedAsFound(r) && r.Since.Before(since) {
|
|
found = append(found, r.Resource)
|
|
continue
|
|
}
|
|
// **The mesh's entries left on a folder whose disk was away are a condition this build found** when its move
|
|
// leaves the module's accesses as they were, or they were stated before this send (novox/hq ADR 0325 §8): they
|
|
// may name an access no longer declared, they wait for the disk, and a delivery that did not bring them is not
|
|
// held for them — said in the verdict, as a directory used as found is. Otherwise they are judged unhealthy.
|
|
changed, known := f.accessesChanged[module]
|
|
changes := !known || changed
|
|
if r.Kind == link.KindKeptEntries && r.State == link.StateUnhealthy && (!changes || r.Since.Before(since)) {
|
|
leftover = append(leftover, r.Resource)
|
|
continue
|
|
}
|
|
// **An access the engine could not judge this look** (its filesystem could not be named — a hung mount) carries
|
|
// no wait. Of a module whose move leaves its accesses as they were it is a condition the move found, not
|
|
// counted; of one whose move changes them it is not yet healthy, held to the gate's bound like anything still
|
|
// unsettled, so the gate fails safe at the bound and never waits for ever.
|
|
if r.Kind == link.KindAccess && r.State == link.StateUnknown && !changes {
|
|
continue
|
|
}
|
|
switch r.State {
|
|
case link.StateHealthy:
|
|
case link.StateStarting:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
|
case link.StateUnhealthy:
|
|
if on, held := f.heldOn[module+"@"+machine]; held {
|
|
// **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a
|
|
// person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the
|
|
// operator's secret, whichever module owes it.
|
|
if on.waits != "" {
|
|
onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s",
|
|
r.Kind, r.Resource, machine, on.on, on.waits))
|
|
continue
|
|
}
|
|
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
|
r.Resource, machine, on.on)
|
|
}
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
|
default:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
|
|
reasonAfter(r.Reason))
|
|
}
|
|
}
|
|
if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 || len(leftover) > 0 {
|
|
said := onWaiting
|
|
if waits {
|
|
said = append(said, wait)
|
|
}
|
|
if len(forOperator) > 0 {
|
|
said = append(said, operatorWaitSaid(module, machine, forOperator))
|
|
}
|
|
if len(found) > 0 {
|
|
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
|
|
"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))
|
|
}
|
|
if len(leftover) > 0 {
|
|
said = append(said, fmt.Sprintf("on %s, the mesh's entries %s could not be taken out while their disk was "+
|
|
"away, stated before this send: they are taken out when it is back", machine, strings.Join(leftover, ", ")))
|
|
}
|
|
return healthPerson, strings.Join(said, "; ")
|
|
}
|
|
return healthGood, ""
|
|
}
|
|
|
|
// usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339).
|
|
func usedAsFound(r inventory.ResourceHealth) bool {
|
|
return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound)
|
|
}
|
|
|
|
func reasonAfter(s string) string {
|
|
if s == "" {
|
|
return ""
|
|
}
|
|
return ": " + s
|
|
}
|
|
|
|
// healthLines is what `node show` says of a machine's long-running resources: each with its state and
|
|
// since when, an unhealthy one said once marked unconfirmed.
|
|
func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
|
if !had {
|
|
return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
|
|
}
|
|
if len(h.Resources) == 0 {
|
|
return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
|
|
}
|
|
out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
|
|
for _, r := range h.Resources {
|
|
line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
|
|
r.Since.Local().Format("2006-01-02 15:04"))
|
|
if r.Reason != "" {
|
|
line += " — " + r.Reason
|
|
}
|
|
if r.Restarts > 0 {
|
|
line += ", " + restartsWords(r.Restarts)
|
|
}
|
|
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
|
line += " (unconfirmed: said once)"
|
|
}
|
|
out = append(out, line)
|
|
// A kept folder's users, as the engine resolved them (novox/hq ADR 0325 §4), the runner's note beside it.
|
|
for _, e := range r.Entries {
|
|
entry := fmt.Sprintf(" entry %s (uid %d, %s, from the %s)", e.Identity, e.UID, e.Use, e.From)
|
|
if e.Note != "" {
|
|
entry += " — " + e.Note
|
|
}
|
|
out = append(out, entry)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// accountGroups is every account group a manifest declares, as "<account>/<group>": a user resource's
|
|
// groups (ADR 0252).
|
|
func accountGroups(m catalogue.Manifest) map[string]bool {
|
|
out := map[string]bool{}
|
|
for _, r := range m.Resources {
|
|
if t, _ := r["type"].(string); t != "user" {
|
|
continue
|
|
}
|
|
name, _ := r["name"].(string)
|
|
groups, _ := r["groups"].([]any)
|
|
for _, g := range groups {
|
|
if group, ok := g.(string); ok && group != "" {
|
|
out[name+"/"+group] = true
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// addsAccountGroups is whether a move from one manifest of a module to another puts an account in a group the
|
|
// earlier one did not (issue 318 review): the only send that can bring a wait for a new login. A module new to
|
|
// the machine (no earlier manifest) adds every group it declares.
|
|
func addsAccountGroups(from catalogue.Manifest, hadFrom bool, to catalogue.Manifest) bool {
|
|
before := map[string]bool{}
|
|
if hadFrom {
|
|
before = accountGroups(from)
|
|
}
|
|
for g := range accountGroups(to) {
|
|
if !before[g] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// kindUsedAsFound is a module's condition while the node-engine uses one of its directories as found (novox/hq
|
|
// issue 339): its own kind, the operator's, never urgent, and never read by the gate as a fault of a build.
|
|
const kindUsedAsFound = "directory-used-as-found"
|
|
|
|
// usedAsFoundKey is a module's used-as-found condition on a machine.
|
|
func usedAsFoundKey(module, node string) string {
|
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindUsedAsFound)
|
|
}
|
|
|
|
// foundWait is whether everything unhealthy of a module on a machine is a directory used as found, and that in
|
|
// one sentence. Anything else unhealthy beside it is judged as a fault, with the directory among its resources.
|
|
func foundWait(module, node string, rs []inventory.ResourceHealth) (string, bool) {
|
|
var ids, why []string
|
|
for _, r := range rs {
|
|
if r.Module != module || r.State != link.StateUnhealthy {
|
|
continue
|
|
}
|
|
if !usedAsFound(r) {
|
|
return "", false
|
|
}
|
|
ids = append(ids, r.Resource)
|
|
why = append(why, strings.TrimSpace(strings.TrimPrefix(r.Reason, link.ReasonUsedAsFound)))
|
|
}
|
|
if len(ids) == 0 {
|
|
return "", false
|
|
}
|
|
return fmt.Sprintf("%s on %s uses %s as found: %s", module, node, strings.Join(ids, ", "),
|
|
strings.Join(why, "; ")), true
|
|
}
|
|
|
|
// usedAsFoundObservation is a module whose directory the node-engine uses as found, in words: the operator's, a
|
|
// warning however long it stays, its summary naming the directories and their owners; the paths are evidence.
|
|
func usedAsFoundObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
o := moduleUnhealthyObservation(module, node, rs)
|
|
o.Token, o.Kind, o.Resolver, o.Severity, o.Summary = kindUsedAsFound, kindUsedAsFound, conditions.ResolverOperator,
|
|
conditions.Warning, said
|
|
o.Headline = fmt.Sprintf("%s waits for a directory on %s", module, node)
|
|
o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+
|
|
"The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.",
|
|
module, node, module, module)
|
|
// Plain words (ADR 0253): the line itself — `nox node hand-over <node> <path>` on the control-node (ADR 0272,
|
|
// issue 356) — is in the summary and the evidence, which name the directory; a path is never in these.
|
|
o.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type."
|
|
o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node)
|
|
o.Actions = nil
|
|
return o
|
|
}
|