jochen
b9fc09c375
The third review of #170 (hq issue 339): a setting overrides any key a provider serves, so any caller of the settings verb could move a database's port, a registry's port or an issuer to a listener of its own and collect what consumers present. TerminalKeys now derives from the manifest: places, accesses, every served key and every setting a served value asks for, and every setting a file marked `trusted` asks for. `trusted` is the catalogue's word, taken out before the declaration; `module check` warns of a file that asks for a setting without saying, and refuses it from 2026-10-30. The hand list is gone. A directory used as found is now its own condition kind, the operator's, never urgent, and the gate exempts it where it exempts a relogin.
685 lines
29 KiB
Go
685 lines
29 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
|
|
// Phase A).
|
|
//
|
|
// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
|
|
// machine states, in every report and as an event between reports, the state of every long-running
|
|
// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
|
|
// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
|
|
// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
|
|
// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
|
|
// the stated health: a judging passes a module only when every long-running resource of it on that
|
|
// machine is stated healthy, so a resource still starting is not yet a pass.
|
|
//
|
|
// **An engine older than the judging states nothing**, and its machine's health is not known: never
|
|
// healthy, never a reason to raise anything, and the gate judges it as it did before.
|
|
|
|
// The condition a module's health raises.
|
|
const (
|
|
kindModuleUnhealthy = "module-unhealthy"
|
|
// sourceHealth is what raised it: the machine's own statement.
|
|
sourceHealth = "health"
|
|
// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
|
|
moduleUnhealthyUrgentAfter = 4 * time.Hour
|
|
// moduleUnhealthyAfter is how many statements in a row raise it.
|
|
moduleUnhealthyAfter = 2
|
|
)
|
|
|
|
// healthRefused counts the statements refused as older than the one kept, for the log and a test.
|
|
var healthRefused atomic.Int64
|
|
|
|
// moduleHealth keeps what the machines state, for the link (link.Healths).
|
|
type moduleHealth struct {
|
|
inv *inventory.Inventory
|
|
keeper func() *conditions.Keeper
|
|
}
|
|
|
|
func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
|
|
return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
|
|
}
|
|
|
|
// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
|
|
// older statement than the one kept is refused, by when the engine looked.
|
|
func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
|
|
now time.Time) error {
|
|
if h.Contract == 0 {
|
|
return nil
|
|
}
|
|
prev, had, err := inv.HealthOf(ctx, node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if had && h.At.Before(prev.SaidAt) {
|
|
healthRefused.Add(1)
|
|
return nil
|
|
}
|
|
unhealthy := map[string][]inventory.ResourceHealth{}
|
|
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
|
for _, r := range h.Resources {
|
|
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
|
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
|
Check: r.Check, Needs: r.Needs, Account: r.Account}
|
|
resources = append(resources, kept)
|
|
if r.State == link.StateUnhealthy && r.Module != "" {
|
|
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
|
}
|
|
}
|
|
streaks := map[string]int{}
|
|
for module := range unhealthy {
|
|
streaks[module] = prev.Streaks[module] + 1
|
|
}
|
|
var network *inventory.NetworkHealth
|
|
if h.Network != nil {
|
|
network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}}
|
|
for _, p := range h.Network.Parts {
|
|
network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
|
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak})
|
|
}
|
|
}
|
|
stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
|
|
HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)})
|
|
if err != nil || !stored {
|
|
if err == nil {
|
|
healthRefused.Add(1)
|
|
}
|
|
return err
|
|
}
|
|
if k == nil {
|
|
return nil
|
|
}
|
|
err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
|
|
// And the machine's own failed units (issue 315): what no module places, one finding for the machine.
|
|
if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil {
|
|
if err == nil {
|
|
err = uerr
|
|
} else {
|
|
err = fmt.Errorf("%w; %v", err, uerr)
|
|
}
|
|
}
|
|
// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
|
|
// machine can hold another's finding, or release it.
|
|
if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
|
|
if err == nil {
|
|
return nerr
|
|
}
|
|
return fmt.Errorf("%w; %v", err, nerr)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
|
|
// resource of it is unhealthy — or on the first while it is already open — and clears every one this
|
|
// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48
|
|
// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone
|
|
// waits on it.
|
|
func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string,
|
|
unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
|
|
open, err := k.Open(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
standing := map[string]conditions.Condition{}
|
|
for _, c := range open {
|
|
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
|
|
c.Subject.Machine == node {
|
|
standing[c.Key] = c
|
|
}
|
|
}
|
|
var hold *holding
|
|
if inv != nil {
|
|
if hold, err = readHolding(ctx, inv, open); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
var problems []string
|
|
modules := make([]string, 0, len(unhealthy))
|
|
for m := range unhealthy {
|
|
modules = append(modules, m)
|
|
}
|
|
sort.Strings(modules)
|
|
seen := map[string]bool{}
|
|
// became is, per module, the kind of condition this statement says of it: what a standing one of the
|
|
// other kind turned into.
|
|
became := map[string]string{}
|
|
heldOn := map[string]string{}
|
|
providers := map[catalogue.Chosen]bool{}
|
|
for _, m := range modules {
|
|
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
|
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
|
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
|
if said, waits := foundWait(m, node, unhealthy[m]); waits {
|
|
o := usedAsFoundObservation(m, node, said, unhealthy[m])
|
|
seen[o.Key()] = true
|
|
became[m] = kindUsedAsFound
|
|
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
|
|
// operator, never urgent, cleared on the first statement that no longer says it.
|
|
if said, waits := personWait(m, node, unhealthy[m]); waits {
|
|
o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
|
|
// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
|
|
// consumers are held under it, and its own condition is where they are said.
|
|
if hold != nil {
|
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
}
|
|
seen[o.Key()] = true
|
|
became[m] = kindReloginNeeded
|
|
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
|
if hold != nil {
|
|
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
|
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
|
heldOn[o.Key()] = p.Module + " on " + p.Node
|
|
providers[p] = true
|
|
continue
|
|
}
|
|
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
|
}
|
|
seen[o.Key()] = true
|
|
became[m] = kindModuleUnhealthy
|
|
c, isOpen := standing[o.Key()]
|
|
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
|
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
|
}
|
|
if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
|
|
o.Severity = conditions.Urgent
|
|
}
|
|
if _, err := k.Observe(ctx, o); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
for key, c := range standing {
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
module := strings.TrimSuffix(c.Subject.ID, "."+node)
|
|
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
|
|
if c.Kind == kindReloginNeeded {
|
|
why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
|
|
}
|
|
if c.Kind == kindUsedAsFound {
|
|
why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
|
|
}
|
|
if on, held := heldOn[key]; held {
|
|
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
|
}
|
|
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
|
// line says what it became.
|
|
resolved := ""
|
|
switch {
|
|
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
|
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
|
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
|
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
|
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
|
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
|
}
|
|
if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
// And each provider a consumer here now waits on, when its own condition is open: said again with who
|
|
// waits on it, so the wait is listed at the provider whichever machine's statement arrived first.
|
|
for p := range providers {
|
|
if err := sayWaiters(ctx, k, hold, p, now); err != nil {
|
|
problems = append(problems, err.Error())
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
|
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
|
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
|
var raisedAt *conditions.Condition
|
|
for i, c := range hold.open {
|
|
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
|
raisedAt = &hold.open[i]
|
|
}
|
|
}
|
|
if raisedAt == nil {
|
|
return nil
|
|
}
|
|
var rs []inventory.ResourceHealth
|
|
for _, r := range hold.healths[p.Node].Resources {
|
|
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
|
rs = append(rs, r)
|
|
}
|
|
}
|
|
if len(rs) == 0 {
|
|
return nil
|
|
}
|
|
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
|
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
|
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
|
}
|
|
if o.Key() != raisedAt.Key {
|
|
return nil // its own statement says it next
|
|
}
|
|
sayWaitingOn(&o, hold.waitersOn(p))
|
|
_, err := k.Observe(ctx, o)
|
|
return err
|
|
}
|
|
|
|
// reloginKey is a module's relogin-needed condition on a machine.
|
|
func reloginKey(module, node string) string {
|
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
|
}
|
|
|
|
// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
|
|
func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
|
|
if inv == nil {
|
|
return ""
|
|
}
|
|
n, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return n.Account
|
|
}
|
|
|
|
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
|
|
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
|
|
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
|
|
// person in again, and a restart of the module's service would start it in the same session. operator is
|
|
// the machine's operator account, when known: the words say "your account" only for it.
|
|
func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
o := moduleUnhealthyObservation(module, node, rs)
|
|
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
|
|
accounts := waitingAccounts(module, rs)
|
|
yours := operator != "" && len(accounts) > 0
|
|
for _, a := range accounts {
|
|
yours = yours && a == operator
|
|
}
|
|
w := reloginWords(module, node, yours)
|
|
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
|
|
return o
|
|
}
|
|
|
|
// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
|
|
// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
|
|
// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
|
|
func reloginWords(module, node string, yours bool) words {
|
|
if yours {
|
|
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
|
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
|
|
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
|
|
"cannot run until you log in again. Its update is in place and nothing was undone.",
|
|
conditions.Capital(module), module),
|
|
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
|
|
}
|
|
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
|
Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
|
|
Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
|
|
"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
|
|
conditions.Capital(module), node, module),
|
|
Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
|
|
}
|
|
|
|
// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
|
|
func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, r := range rs {
|
|
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
|
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
|
|
seen[accountOf(r)] = true
|
|
out = append(out, accountOf(r))
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
|
// waits on it, whether it is not working or waits for a new login.
|
|
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
|
if len(waiters) == 0 {
|
|
return
|
|
}
|
|
o.Severity = conditions.Urgent
|
|
o.Said += "; " + waitingWords(waiters)
|
|
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
|
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
|
}
|
|
|
|
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
|
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
|
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
var words, said, plain []string
|
|
needs := moduleNeeds(node, rs)
|
|
for _, r := range rs {
|
|
plain = append(plain, resourcePlainWords(r))
|
|
if r.Kind == link.KindUnit {
|
|
// A failed unit is named by the unit, which is what a person looks for (issue 315); the
|
|
// resource that places it — a package, a file — is evidence.
|
|
words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason))
|
|
said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource,
|
|
orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
|
continue
|
|
}
|
|
words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
|
|
said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
|
|
r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
|
|
r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
|
|
}
|
|
return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
|
|
Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
|
|
Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
|
|
Said: strings.Join(said, "; "),
|
|
Headline: fmt.Sprintf("%s not working on %s", module, node),
|
|
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
|
namesWords(plain, 3)),
|
|
Needs: needs,
|
|
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
|
}
|
|
|
|
// reasonWords is why a resource is unhealthy, as a person reads it.
|
|
func reasonWords(r inventory.ResourceHealth) string {
|
|
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
|
|
if r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) {
|
|
return fmt.Sprintf("waits for a new login of %s, which is in the group and logged in before it was", accountOf(r))
|
|
}
|
|
switch r.Reason {
|
|
case "restarting":
|
|
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
|
case "down":
|
|
return "is not running"
|
|
case "":
|
|
return "is unhealthy"
|
|
}
|
|
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
|
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
|
if r.Check != "" {
|
|
return "fails its " + r.Check + " check"
|
|
}
|
|
return "is unhealthy: " + r.Reason
|
|
}
|
|
|
|
func orNotSaid(s string) string {
|
|
if s == "" {
|
|
return "no reason said"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
|
|
// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
|
|
const kindReloginNeeded = "relogin-needed"
|
|
|
|
// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
|
|
// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
|
|
// excused:
|
|
//
|
|
// - at least one of the module's resources is its account (kind account), unhealthy with a reason that
|
|
// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
|
|
// running began before;
|
|
// - every other unhealthy resource of the module runs in the own service manager of one of those very
|
|
// accounts (Account names it): the manager that began before the group and does not hold it.
|
|
//
|
|
// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
|
|
// another account's manager, a resource whose engine does not say whose manager it is in, an account
|
|
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
|
|
// resource still starting is not unhealthy and does not make a wait either. Pure.
|
|
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
|
|
accounts := waitingAccounts(module, rs)
|
|
if len(accounts) == 0 {
|
|
return "", false
|
|
}
|
|
waiting := map[string]bool{}
|
|
for _, a := range accounts {
|
|
waiting[a] = true
|
|
}
|
|
var units []string
|
|
for _, r := range rs {
|
|
if r.Module != module || r.State != link.StateUnhealthy {
|
|
continue
|
|
}
|
|
switch {
|
|
case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
|
|
case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
|
|
units = append(units, r.Target)
|
|
default:
|
|
return "", false
|
|
}
|
|
}
|
|
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
|
|
"login began before it was; log out of %[1]s completely and log in again, or restart it", machine, module,
|
|
strings.Join(accounts, ", "))
|
|
if len(units) > 0 {
|
|
sort.Strings(units)
|
|
said += fmt.Sprintf(" (until then %s cannot run)", strings.Join(units, ", "))
|
|
}
|
|
return said, true
|
|
}
|
|
|
|
// accountOf is the account a resource of kind account is: named by the engine, or its target.
|
|
func accountOf(r inventory.ResourceHealth) string {
|
|
if r.Account != "" {
|
|
return r.Account
|
|
}
|
|
return r.Target
|
|
}
|
|
|
|
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
|
|
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
|
|
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
|
|
//
|
|
// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
|
|
// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
|
|
// and not something a machine can meet within a bound.
|
|
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
|
|
if f.healthErr != nil {
|
|
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
|
|
}
|
|
h, states := f.health[machine]
|
|
if !states {
|
|
return healthGood, ""
|
|
}
|
|
if h.HeardAt.Before(since) {
|
|
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
|
}
|
|
wait, waits := personWait(module, machine, h.Resources)
|
|
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
|
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
|
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
|
// have brought a wait, so a later build sent while the login is still owed is judged as before.
|
|
if waits && !f.groupsAdded[module] {
|
|
waits = false
|
|
}
|
|
var found []string
|
|
for _, r := range h.Resources {
|
|
if r.Module != module {
|
|
continue
|
|
}
|
|
if waits && r.State == link.StateUnhealthy {
|
|
continue
|
|
}
|
|
// **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine
|
|
// left its owner and mode, and only someone at the machine can hand it over. It is no fault of this
|
|
// build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict
|
|
// carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy.
|
|
if usedAsFound(r) && r.Since.Before(since) {
|
|
found = append(found, r.Resource)
|
|
continue
|
|
}
|
|
switch r.State {
|
|
case link.StateHealthy:
|
|
case link.StateStarting:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
|
case link.StateUnhealthy:
|
|
if on, held := f.heldOn[module+"@"+machine]; held {
|
|
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
|
r.Resource, machine, on)
|
|
}
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
|
default:
|
|
return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
|
|
reasonAfter(r.Reason))
|
|
}
|
|
}
|
|
if waits || len(found) > 0 {
|
|
var said []string
|
|
if waits {
|
|
said = append(said, wait)
|
|
}
|
|
if len(found) > 0 {
|
|
said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for a person to hand it over "+
|
|
"(`mesh-host hand-over <directory>` at the machine)", machine, module, strings.Join(found, ", ")))
|
|
}
|
|
return healthPerson, strings.Join(said, "; ")
|
|
}
|
|
return healthGood, ""
|
|
}
|
|
|
|
// usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339).
|
|
func usedAsFound(r inventory.ResourceHealth) bool {
|
|
return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound)
|
|
}
|
|
|
|
func reasonAfter(s string) string {
|
|
if s == "" {
|
|
return ""
|
|
}
|
|
return ": " + s
|
|
}
|
|
|
|
// healthLines is what `node show` says of a machine's long-running resources: each with its state and
|
|
// since when, an unhealthy one said once marked unconfirmed.
|
|
func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
|
if !had {
|
|
return []string{" its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
|
|
}
|
|
if len(h.Resources) == 0 {
|
|
return []string{fmt.Sprintf(" it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
|
|
}
|
|
out := []string{fmt.Sprintf(" what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
|
|
for _, r := range h.Resources {
|
|
line := fmt.Sprintf(" %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
|
|
r.Since.Local().Format("2006-01-02 15:04"))
|
|
if r.Reason != "" {
|
|
line += " — " + r.Reason
|
|
}
|
|
if r.Restarts > 0 {
|
|
line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
|
|
}
|
|
if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
|
|
line += " (unconfirmed: said once)"
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// accountGroups is every account group a manifest declares, as "<account>/<group>": a user resource's
|
|
// groups (ADR 0252).
|
|
func accountGroups(m catalogue.Manifest) map[string]bool {
|
|
out := map[string]bool{}
|
|
for _, r := range m.Resources {
|
|
if t, _ := r["type"].(string); t != "user" {
|
|
continue
|
|
}
|
|
name, _ := r["name"].(string)
|
|
groups, _ := r["groups"].([]any)
|
|
for _, g := range groups {
|
|
if group, ok := g.(string); ok && group != "" {
|
|
out[name+"/"+group] = true
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// addsAccountGroups is whether a move from one manifest of a module to another puts an account in a group the
|
|
// earlier one did not (issue 318 review): the only send that can bring a wait for a new login. A module new to
|
|
// the machine (no earlier manifest) adds every group it declares.
|
|
func addsAccountGroups(from catalogue.Manifest, hadFrom bool, to catalogue.Manifest) bool {
|
|
before := map[string]bool{}
|
|
if hadFrom {
|
|
before = accountGroups(from)
|
|
}
|
|
for g := range accountGroups(to) {
|
|
if !before[g] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// kindUsedAsFound is a module's condition while the node-engine uses one of its directories as found (novox/hq
|
|
// issue 339): its own kind, the operator's, never urgent, and never read by the gate as a fault of a build.
|
|
const kindUsedAsFound = "directory-used-as-found"
|
|
|
|
// usedAsFoundKey is a module's used-as-found condition on a machine.
|
|
func usedAsFoundKey(module, node string) string {
|
|
return conditions.Key(conditions.ScopeModule, module+"."+node, kindUsedAsFound)
|
|
}
|
|
|
|
// foundWait is whether everything unhealthy of a module on a machine is a directory used as found, and that in
|
|
// one sentence. Anything else unhealthy beside it is judged as a fault, with the directory among its resources.
|
|
func foundWait(module, node string, rs []inventory.ResourceHealth) (string, bool) {
|
|
var ids, why []string
|
|
for _, r := range rs {
|
|
if r.Module != module || r.State != link.StateUnhealthy {
|
|
continue
|
|
}
|
|
if !usedAsFound(r) {
|
|
return "", false
|
|
}
|
|
ids = append(ids, r.Resource)
|
|
why = append(why, strings.TrimSpace(strings.TrimPrefix(r.Reason, link.ReasonUsedAsFound)))
|
|
}
|
|
if len(ids) == 0 {
|
|
return "", false
|
|
}
|
|
return fmt.Sprintf("%s on %s uses %s as found: %s", module, node, strings.Join(ids, ", "),
|
|
strings.Join(why, "; ")), true
|
|
}
|
|
|
|
// usedAsFoundObservation is a module whose directory the node-engine uses as found, in words: the operator's, a
|
|
// warning however long it stays, its summary naming the directories and their owners; the paths are evidence.
|
|
func usedAsFoundObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
|
|
o := moduleUnhealthyObservation(module, node, rs)
|
|
o.Token, o.Kind, o.Resolver, o.Severity, o.Summary = kindUsedAsFound, kindUsedAsFound, conditions.ResolverOperator,
|
|
conditions.Warning, said
|
|
o.Headline = fmt.Sprintf("%s waits for a directory on %s", module, node)
|
|
o.Explanation = fmt.Sprintf("A directory of %s was already on %s, with another owner or mode than %s declares. "+
|
|
"The mesh left it as it was rather than hand it to an account, so %s may not be able to use it.",
|
|
module, node, module, module)
|
|
o.Needs = fmt.Sprintf("on %s, run mesh-host hand-over with the directory's path as root.", node)
|
|
o.Resolved = fmt.Sprintf("%s's directory on %s is the mesh's", module, node)
|
|
o.Actions = nil
|
|
return o
|
|
}
|