Files
mesh-host/internal/seatwait/nodefindings.go
T
jschoubben a3fea9106a H-find: the findings keep the keys the watchman raised them under
Each finding's id carries its token as a ref (A-14), so machine.<node>.network, .rewritten, .unreachable, .units and module.<module>.<node>.unhealthy and .kept
read as before.

Builds: design 47 G A-197

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 06:02:07 +01:00

575 lines
20 KiB
Go

// NodeKeeper keeps the findings only the node-engine knows of itself (design 47 G A-197, ADR 0596, A-149): a have's failed
// phase (apply-failing) and its health (module-unhealthy), what it keeps of a module as it runs (kept-as-it-runs), and each
// part of its own health report (machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed,
// directory-used-as-found, node-account-fallback), under the kind words the watchman raised them under. Each is raised and
// cleared where the engine knows the fact, through its finding writer, never read back from the have or the report by
// another part.
//
// The keeper holds the facts it was told (the haves, the last report) and, at every telling, makes the findings of this
// node those the facts say: what the writer holds of its kinds for this node and no fact says is cleared, what a fact says
// and the writer does not hold, or holds in other words, is written. Nothing runs on a timer: a bound (the apply interval,
// the minute before a failing health is said, the four hours before it is urgent) is judged at each telling, and the engine
// tells at every pass over its haves and every look at its health.
package seatwait
import (
"context"
"fmt"
"maps"
"slices"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// The kinds the node-engine raises of itself, the words the watchman raised them under.
const (
KindApplyFailing = "apply-failing"
KindModuleUnhealthy = "module-unhealthy"
KindKeptAsRuns = "kept-as-it-runs"
KindMachineNetwork = "machine-network"
KindNetworkRewritten = "network-rewritten"
KindNetworkUnreachable = "network-unreachable"
KindMachineUnits = "machine-units"
KindReloginNeeded = "relogin-needed"
KindUsedAsFound = "directory-used-as-found"
KindAccountFallback = "node-account-fallback"
)
// NodeKinds is every kind the keeper writes, for the finding writer's list of kinds.
var NodeKinds = []string{KindApplyFailing, KindModuleUnhealthy, KindKeptAsRuns, KindMachineNetwork, KindNetworkRewritten,
KindNetworkUnreachable, KindMachineUnits, KindReloginNeeded, KindUsedAsFound, KindAccountFallback}
// The bounds the facts are judged by.
const (
// ApplyEvery is how long a have may keep saying failed before the node says it does not apply its update.
ApplyEvery = 5 * time.Minute
// UnhealthyConfirmAfter is how long a have must keep saying its health failed before the module is said unhealthy.
UnhealthyConfirmAfter = time.Minute
// UnhealthyUrgentAfter is how long a module stays unhealthy before it is urgent.
UnhealthyUrgentAfter = 4 * time.Hour
// ControlNodeModule is the module whose have names the control node: a failing network there is urgent.
ControlNodeModule = "nats"
// PackageManagerKey is how a node-engine names a package manager that does not answer among what failed.
PackageManagerKey = "the package manager ("
)
const (
partResolvConf = "resolv-conf"
partNames = "names"
partTunnel = "tunnel"
partBus = "bus"
stateUnhealthy = "unhealthy"
phaseFailed = nodeengineclient.PhaseFailed
)
// NodeKeeper keeps the findings of one node-engine. Tell it with Have, HaveGone and Report; every telling reconciles.
type NodeKeeper struct {
Node string
Observer string
// ObserverOf, when set, answers the observer at each write (see Waiter).
ObserverOf func() string
Writer Writer
// Ctx is the run's context for the writer's acts; nil is the background.
Ctx context.Context
// Now is the clock; nil is time.Now.
Now func() time.Time
// Say tells the node's journal; nil says nothing.
Say func(string)
mu sync.Mutex
haves map[string]nodeengine.HaveByNodeModuleValue
failedSince map[string]time.Time
sickSince map[string]time.Time
keptSince map[string]time.Time
report *nodeengine.HealthByNodeValue
byAddress map[string]string
}
// Have is a module's have as the node-engine wrote it.
func (k *NodeKeeper) Have(module string, h nodeengine.HaveByNodeModuleValue) {
k.mu.Lock()
defer k.mu.Unlock()
if k.haves == nil {
k.haves, k.failedSince, k.sickSince, k.keptSince = map[string]nodeengine.HaveByNodeModuleValue{}, map[string]time.Time{},
map[string]time.Time{}, map[string]time.Time{}
}
if h.Phase == nodeengineclient.PhaseRemoved {
k.forget(module)
k.reconcile()
return
}
k.haves[module] = h
now := k.now()
setSince(k.failedSince, module, h.Phase == phaseFailed, now)
setSince(k.sickSince, module, h.Health == nodeengineclient.HealthFailed, now)
setSince(k.keptSince, module, len(h.Kept) > 0, now)
k.reconcile()
}
// HaveGone is a module no longer held by the node.
func (k *NodeKeeper) HaveGone(module string) {
k.mu.Lock()
defer k.mu.Unlock()
k.forget(module)
k.reconcile()
}
// Report is the node's health report as it stands now (written or not), and the machines by the address of each, for the
// peers whose tunnel it does not see. Told at every look, so a bound is judged then.
func (k *NodeKeeper) Report(h nodeengine.HealthByNodeValue, byAddress map[string]string) {
k.mu.Lock()
defer k.mu.Unlock()
if h.At.IsZero() {
h.At = k.now()
}
k.report, k.byAddress = &h, byAddress
k.reconcile()
}
func (k *NodeKeeper) forget(module string) {
delete(k.haves, module)
delete(k.failedSince, module)
delete(k.sickSince, module)
delete(k.keptSince, module)
}
func setSince(m map[string]time.Time, module string, on bool, now time.Time) {
if !on {
delete(m, module)
} else if _, ok := m[module]; !ok {
m[module] = now
}
}
func (k *NodeKeeper) now() time.Time {
if k.Now != nil {
return k.Now()
}
return time.Now()
}
func (k *NodeKeeper) say(s string) {
if k.Say != nil {
k.Say(s)
}
}
func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") }
func firstLine(s string) string {
line, _, _ := strings.Cut(s, "\n")
return line
}
func orWords(s, or string) string {
if strings.TrimSpace(s) == "" {
return or
}
return s
}
// ownKey is the identity of a finding the keeper writes.
func ownKey(scope, id, kind string) string { return scope + "." + id + "." + kind }
// reconcile makes this node's findings of the keeper's kinds those the facts say. The caller holds the lock.
func (k *NodeKeeper) reconcile() {
ctx := k.Ctx
if ctx == nil {
ctx = context.Background()
}
want := map[string]condseat.Finding{}
add := func(f condseat.Finding) { want[ownKey(f.Scope, f.ID, f.Kind)] = f }
k.haveFindings(add)
k.reportFindings(add)
held := map[string]condseat.Finding{}
for _, f := range k.Writer.Held() {
if k.owns(f) {
held[ownKey(f.Scope, f.ID, f.Kind)] = f
}
}
for _, key := range slices.Sorted(maps.Keys(held)) {
if _, still := want[key]; still {
continue
}
f := held[key]
if err := k.Writer.Clear(ctx, f.Scope, f.ID, f.Kind, "the node-engine no longer finds it so"); err != nil {
k.say(fmt.Sprintf("the finding %s of %s cannot be cleared: %v", f.Kind, k.Node, err))
}
}
for _, key := range slices.Sorted(maps.Keys(want)) {
f := want[key]
if was, ok := held[key]; ok {
f.Since, f.At = was.Since, was.At // the same thing seen again writes the same words
if sameFinding(was, f) {
continue
}
}
if err := k.Writer.Write(ctx, f); err != nil {
k.say(fmt.Sprintf("the finding %s of %s cannot be written: %v", f.Kind, k.Node, err))
continue
}
k.say(f.Summary)
}
}
// owns says a held finding is one the keeper writes for this node.
func (k *NodeKeeper) owns(f condseat.Finding) bool {
if !slices.Contains(NodeKinds, f.Kind) {
return false
}
if f.Kind == KindNetworkUnreachable {
return strings.HasSuffix(f.ID, "."+k.Node+".unreachable")
}
return f.Subject.Machine == k.Node
}
func sameFinding(a, b condseat.Finding) bool {
if a.Summary != b.Summary || a.Severity != b.Severity || a.Headline != b.Headline || a.Needs != b.Needs ||
a.Explanation != b.Explanation || !slices.Equal(a.Subject.Also, b.Subject.Also) || len(a.Evidence) != len(b.Evidence) {
return false
}
for i := range a.Evidence {
if a.Evidence[i].Said != b.Evidence[i].Said {
return false
}
}
return true
}
// finding is a finding of this node's: observed by the engine's own module, about a machine of the scope.
//
// The id and ref are the condition's key the watchman raised it under, `<scope>.<id>.<token>` (A-14): a token other than the kind
// goes into the id as `<id>.<ref>` and is kept in ref, as the watchman's findings were written.
func (k *NodeKeeper) finding(scope, id, kind, ref, machine string, severity string, since time.Time, summary, said string) condseat.Finding {
since = since.UTC()
if ref != "" {
id += "." + ref
}
f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: kind, Scope: scope, ID: id, Severity: severity,
Ref: ref, Source: kind, Summary: summary, At: since, Since: since,
Subject: condseat.Subject{Scope: scope, ID: id, Machine: machine}}
if said != "" {
f.Evidence = []condseat.Evidence{{At: since, Said: said}}
}
return f
}
// haveFindings are the findings the haves say: apply-failing, module-unhealthy and kept-as-it-runs.
func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) {
now := k.now()
type failure struct{ module, words string }
var failed []failure
var since time.Time
for _, m := range slices.Sorted(maps.Keys(k.failedSince)) {
failed = append(failed, failure{m, firstLine(orWords(k.haves[m].Detail, "no reason given"))})
if at := k.failedSince[m]; since.IsZero() || at.Before(since) {
since = at
}
}
if len(failed) > 0 && now.Sub(since) > ApplyEvery {
var names, words []string
tool := ""
for _, f := range failed {
names = append(names, f.module)
words = append(words, f.module+": "+f.words)
w := strings.TrimPrefix(f.words, "refused: ")
if i := strings.Index(w, PackageManagerKey); i >= 0 && tool == "" {
tool = strings.TrimSuffix(strings.SplitN(w[i+len(PackageManagerKey):], ")", 2)[0], ")")
}
}
f := k.finding(scopeMachine, k.Node, KindApplyFailing, "", k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; "))
f.Headline = k.Node + " is not applying its update"
if tool != "" {
f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, since %s (bound %s)",
k.Node, tool, stampMin(since), ApplyEvery)
f.Headline = k.Node + " cannot install packages"
} else {
f.Summary = fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", k.Node, strings.Join(names, ", "), stampMin(since), ApplyEvery)
f.Explanation = fmt.Sprintf("%s has failed to apply its updates, and trying again has not helped. The details say what fails.", k.Node)
}
add(f)
}
for _, m := range slices.Sorted(maps.Keys(k.sickSince)) {
at := k.sickSince[m]
if now.Sub(at) < UnhealthyConfirmAfter {
continue
}
severity := condseat.FindingSeverityWarning
if now.Sub(at) >= UnhealthyUrgentAfter {
severity = condseat.FindingSeverityUrgent
}
h := k.haves[m]
f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, "unhealthy", k.Node, severity, at,
fmt.Sprintf("%s on %s is not healthy: %s", m, k.Node, orWords(h.Detail, "its check failed")), firstLine(orWords(h.Detail, "failed")))
f.Headline = fmt.Sprintf("%s not working on %s", m, k.Node)
add(f)
}
for _, m := range slices.Sorted(maps.Keys(k.keptSince)) {
words := strings.Join(k.haves[m].Kept, "; ")
f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, "kept", k.Node, condseat.FindingSeverityWarning, k.keptSince[m],
fmt.Sprintf("%s on %s is kept as it runs: %s", m, k.Node, words), firstLine(words))
f.Headline = fmt.Sprintf("%s kept as it runs on %s", m, k.Node)
f.Needs = "a build of " + m + " whose manifest says it (tool-runner, core-credentials)"
add(f)
}
}
func stampMin(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") }
// control says the node runs the module the control node is named by.
func (k *NodeKeeper) control() bool {
h, ok := k.haves[ControlNodeModule]
return ok && h.Phase != nodeengineclient.PhaseRemoved
}
// reportFindings are the findings the node's health report says.
func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) {
h := k.report
if h == nil {
return
}
node := k.Node
var own []nodeengine.NetworkPart
var rewritten *nodeengine.NetworkPart
var parts []nodeengine.NetworkPart
if h.Network != nil {
for _, p := range h.Network.Parts {
if p.State == stateUnhealthy {
parts = append(parts, p)
}
}
}
toward := map[string][]nodeengine.NetworkPart{}
for i, p := range parts {
switch {
case p.Part == partResolvConf:
rewritten = &parts[i]
case len(p.Targets) > 0:
for _, x := range p.Targets {
toward[x] = append(toward[x], p)
}
default:
own = append(own, p)
}
}
if rewritten != nil {
add(k.rewritten(*rewritten, parts))
kept := own[:0]
for _, p := range own {
if p.Part != partNames {
kept = append(kept, p)
}
}
own = kept
}
if len(own) > 0 {
var words, said []string
severity := condseat.FindingSeverityWarning
for _, p := range own {
words = append(words, p.Reason)
said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said))
if p.Part == partBus {
severity = condseat.FindingSeverityUrgent
}
}
if k.control() {
severity = condseat.FindingSeverityUrgent
}
add(k.finding(scopeMachine, node, KindMachineNetwork, "network", node, severity, h.Network.Since,
fmt.Sprintf("%s's network is not healthy: %s", node, strings.Join(words, "; ")), strings.Join(said, " | ")))
}
// A tunnel peer this node does not see points at that peer's machine as a tunnel part toward it would.
for _, p := range h.Peers {
if p.Seen {
continue
}
for _, a := range p.Addresses {
addr, _, _ := strings.Cut(a, "/")
if x := k.byAddress[addr]; x != "" && x != node {
toward[x] = append(toward[x], nodeengine.NetworkPart{Part: partTunnel, State: stateUnhealthy, Since: h.At,
Said: "no handshake with " + x + " within five minutes", Targets: []string{x}})
}
}
}
for _, x := range slices.Sorted(maps.Keys(toward)) {
var what, said []string
since := toward[x][0].Since
severity := condseat.FindingSeverityWarning
for _, p := range toward[x] {
wd := map[string]string{partTunnel: "the tunnel to it", partBus: "the bus on it", partNames: "its resolver"}[p.Part]
if wd == "" {
wd = p.Part
}
if !slices.Contains(what, wd) {
what = append(what, wd)
}
said = append(said, fmt.Sprintf("%s: %s: %s", node, p.Part, p.Said))
if p.Since.Before(since) {
since = p.Since
}
if p.Part == partBus {
severity = condseat.FindingSeverityUrgent
}
}
f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, "unreachable", x, severity, since,
fmt.Sprintf("%s cannot be reached from %s: %s", x, node, strings.Join(what, ", ")), strings.Join(said, " | "))
f.Subject.Also = []string{node}
add(f)
}
if f, ok := k.units(h.Units); ok {
add(f)
}
for _, f := range k.relogin(h.Relogin) {
add(f)
}
for _, f := range k.found(h.Found) {
add(f)
}
if f := h.Fallback; f != nil && f.Node == node {
said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why)
g := k.finding(scopeMachine, node, KindAccountFallback, "", node, condseat.FindingSeverityWarning, f.At, said, said)
g.Headline = fmt.Sprintf("%s fell back to its old bus login", node)
g.Needs = fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node)
add(g)
}
}
func (k *NodeKeeper) rewritten(p nodeengine.NetworkPart, all []nodeengine.NetworkPart) condseat.Finding {
m := k.Node
writer := ""
if p.Writer != "" {
writer = " (" + p.Writer + ")"
}
cost := "the names still resolve through what it wrote"
said := []string{p.Part + ": " + p.Said}
for _, q := range all {
if q.Part == partNames {
cost = strings.TrimSuffix(q.Reason, ".")
said = append(said, q.Part+": "+q.Said)
}
}
id := m
summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+
"back at its next reconcile, or that program gives it back", m, writer, cost)
if p.Owner != "" {
id = m + "." + p.Owner
summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+
"writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
}
severity := condseat.FindingSeverityWarning
if k.control() {
severity = condseat.FindingSeverityUrgent
}
return k.finding(scopeMachine, id, KindNetworkRewritten, "rewritten", m, severity, p.Since, summary,
fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | ")))
}
func (k *NodeKeeper) units(u *nodeengine.UnitsHealth) (condseat.Finding, bool) {
if u == nil || len(u.Failed) == 0 {
return condseat.Finding{}, false
}
node := k.Node
var names, said []string
since := u.Failed[0].Since
for _, f := range u.Failed {
where := ""
if f.Scope == "user" {
where = " (the account's own manager)"
}
names = append(names, f.Unit+where)
scope, result := f.Scope, f.Result
if scope == "" {
scope = "system"
}
if result == "" {
result = "no reason said"
}
line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result)
if f.Load != "" && f.Load != "loaded" {
line += ", its unit " + f.Load
}
if f.Resource != "" {
line += ", named by the mesh's own " + f.Resource
}
said = append(said, line+", since "+stampSec(f.Since))
if f.Since.Before(since) {
since = f.Since
}
}
failed := "1 failed service"
if len(u.Failed) > 1 {
failed = fmt.Sprintf("%d failed services", len(u.Failed))
}
f := k.finding(scopeMachine, node, KindMachineUnits, "units", node, condseat.FindingSeverityWarning, since,
fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+
"mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")),
strings.Join(said, "; "))
f.Headline = strings.ToUpper(failed[:1]) + failed[1:] + " on " + node
f.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)
return f, true
}
func orModule(m string) string {
if m == "" {
return "_"
}
return m
}
func (k *NodeKeeper) relogin(accounts []nodeengine.NodeAccount) []condseat.Finding {
by := map[string][]nodeengine.NodeAccount{}
for _, a := range accounts {
by[a.Module] = append(by[a.Module], a)
}
var out []condseat.Finding
for _, module := range slices.Sorted(maps.Keys(by)) {
var names []string
since := by[module][0].Since
for _, a := range by[module] {
names = append(names, a.Account)
if a.Since.Before(since) {
since = a.Since
}
}
sort.Strings(names)
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+
"before it was; log out of %[1]s completely and log in again, or restart it", k.Node, orModule(module), strings.Join(names, ", "))
out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, "", k.Node, condseat.FindingSeverityWarning, since, said, said))
}
return out
}
func (k *NodeKeeper) found(dirs []nodeengine.NodeDirectory) []condseat.Finding {
by := map[string][]nodeengine.NodeDirectory{}
for _, d := range dirs {
by[d.Module] = append(by[d.Module], d)
}
var out []condseat.Finding
for _, module := range slices.Sorted(maps.Keys(by)) {
var ids, why []string
since := by[module][0].Since
for _, d := range by[module] {
ids = append(ids, d.Resource)
why = append(why, d.Reason)
if d.Since.Before(since) {
since = d.Since
}
}
m := orModule(module)
said := fmt.Sprintf("%s on %s uses %s as found: %s", m, k.Node, strings.Join(ids, ", "), strings.Join(why, "; "))
f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, "", k.Node, condseat.FindingSeverityWarning, since, said, said)
f.Headline = fmt.Sprintf("%s waits for a directory on %s", m, k.Node)
f.Explanation = fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+
"and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, k.Node, m)
f.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type."
out = append(out, f)
}
return out
}