// NodeKeeper keeps the findings only the node-engine knows of itself (design 47 G A-197, ADR 0596, A-149): a have's failed // phase (apply-failing) and its health (module-unhealthy), what it keeps of a module as it runs (kept-as-it-runs), and each // part of its own health report (machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed, // directory-used-as-found, node-account-fallback), under the kind words the watchman raised them under. Each is raised and // cleared where the engine knows the fact, through its finding writer, never read back from the have or the report by // another part. // // The keeper holds the facts it was told (the haves, the last report) and, at every telling, makes the findings of this // node those the facts say: what the writer holds of its kinds for this node and no fact says is cleared, what a fact says // and the writer does not hold, or holds in other words, is written. Nothing runs on a timer: a bound (the apply interval, // the minute before a failing health is said, the four hours before it is urgent) is judged at each telling, and the engine // tells at every pass over its haves and every look at its health. package seatwait import ( "context" "fmt" "maps" "slices" "sort" "strings" "sync" "time" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" ) // The kinds the node-engine raises of itself, the words the watchman raised them under. const ( KindApplyFailing = "apply-failing" KindModuleUnhealthy = "module-unhealthy" KindKeptAsRuns = "kept-as-it-runs" KindMachineNetwork = "machine-network" KindNetworkRewritten = "network-rewritten" KindNetworkUnreachable = "network-unreachable" KindMachineUnits = "machine-units" KindReloginNeeded = "relogin-needed" KindUsedAsFound = "directory-used-as-found" KindAccountFallback = "node-account-fallback" ) // NodeKinds is every kind the keeper writes, for the finding writer's list of kinds. var NodeKinds = []string{KindApplyFailing, KindModuleUnhealthy, KindKeptAsRuns, KindMachineNetwork, KindNetworkRewritten, KindNetworkUnreachable, KindMachineUnits, KindReloginNeeded, KindUsedAsFound, KindAccountFallback} // The bounds the facts are judged by. const ( // ApplyEvery is how long a have may keep saying failed before the node says it does not apply its update. ApplyEvery = 5 * time.Minute // UnhealthyConfirmAfter is how long a have must keep saying its health failed before the module is said unhealthy. UnhealthyConfirmAfter = time.Minute // UnhealthyUrgentAfter is how long a module stays unhealthy before it is urgent. UnhealthyUrgentAfter = 4 * time.Hour // ControlNodeModule is the module whose have names the control node: a failing network there is urgent. ControlNodeModule = "nats" // PackageManagerKey is how a node-engine names a package manager that does not answer among what failed. PackageManagerKey = "the package manager (" ) const ( partResolvConf = "resolv-conf" partNames = "names" partTunnel = "tunnel" partBus = "bus" stateUnhealthy = "unhealthy" phaseFailed = nodeengineclient.PhaseFailed ) // NodeKeeper keeps the findings of one node-engine. Tell it with Have, HaveGone and Report; every telling reconciles. type NodeKeeper struct { Node string Observer string // ObserverOf, when set, answers the observer at each write (see Waiter). ObserverOf func() string Writer Writer // Ctx is the run's context for the writer's acts; nil is the background. Ctx context.Context // Now is the clock; nil is time.Now. Now func() time.Time // Say tells the node's journal; nil says nothing. Say func(string) mu sync.Mutex haves map[string]nodeengine.HaveByNodeModuleValue failedSince map[string]time.Time sickSince map[string]time.Time keptSince map[string]time.Time report *nodeengine.HealthByNodeValue byAddress map[string]string } // Have is a module's have as the node-engine wrote it. func (k *NodeKeeper) Have(module string, h nodeengine.HaveByNodeModuleValue) { k.mu.Lock() defer k.mu.Unlock() if k.haves == nil { k.haves, k.failedSince, k.sickSince, k.keptSince = map[string]nodeengine.HaveByNodeModuleValue{}, map[string]time.Time{}, map[string]time.Time{}, map[string]time.Time{} } if h.Phase == nodeengineclient.PhaseRemoved { k.forget(module) k.reconcile() return } k.haves[module] = h now := k.now() setSince(k.failedSince, module, h.Phase == phaseFailed, now) setSince(k.sickSince, module, h.Health == nodeengineclient.HealthFailed, now) setSince(k.keptSince, module, len(h.Kept) > 0, now) k.reconcile() } // HaveGone is a module no longer held by the node. func (k *NodeKeeper) HaveGone(module string) { k.mu.Lock() defer k.mu.Unlock() k.forget(module) k.reconcile() } // Report is the node's health report as it stands now (written or not), and the machines by the address of each, for the // peers whose tunnel it does not see. Told at every look, so a bound is judged then. func (k *NodeKeeper) Report(h nodeengine.HealthByNodeValue, byAddress map[string]string) { k.mu.Lock() defer k.mu.Unlock() if h.At.IsZero() { h.At = k.now() } k.report, k.byAddress = &h, byAddress k.reconcile() } func (k *NodeKeeper) forget(module string) { delete(k.haves, module) delete(k.failedSince, module) delete(k.sickSince, module) delete(k.keptSince, module) } func setSince(m map[string]time.Time, module string, on bool, now time.Time) { if !on { delete(m, module) } else if _, ok := m[module]; !ok { m[module] = now } } func (k *NodeKeeper) now() time.Time { if k.Now != nil { return k.Now() } return time.Now() } func (k *NodeKeeper) say(s string) { if k.Say != nil { k.Say(s) } } func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") } func firstLine(s string) string { line, _, _ := strings.Cut(s, "\n") return line } func orWords(s, or string) string { if strings.TrimSpace(s) == "" { return or } return s } // ownKey is the identity of a finding the keeper writes. func ownKey(scope, id, kind string) string { return scope + "." + id + "." + kind } // reconcile makes this node's findings of the keeper's kinds those the facts say. The caller holds the lock. func (k *NodeKeeper) reconcile() { ctx := k.Ctx if ctx == nil { ctx = context.Background() } want := map[string]condseat.Finding{} add := func(f condseat.Finding) { want[ownKey(f.Scope, f.ID, f.Kind)] = f } k.haveFindings(add) k.reportFindings(add) held := map[string]condseat.Finding{} for _, f := range k.Writer.Held() { if k.owns(f) { held[ownKey(f.Scope, f.ID, f.Kind)] = f } } for _, key := range slices.Sorted(maps.Keys(held)) { if _, still := want[key]; still { continue } f := held[key] if err := k.Writer.Clear(ctx, f.Scope, f.ID, f.Kind, "the node-engine no longer finds it so"); err != nil { k.say(fmt.Sprintf("the finding %s of %s cannot be cleared: %v", f.Kind, k.Node, err)) } } for _, key := range slices.Sorted(maps.Keys(want)) { f := want[key] if was, ok := held[key]; ok { f.Since, f.At = was.Since, was.At // the same thing seen again writes the same words if sameFinding(was, f) { continue } } if err := k.Writer.Write(ctx, f); err != nil { k.say(fmt.Sprintf("the finding %s of %s cannot be written: %v", f.Kind, k.Node, err)) continue } k.say(f.Summary) } } // owns says a held finding is one the keeper writes for this node. func (k *NodeKeeper) owns(f condseat.Finding) bool { if !slices.Contains(NodeKinds, f.Kind) { return false } if f.Kind == KindNetworkUnreachable { return strings.HasSuffix(f.ID, "."+k.Node+".unreachable") } return f.Subject.Machine == k.Node } func sameFinding(a, b condseat.Finding) bool { if a.Summary != b.Summary || a.Severity != b.Severity || a.Headline != b.Headline || a.Needs != b.Needs || a.Explanation != b.Explanation || !slices.Equal(a.Subject.Also, b.Subject.Also) || len(a.Evidence) != len(b.Evidence) { return false } for i := range a.Evidence { if a.Evidence[i].Said != b.Evidence[i].Said { return false } } return true } // finding is a finding of this node's: observed by the engine's own module, about a machine of the scope. // // The id and ref are the condition's key the watchman raised it under, `..` (A-14): a token other than the kind // goes into the id as `.` and is kept in ref, as the watchman's findings were written. func (k *NodeKeeper) finding(scope, id, kind, ref, machine string, severity string, since time.Time, summary, said string) condseat.Finding { since = since.UTC() if ref != "" { id += "." + ref } f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: kind, Scope: scope, ID: id, Severity: severity, Ref: ref, Source: kind, Summary: summary, At: since, Since: since, Subject: condseat.Subject{Scope: scope, ID: id, Machine: machine}} if said != "" { f.Evidence = []condseat.Evidence{{At: since, Said: said}} } return f } // haveFindings are the findings the haves say: apply-failing, module-unhealthy and kept-as-it-runs. func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) { now := k.now() type failure struct{ module, words string } var failed []failure var since time.Time for _, m := range slices.Sorted(maps.Keys(k.failedSince)) { failed = append(failed, failure{m, firstLine(orWords(k.haves[m].Detail, "no reason given"))}) if at := k.failedSince[m]; since.IsZero() || at.Before(since) { since = at } } if len(failed) > 0 && now.Sub(since) > ApplyEvery { var names, words []string tool := "" for _, f := range failed { names = append(names, f.module) words = append(words, f.module+": "+f.words) w := strings.TrimPrefix(f.words, "refused: ") if i := strings.Index(w, PackageManagerKey); i >= 0 && tool == "" { tool = strings.TrimSuffix(strings.SplitN(w[i+len(PackageManagerKey):], ")", 2)[0], ")") } } f := k.finding(scopeMachine, k.Node, KindApplyFailing, "", k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; ")) f.Headline = k.Node + " is not applying its update" if tool != "" { f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, since %s (bound %s)", k.Node, tool, stampMin(since), ApplyEvery) f.Headline = k.Node + " cannot install packages" } else { f.Summary = fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", k.Node, strings.Join(names, ", "), stampMin(since), ApplyEvery) f.Explanation = fmt.Sprintf("%s has failed to apply its updates, and trying again has not helped. The details say what fails.", k.Node) } add(f) } for _, m := range slices.Sorted(maps.Keys(k.sickSince)) { at := k.sickSince[m] if now.Sub(at) < UnhealthyConfirmAfter { continue } severity := condseat.FindingSeverityWarning if now.Sub(at) >= UnhealthyUrgentAfter { severity = condseat.FindingSeverityUrgent } h := k.haves[m] f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, "unhealthy", k.Node, severity, at, fmt.Sprintf("%s on %s is not healthy: %s", m, k.Node, orWords(h.Detail, "its check failed")), firstLine(orWords(h.Detail, "failed"))) f.Headline = fmt.Sprintf("%s not working on %s", m, k.Node) add(f) } for _, m := range slices.Sorted(maps.Keys(k.keptSince)) { words := strings.Join(k.haves[m].Kept, "; ") f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, "kept", k.Node, condseat.FindingSeverityWarning, k.keptSince[m], fmt.Sprintf("%s on %s is kept as it runs: %s", m, k.Node, words), firstLine(words)) f.Headline = fmt.Sprintf("%s kept as it runs on %s", m, k.Node) f.Needs = "a build of " + m + " whose manifest says it (tool-runner, core-credentials)" add(f) } } func stampMin(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") } // control says the node runs the module the control node is named by. func (k *NodeKeeper) control() bool { h, ok := k.haves[ControlNodeModule] return ok && h.Phase != nodeengineclient.PhaseRemoved } // reportFindings are the findings the node's health report says. func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) { h := k.report if h == nil { return } node := k.Node var own []nodeengine.NetworkPart var rewritten *nodeengine.NetworkPart var parts []nodeengine.NetworkPart if h.Network != nil { for _, p := range h.Network.Parts { if p.State == stateUnhealthy { parts = append(parts, p) } } } toward := map[string][]nodeengine.NetworkPart{} for i, p := range parts { switch { case p.Part == partResolvConf: rewritten = &parts[i] case len(p.Targets) > 0: for _, x := range p.Targets { toward[x] = append(toward[x], p) } default: own = append(own, p) } } if rewritten != nil { add(k.rewritten(*rewritten, parts)) kept := own[:0] for _, p := range own { if p.Part != partNames { kept = append(kept, p) } } own = kept } if len(own) > 0 { var words, said []string severity := condseat.FindingSeverityWarning for _, p := range own { words = append(words, p.Reason) said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said)) if p.Part == partBus { severity = condseat.FindingSeverityUrgent } } if k.control() { severity = condseat.FindingSeverityUrgent } add(k.finding(scopeMachine, node, KindMachineNetwork, "network", node, severity, h.Network.Since, fmt.Sprintf("%s's network is not healthy: %s", node, strings.Join(words, "; ")), strings.Join(said, " | "))) } // A tunnel peer this node does not see points at that peer's machine as a tunnel part toward it would. for _, p := range h.Peers { if p.Seen { continue } for _, a := range p.Addresses { addr, _, _ := strings.Cut(a, "/") if x := k.byAddress[addr]; x != "" && x != node { toward[x] = append(toward[x], nodeengine.NetworkPart{Part: partTunnel, State: stateUnhealthy, Since: h.At, Said: "no handshake with " + x + " within five minutes", Targets: []string{x}}) } } } for _, x := range slices.Sorted(maps.Keys(toward)) { var what, said []string since := toward[x][0].Since severity := condseat.FindingSeverityWarning for _, p := range toward[x] { wd := map[string]string{partTunnel: "the tunnel to it", partBus: "the bus on it", partNames: "its resolver"}[p.Part] if wd == "" { wd = p.Part } if !slices.Contains(what, wd) { what = append(what, wd) } said = append(said, fmt.Sprintf("%s: %s: %s", node, p.Part, p.Said)) if p.Since.Before(since) { since = p.Since } if p.Part == partBus { severity = condseat.FindingSeverityUrgent } } f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, "unreachable", x, severity, since, fmt.Sprintf("%s cannot be reached from %s: %s", x, node, strings.Join(what, ", ")), strings.Join(said, " | ")) f.Subject.Also = []string{node} add(f) } if f, ok := k.units(h.Units); ok { add(f) } for _, f := range k.relogin(h.Relogin) { add(f) } for _, f := range k.found(h.Found) { add(f) } if f := h.Fallback; f != nil && f.Node == node { said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why) g := k.finding(scopeMachine, node, KindAccountFallback, "", node, condseat.FindingSeverityWarning, f.At, said, said) g.Headline = fmt.Sprintf("%s fell back to its old bus login", node) g.Needs = fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node) add(g) } } func (k *NodeKeeper) rewritten(p nodeengine.NetworkPart, all []nodeengine.NetworkPart) condseat.Finding { m := k.Node writer := "" if p.Writer != "" { writer = " (" + p.Writer + ")" } cost := "the names still resolve through what it wrote" said := []string{p.Part + ": " + p.Said} for _, q := range all { if q.Part == partNames { cost = strings.TrimSuffix(q.Reason, ".") said = append(said, q.Part+": "+q.Said) } } id := m summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+ "back at its next reconcile, or that program gives it back", m, writer, cost) if p.Owner != "" { id = m + "." + p.Owner summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+ "writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) } severity := condseat.FindingSeverityWarning if k.control() { severity = condseat.FindingSeverityUrgent } return k.finding(scopeMachine, id, KindNetworkRewritten, "rewritten", m, severity, p.Since, summary, fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))) } func (k *NodeKeeper) units(u *nodeengine.UnitsHealth) (condseat.Finding, bool) { if u == nil || len(u.Failed) == 0 { return condseat.Finding{}, false } node := k.Node var names, said []string since := u.Failed[0].Since for _, f := range u.Failed { where := "" if f.Scope == "user" { where = " (the account's own manager)" } names = append(names, f.Unit+where) scope, result := f.Scope, f.Result if scope == "" { scope = "system" } if result == "" { result = "no reason said" } line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result) if f.Load != "" && f.Load != "loaded" { line += ", its unit " + f.Load } if f.Resource != "" { line += ", named by the mesh's own " + f.Resource } said = append(said, line+", since "+stampSec(f.Since)) if f.Since.Before(since) { since = f.Since } } failed := "1 failed service" if len(u.Failed) > 1 { failed = fmt.Sprintf("%d failed services", len(u.Failed)) } f := k.finding(scopeMachine, node, KindMachineUnits, "units", node, condseat.FindingSeverityWarning, since, fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+ "mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")), strings.Join(said, "; ")) f.Headline = strings.ToUpper(failed[:1]) + failed[1:] + " on " + node f.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node) return f, true } func orModule(m string) string { if m == "" { return "_" } return m } func (k *NodeKeeper) relogin(accounts []nodeengine.NodeAccount) []condseat.Finding { by := map[string][]nodeengine.NodeAccount{} for _, a := range accounts { by[a.Module] = append(by[a.Module], a) } var out []condseat.Finding for _, module := range slices.Sorted(maps.Keys(by)) { var names []string since := by[module][0].Since for _, a := range by[module] { names = append(names, a.Account) if a.Since.Before(since) { since = a.Since } } sort.Strings(names) said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+ "before it was; log out of %[1]s completely and log in again, or restart it", k.Node, orModule(module), strings.Join(names, ", ")) out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, "", k.Node, condseat.FindingSeverityWarning, since, said, said)) } return out } func (k *NodeKeeper) found(dirs []nodeengine.NodeDirectory) []condseat.Finding { by := map[string][]nodeengine.NodeDirectory{} for _, d := range dirs { by[d.Module] = append(by[d.Module], d) } var out []condseat.Finding for _, module := range slices.Sorted(maps.Keys(by)) { var ids, why []string since := by[module][0].Since for _, d := range by[module] { ids = append(ids, d.Resource) why = append(why, d.Reason) if d.Since.Before(since) { since = d.Since } } m := orModule(module) said := fmt.Sprintf("%s on %s uses %s as found: %s", m, k.Node, strings.Join(ids, ", "), strings.Join(why, "; ")) f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, "", k.Node, condseat.FindingSeverityWarning, since, said, said) f.Headline = fmt.Sprintf("%s waits for a directory on %s", m, k.Node) f.Explanation = fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+ "and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, k.Node, m) f.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type." out = append(out, f) } return out }