From 4b215ba6aee42e3ee0644ddd2d8bf75b33637039 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 06:00:42 +0100 Subject: [PATCH] H-find: the node-engine raises what only it knows of itself as its own findings The have's failed phase (apply-failing), unhealthy health (module-unhealthy) and kept words (kept-as-it-runs), and each part of its health report (machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed, directory-used-as-found, node-account-fallback), raised and cleared by the held writer where the engine knows the fact. Builds: design 47 G A-197 (ADR 0596, A-192, A-149) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/main.go | 10 +- cmd/mesh-host/nodeagent.go | 11 + cmd/mesh-host/seatlist.go | 2 +- internal/link/have.go | 26 ++ internal/link/have_test.go | 11 + internal/link/session.go | 4 + internal/seatwait/nodefindings.go | 568 +++++++++++++++++++++++++ internal/seatwait/nodefindings_test.go | 104 +++++ 8 files changed, 734 insertions(+), 2 deletions(-) create mode 100644 internal/seatwait/nodefindings.go create mode 100644 internal/seatwait/nodefindings_test.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index eb47ba29..863ced65 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1402,7 +1402,15 @@ func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, if c := machinesNow.Load(); c != nil { machines.ByAddress, machines.Hub = c.Machines() } - _, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers, machines)) + report := link.NodeHealthOf(queue.Membership.Node, h, peers, machines) + if queue.Fallback != nil && report.Fallback == nil { + report.Fallback = queue.Fallback() // node-account-fallback (ADR 0583 §6), as the written report carries it + } + _, err := queue.WriteNodeHealth(ctx, report) + // The findings the report says are raised by the engine here, at every look, written to the bus or not (design 47 G A-197). + if keeper := ownFindingsNow.Load(); keeper != nil { + keeper.Report(report, machines.ByAddress) + } switch { case err != nil && err.Error() != *unwritten: *unwritten = err.Error() diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index d81de981..09d85816 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -616,6 +616,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus // The marks of the haves run off the composition pass, each bounded (see marker). marks := newMarker(run) heldFindings := &seatwait.HeldKeeper{Node: node, ObserverOf: findings.observer, Writer: findings, Ctx: run, Say: say} + // And what only this engine knows of itself, raised where it knows it (design 47 G A-197): its haves' failed phase, health + // and kept words as they are written, and each part of its health report at every look. + ownFindings := &seatwait.NodeKeeper{Node: node, ObserverOf: findings.observer, Writer: findings, Ctx: run, Say: say} + session.SetHaveSaid(ownFindings.Have, ownFindings.HaveGone) + ownFindingsNow.Store(ownFindings) hooks := composerHooks{ entryNotFound: func(fp string, notFound bool) { if err := seatwait.SayEntryNotFound(run, findings, node, findings.observer(), fp, notFound, time.Now); err != nil { @@ -707,6 +712,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus cancel() current.set(nil) composerNow.Store(nil) + ownFindingsNow.Store(nil) + session.SetHaveSaid(nil, nil) session.SetManifestOf(nil) apply.UseEngineModule(nil) shadowNow.Store(nil) @@ -909,6 +916,10 @@ func busyModules() []string { // (writeHealthReport); nil on a node that does not compose its own, whose parts then name no target. var machinesNow atomic.Pointer[selfcompose.Composer] +// ownFindingsNow is the keeper of the findings this engine raises of itself in the run that composes the node's own now, for the +// health loop (writeHealthReport); nil while no run holds one. +var ownFindingsNow atomic.Pointer[seatwait.NodeKeeper] + // composerNow is the composer of the run that composes the node's own now, nil while the node applies the controller's // compositions or no run is going: what the verb compose asks a pass of (issue 774). var composerNow atomic.Pointer[selfcompose.Composer] diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index 9eb2aa56..c23ceef8 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -124,7 +124,7 @@ func newFindingsOfTheNode(session *link.Session, node, module string, say func(s Module: module, Node: node, State: link.FindingsState{Session: session}, - Kinds: []string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, + Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, seatwait.NodeKinds...), Asked: condseat.SubjectFindingsAsked, Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) }, Publish: session.PublishEvent, diff --git a/internal/link/have.go b/internal/link/have.go index f1a5a696..75e31972 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -440,6 +440,9 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { s.written = map[string]string{} } s.written[module] = f.Build + if say, _ := q.haveSaid(); say != nil { + said = append(said, func() { say(module, h) }) + } if ev, ok := s.runningToSay(module, h, q.manifestOf()); ok { said = append(said, func() { q.sayRunning(ev) }) } @@ -555,6 +558,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string, s switch { case errors.Is(err, ErrHaveAbsent): delete(s.removed, module) // nothing to say: the key is not there + if _, gone := q.haveSaid(); gone != nil { + *said = append(*said, func() { gone(module) }) + } continue case err != nil: q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing more there: %v", key, err)) @@ -563,6 +569,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string, s } if c, err := nodeengineclient.DecodeHave(current); err == nil && c.Phase == nodeengineclient.PhaseRemoved { delete(s.removed, module) + if _, gone := q.haveSaid(); gone != nil { + *said = append(*said, func() { gone(module) }) + } continue } if err := b.Set(ctx, key, raw, revision); err != nil { @@ -571,6 +580,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string, s return false } delete(s.removed, module) + if _, gone := q.haveSaid(); gone != nil { + *said = append(*said, func() { gone(module) }) + } // The module no longer runs here: running says stopped, with no fingerprint, on its own subject (design 47 G §365, §404). delete(s.runningSaid, module) *said = append(*said, func() { q.sayRunning(stoppedRunning(node, module)) }) @@ -983,6 +995,20 @@ func (q *Session) sayRunning(ev nodeengine.Running) { q.sayEngineOn(nodeengine.SubjectRunning(ev.Node, ev.Module), "running", ev) } +// SetHaveSaid gives the session what it tells of each module's have as it writes it (said), and of a module whose have says +// it is gone (gone): the node-engine's findings of itself are made of them (design 47 G A-197). Either may be nil. +func (q *Session) SetHaveSaid(said func(module string, h nodeengine.HaveByNodeModuleValue), gone func(module string)) { + q.haveSay.Lock() + q.haveSaidFn, q.haveGoneFn = said, gone + q.haveSay.Unlock() +} + +func (q *Session) haveSaid() (func(string, nodeengine.HaveByNodeModuleValue), func(string)) { + q.haveSay.Lock() + defer q.haveSay.Unlock() + return q.haveSaidFn, q.haveGoneFn +} + // SetManifestOf gives the session what names the manifest of the build a module runs (the composer's picture of the // catalog half): its digest, or "" when the node does not know it. func (q *Session) SetManifestOf(f func(module, fp string) string) { diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 132fa730..23b5fa5a 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -230,7 +230,13 @@ func aHaveQueue(l *haveLink) (*Session, *[]string) { func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) + // Design 47 G A-197: each have as written is told to the node-engine's findings of itself, the failed one with its detail. + told := map[string]nodeengine.HaveByNodeModuleValue{} + q.SetHaveSaid(func(m string, h nodeengine.HaveByNodeModuleValue) { told[m] = h }, nil) q.SayFacts(context.Background(), appliedModules()) + if told["b"].Phase != nodeengineclient.PhaseFailed || !strings.Contains(told["b"].Detail, "no space") || told["a"].Phase != nodeengineclient.PhaseRunning { + t.Errorf("the haves told were %+v", told) + } a, b := l.have(t, "n1.a"), l.have(t, "n1.b") if a.Phase != nodeengineclient.PhaseRunning || a.Build != "b-1" || a.Health != nodeengineclient.HealthOK { t.Errorf("a: %+v", a) @@ -563,9 +569,14 @@ func onlyA() Report { func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) + var gone []string + q.SetHaveSaid(nil, func(m string) { gone = append(gone, m) }) q.SayFacts(context.Background(), appliedModules()) revA, revB := l.kv["n1.a"].rev, l.kv["n1.b"].rev q.SayFacts(context.Background(), onlyA()) + if len(gone) != 1 || gone[0] != "b" { // design 47 G A-197: a module that left ends its findings + t.Errorf("the modules told gone were %v, want b", gone) + } b := l.have(t, "n1.b") if b.Phase != nodeengineclient.PhaseRemoved || b.Build != "b-2" { t.Errorf("b: %+v", b) diff --git a/internal/link/session.go b/internal/link/session.go index 4c2e3704..dc8b3911 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -29,6 +29,10 @@ type Session struct { // Own is the module that is this node-engine (§527, §634): its discovery, finding writer and vault login names read it. // Nil names none, as a test's session does. Own *OwnName + // haveSay guards haveSaidFn and haveGoneFn (SetHaveSaid). + haveSay sync.Mutex + haveSaidFn func(module string, h nodeengine.HaveByNodeModuleValue) + haveGoneFn func(module string) // Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none. Fallback func() *nodeengine.LoginFallback // linkedUser is the bus user the link open now authenticated as, guarded by mu (WhenLinkedAs). diff --git a/internal/seatwait/nodefindings.go b/internal/seatwait/nodefindings.go new file mode 100644 index 00000000..395815cf --- /dev/null +++ b/internal/seatwait/nodefindings.go @@ -0,0 +1,568 @@ +// NodeKeeper keeps the findings only the node-engine knows of itself (design 47 G A-197, ADR 0596, A-149): a have's failed +// phase (apply-failing) and its health (module-unhealthy), what it keeps of a module as it runs (kept-as-it-runs), and each +// part of its own health report (machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed, +// directory-used-as-found, node-account-fallback), under the kind words the watchman raised them under. Each is raised and +// cleared where the engine knows the fact, through its finding writer, never read back from the have or the report by +// another part. +// +// The keeper holds the facts it was told (the haves, the last report) and, at every telling, makes the findings of this +// node those the facts say: what the writer holds of its kinds for this node and no fact says is cleared, what a fact says +// and the writer does not hold, or holds in other words, is written. Nothing runs on a timer: a bound (the apply interval, +// the minute before a failing health is said, the four hours before it is urgent) is judged at each telling, and the engine +// tells at every pass over its haves and every look at its health. +package seatwait + +import ( + "context" + "fmt" + "maps" + "slices" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" +) + +// The kinds the node-engine raises of itself, the words the watchman raised them under. +const ( + KindApplyFailing = "apply-failing" + KindModuleUnhealthy = "module-unhealthy" + KindKeptAsRuns = "kept-as-it-runs" + KindMachineNetwork = "machine-network" + KindNetworkRewritten = "network-rewritten" + KindNetworkUnreachable = "network-unreachable" + KindMachineUnits = "machine-units" + KindReloginNeeded = "relogin-needed" + KindUsedAsFound = "directory-used-as-found" + KindAccountFallback = "node-account-fallback" +) + +// NodeKinds is every kind the keeper writes, for the finding writer's list of kinds. +var NodeKinds = []string{KindApplyFailing, KindModuleUnhealthy, KindKeptAsRuns, KindMachineNetwork, KindNetworkRewritten, + KindNetworkUnreachable, KindMachineUnits, KindReloginNeeded, KindUsedAsFound, KindAccountFallback} + +// The bounds the facts are judged by. +const ( + // ApplyEvery is how long a have may keep saying failed before the node says it does not apply its update. + ApplyEvery = 5 * time.Minute + // UnhealthyConfirmAfter is how long a have must keep saying its health failed before the module is said unhealthy. + UnhealthyConfirmAfter = time.Minute + // UnhealthyUrgentAfter is how long a module stays unhealthy before it is urgent. + UnhealthyUrgentAfter = 4 * time.Hour + // ControlNodeModule is the module whose have names the control node: a failing network there is urgent. + ControlNodeModule = "nats" + // PackageManagerKey is how a node-engine names a package manager that does not answer among what failed. + PackageManagerKey = "the package manager (" +) + +const ( + partResolvConf = "resolv-conf" + partNames = "names" + partTunnel = "tunnel" + partBus = "bus" + stateUnhealthy = "unhealthy" + phaseFailed = nodeengineclient.PhaseFailed +) + +// NodeKeeper keeps the findings of one node-engine. Tell it with Have, HaveGone and Report; every telling reconciles. +type NodeKeeper struct { + Node string + Observer string + // ObserverOf, when set, answers the observer at each write (see Waiter). + ObserverOf func() string + Writer Writer + // Ctx is the run's context for the writer's acts; nil is the background. + Ctx context.Context + // Now is the clock; nil is time.Now. + Now func() time.Time + // Say tells the node's journal; nil says nothing. + Say func(string) + + mu sync.Mutex + haves map[string]nodeengine.HaveByNodeModuleValue + failedSince map[string]time.Time + sickSince map[string]time.Time + keptSince map[string]time.Time + report *nodeengine.HealthByNodeValue + byAddress map[string]string +} + +// Have is a module's have as the node-engine wrote it. +func (k *NodeKeeper) Have(module string, h nodeengine.HaveByNodeModuleValue) { + k.mu.Lock() + defer k.mu.Unlock() + if k.haves == nil { + k.haves, k.failedSince, k.sickSince, k.keptSince = map[string]nodeengine.HaveByNodeModuleValue{}, map[string]time.Time{}, + map[string]time.Time{}, map[string]time.Time{} + } + if h.Phase == nodeengineclient.PhaseRemoved { + k.forget(module) + k.reconcile() + return + } + k.haves[module] = h + now := k.now() + setSince(k.failedSince, module, h.Phase == phaseFailed, now) + setSince(k.sickSince, module, h.Health == nodeengineclient.HealthFailed, now) + setSince(k.keptSince, module, len(h.Kept) > 0, now) + k.reconcile() +} + +// HaveGone is a module no longer held by the node. +func (k *NodeKeeper) HaveGone(module string) { + k.mu.Lock() + defer k.mu.Unlock() + k.forget(module) + k.reconcile() +} + +// Report is the node's health report as it stands now (written or not), and the machines by the address of each, for the +// peers whose tunnel it does not see. Told at every look, so a bound is judged then. +func (k *NodeKeeper) Report(h nodeengine.HealthByNodeValue, byAddress map[string]string) { + k.mu.Lock() + defer k.mu.Unlock() + if h.At.IsZero() { + h.At = k.now() + } + k.report, k.byAddress = &h, byAddress + k.reconcile() +} + +func (k *NodeKeeper) forget(module string) { + delete(k.haves, module) + delete(k.failedSince, module) + delete(k.sickSince, module) + delete(k.keptSince, module) +} + +func setSince(m map[string]time.Time, module string, on bool, now time.Time) { + if !on { + delete(m, module) + } else if _, ok := m[module]; !ok { + m[module] = now + } +} + +func (k *NodeKeeper) now() time.Time { + if k.Now != nil { + return k.Now() + } + return time.Now() +} + +func (k *NodeKeeper) say(s string) { + if k.Say != nil { + k.Say(s) + } +} + +func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") } + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} + +func orWords(s, or string) string { + if strings.TrimSpace(s) == "" { + return or + } + return s +} + +// ownKey is the identity of a finding the keeper writes. +func ownKey(scope, id, kind string) string { return scope + "." + id + "." + kind } + +// reconcile makes this node's findings of the keeper's kinds those the facts say. The caller holds the lock. +func (k *NodeKeeper) reconcile() { + ctx := k.Ctx + if ctx == nil { + ctx = context.Background() + } + want := map[string]condseat.Finding{} + add := func(f condseat.Finding) { want[ownKey(f.Scope, f.ID, f.Kind)] = f } + k.haveFindings(add) + k.reportFindings(add) + held := map[string]condseat.Finding{} + for _, f := range k.Writer.Held() { + if k.owns(f) { + held[ownKey(f.Scope, f.ID, f.Kind)] = f + } + } + for _, key := range slices.Sorted(maps.Keys(held)) { + if _, still := want[key]; still { + continue + } + f := held[key] + if err := k.Writer.Clear(ctx, f.Scope, f.ID, f.Kind, "the node-engine no longer finds it so"); err != nil { + k.say(fmt.Sprintf("the finding %s of %s cannot be cleared: %v", f.Kind, k.Node, err)) + } + } + for _, key := range slices.Sorted(maps.Keys(want)) { + f := want[key] + if was, ok := held[key]; ok { + f.Since, f.At = was.Since, was.At // the same thing seen again writes the same words + if sameFinding(was, f) { + continue + } + } + if err := k.Writer.Write(ctx, f); err != nil { + k.say(fmt.Sprintf("the finding %s of %s cannot be written: %v", f.Kind, k.Node, err)) + continue + } + k.say(f.Summary) + } +} + +// owns says a held finding is one the keeper writes for this node. +func (k *NodeKeeper) owns(f condseat.Finding) bool { + if !slices.Contains(NodeKinds, f.Kind) { + return false + } + if f.Kind == KindNetworkUnreachable { + return strings.HasSuffix(f.ID, "."+k.Node) + } + return f.Subject.Machine == k.Node +} + +func sameFinding(a, b condseat.Finding) bool { + if a.Summary != b.Summary || a.Severity != b.Severity || a.Headline != b.Headline || a.Needs != b.Needs || + a.Explanation != b.Explanation || !slices.Equal(a.Subject.Also, b.Subject.Also) || len(a.Evidence) != len(b.Evidence) { + return false + } + for i := range a.Evidence { + if a.Evidence[i].Said != b.Evidence[i].Said { + return false + } + } + return true +} + +// finding is a finding of this node's: observed by the engine's own module, about a machine of the scope. +func (k *NodeKeeper) finding(scope, id, kind, machine string, severity string, since time.Time, summary, said string) condseat.Finding { + since = since.UTC() + f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: kind, Scope: scope, ID: id, Severity: severity, + Source: kind, Summary: summary, At: since, Since: since, + Subject: condseat.Subject{Scope: scope, ID: id, Machine: machine}} + if said != "" { + f.Evidence = []condseat.Evidence{{At: since, Said: said}} + } + return f +} + +// haveFindings are the findings the haves say: apply-failing, module-unhealthy and kept-as-it-runs. +func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) { + now := k.now() + type failure struct{ module, words string } + var failed []failure + var since time.Time + for _, m := range slices.Sorted(maps.Keys(k.failedSince)) { + failed = append(failed, failure{m, firstLine(orWords(k.haves[m].Detail, "no reason given"))}) + if at := k.failedSince[m]; since.IsZero() || at.Before(since) { + since = at + } + } + if len(failed) > 0 && now.Sub(since) > ApplyEvery { + var names, words []string + tool := "" + for _, f := range failed { + names = append(names, f.module) + words = append(words, f.module+": "+f.words) + w := strings.TrimPrefix(f.words, "refused: ") + if i := strings.Index(w, PackageManagerKey); i >= 0 && tool == "" { + tool = strings.TrimSuffix(strings.SplitN(w[i+len(PackageManagerKey):], ")", 2)[0], ")") + } + } + f := k.finding(scopeMachine, k.Node, KindApplyFailing, k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; ")) + f.Headline = k.Node + " is not applying its update" + if tool != "" { + f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, since %s (bound %s)", + k.Node, tool, stampMin(since), ApplyEvery) + f.Headline = k.Node + " cannot install packages" + } else { + f.Summary = fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", k.Node, strings.Join(names, ", "), stampMin(since), ApplyEvery) + f.Explanation = fmt.Sprintf("%s has failed to apply its updates, and trying again has not helped. The details say what fails.", k.Node) + } + add(f) + } + for _, m := range slices.Sorted(maps.Keys(k.sickSince)) { + at := k.sickSince[m] + if now.Sub(at) < UnhealthyConfirmAfter { + continue + } + severity := condseat.FindingSeverityWarning + if now.Sub(at) >= UnhealthyUrgentAfter { + severity = condseat.FindingSeverityUrgent + } + h := k.haves[m] + f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, k.Node, severity, at, + fmt.Sprintf("%s on %s is not healthy: %s", m, k.Node, orWords(h.Detail, "its check failed")), firstLine(orWords(h.Detail, "failed"))) + f.Headline = fmt.Sprintf("%s not working on %s", m, k.Node) + add(f) + } + for _, m := range slices.Sorted(maps.Keys(k.keptSince)) { + words := strings.Join(k.haves[m].Kept, "; ") + f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, k.Node, condseat.FindingSeverityWarning, k.keptSince[m], + fmt.Sprintf("%s on %s is kept as it runs: %s", m, k.Node, words), firstLine(words)) + f.Headline = fmt.Sprintf("%s kept as it runs on %s", m, k.Node) + f.Needs = "a build of " + m + " whose manifest says it (tool-runner, core-credentials)" + add(f) + } +} + +func stampMin(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") } + +// control says the node runs the module the control node is named by. +func (k *NodeKeeper) control() bool { + h, ok := k.haves[ControlNodeModule] + return ok && h.Phase != nodeengineclient.PhaseRemoved +} + +// reportFindings are the findings the node's health report says. +func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) { + h := k.report + if h == nil { + return + } + node := k.Node + var own []nodeengine.NetworkPart + var rewritten *nodeengine.NetworkPart + var parts []nodeengine.NetworkPart + if h.Network != nil { + for _, p := range h.Network.Parts { + if p.State == stateUnhealthy { + parts = append(parts, p) + } + } + } + toward := map[string][]nodeengine.NetworkPart{} + for i, p := range parts { + switch { + case p.Part == partResolvConf: + rewritten = &parts[i] + case len(p.Targets) > 0: + for _, x := range p.Targets { + toward[x] = append(toward[x], p) + } + default: + own = append(own, p) + } + } + if rewritten != nil { + add(k.rewritten(*rewritten, parts)) + kept := own[:0] + for _, p := range own { + if p.Part != partNames { + kept = append(kept, p) + } + } + own = kept + } + if len(own) > 0 { + var words, said []string + severity := condseat.FindingSeverityWarning + for _, p := range own { + words = append(words, p.Reason) + said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said)) + if p.Part == partBus { + severity = condseat.FindingSeverityUrgent + } + } + if k.control() { + severity = condseat.FindingSeverityUrgent + } + add(k.finding(scopeMachine, node, KindMachineNetwork, node, severity, h.Network.Since, + fmt.Sprintf("%s's network is not healthy: %s", node, strings.Join(words, "; ")), strings.Join(said, " | "))) + } + // A tunnel peer this node does not see points at that peer's machine as a tunnel part toward it would. + for _, p := range h.Peers { + if p.Seen { + continue + } + for _, a := range p.Addresses { + addr, _, _ := strings.Cut(a, "/") + if x := k.byAddress[addr]; x != "" && x != node { + toward[x] = append(toward[x], nodeengine.NetworkPart{Part: partTunnel, State: stateUnhealthy, Since: h.At, + Said: "no handshake with " + x + " within five minutes", Targets: []string{x}}) + } + } + } + for _, x := range slices.Sorted(maps.Keys(toward)) { + var what, said []string + since := toward[x][0].Since + severity := condseat.FindingSeverityWarning + for _, p := range toward[x] { + wd := map[string]string{partTunnel: "the tunnel to it", partBus: "the bus on it", partNames: "its resolver"}[p.Part] + if wd == "" { + wd = p.Part + } + if !slices.Contains(what, wd) { + what = append(what, wd) + } + said = append(said, fmt.Sprintf("%s: %s: %s", node, p.Part, p.Said)) + if p.Since.Before(since) { + since = p.Since + } + if p.Part == partBus { + severity = condseat.FindingSeverityUrgent + } + } + f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, x, severity, since, + fmt.Sprintf("%s cannot be reached from %s: %s", x, node, strings.Join(what, ", ")), strings.Join(said, " | ")) + f.Subject.Also = []string{node} + add(f) + } + if f, ok := k.units(h.Units); ok { + add(f) + } + for _, f := range k.relogin(h.Relogin) { + add(f) + } + for _, f := range k.found(h.Found) { + add(f) + } + if f := h.Fallback; f != nil && f.Node == node { + said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why) + g := k.finding(scopeMachine, node, KindAccountFallback, node, condseat.FindingSeverityWarning, f.At, said, said) + g.Headline = fmt.Sprintf("%s fell back to its old bus login", node) + g.Needs = fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node) + add(g) + } +} + +func (k *NodeKeeper) rewritten(p nodeengine.NetworkPart, all []nodeengine.NetworkPart) condseat.Finding { + m := k.Node + writer := "" + if p.Writer != "" { + writer = " (" + p.Writer + ")" + } + cost := "the names still resolve through what it wrote" + said := []string{p.Part + ": " + p.Said} + for _, q := range all { + if q.Part == partNames { + cost = strings.TrimSuffix(q.Reason, ".") + said = append(said, q.Part+": "+q.Said) + } + } + id := m + summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+ + "back at its next reconcile, or that program gives it back", m, writer, cost) + if p.Owner != "" { + id = m + "." + p.Owner + summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+ + "writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) + } + severity := condseat.FindingSeverityWarning + if k.control() { + severity = condseat.FindingSeverityUrgent + } + return k.finding(scopeMachine, id, KindNetworkRewritten, m, severity, p.Since, summary, + fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))) +} + +func (k *NodeKeeper) units(u *nodeengine.UnitsHealth) (condseat.Finding, bool) { + if u == nil || len(u.Failed) == 0 { + return condseat.Finding{}, false + } + node := k.Node + var names, said []string + since := u.Failed[0].Since + for _, f := range u.Failed { + where := "" + if f.Scope == "user" { + where = " (the account's own manager)" + } + names = append(names, f.Unit+where) + scope, result := f.Scope, f.Result + if scope == "" { + scope = "system" + } + if result == "" { + result = "no reason said" + } + line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result) + if f.Load != "" && f.Load != "loaded" { + line += ", its unit " + f.Load + } + if f.Resource != "" { + line += ", named by the mesh's own " + f.Resource + } + said = append(said, line+", since "+stampSec(f.Since)) + if f.Since.Before(since) { + since = f.Since + } + } + failed := "1 failed service" + if len(u.Failed) > 1 { + failed = fmt.Sprintf("%d failed services", len(u.Failed)) + } + f := k.finding(scopeMachine, node, KindMachineUnits, node, condseat.FindingSeverityWarning, since, + fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+ + "mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")), + strings.Join(said, "; ")) + f.Headline = strings.ToUpper(failed[:1]) + failed[1:] + " on " + node + f.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node) + return f, true +} + +func orModule(m string) string { + if m == "" { + return "_" + } + return m +} + +func (k *NodeKeeper) relogin(accounts []nodeengine.NodeAccount) []condseat.Finding { + by := map[string][]nodeengine.NodeAccount{} + for _, a := range accounts { + by[a.Module] = append(by[a.Module], a) + } + var out []condseat.Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var names []string + since := by[module][0].Since + for _, a := range by[module] { + names = append(names, a.Account) + if a.Since.Before(since) { + since = a.Since + } + } + sort.Strings(names) + said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+ + "before it was; log out of %[1]s completely and log in again, or restart it", k.Node, orModule(module), strings.Join(names, ", ")) + out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, k.Node, condseat.FindingSeverityWarning, since, said, said)) + } + return out +} + +func (k *NodeKeeper) found(dirs []nodeengine.NodeDirectory) []condseat.Finding { + by := map[string][]nodeengine.NodeDirectory{} + for _, d := range dirs { + by[d.Module] = append(by[d.Module], d) + } + var out []condseat.Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var ids, why []string + since := by[module][0].Since + for _, d := range by[module] { + ids = append(ids, d.Resource) + why = append(why, d.Reason) + if d.Since.Before(since) { + since = d.Since + } + } + m := orModule(module) + said := fmt.Sprintf("%s on %s uses %s as found: %s", m, k.Node, strings.Join(ids, ", "), strings.Join(why, "; ")) + f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, k.Node, condseat.FindingSeverityWarning, since, said, said) + f.Headline = fmt.Sprintf("%s waits for a directory on %s", m, k.Node) + f.Explanation = fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+ + "and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, k.Node, m) + f.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type." + out = append(out, f) + } + return out +} diff --git a/internal/seatwait/nodefindings_test.go b/internal/seatwait/nodefindings_test.go new file mode 100644 index 00000000..8324bc7e --- /dev/null +++ b/internal/seatwait/nodefindings_test.go @@ -0,0 +1,104 @@ +package seatwait + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" +) + +// Design 47 G A-197: what only the node-engine knows of itself it raises as its own findings, under the kind words the watchman +// raised them under, and clears where it knows the fact no longer holds. +func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) { + at := time.Date(2026, 11, 12, 8, 0, 0, 0, time.UTC) + type step struct { + do func(k *NodeKeeper, advance func(time.Duration)) + want []string // "scope.id.kind" of what stands after it, sorted + } + failed := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseFailed, Detail: "refused: the package manager (pacman) does not answer"} + sick := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "port closed"} + report := func(h nodeengine.HealthByNodeValue) func(*NodeKeeper, func(time.Duration)) { + return func(k *NodeKeeper, _ func(time.Duration)) { k.Report(h, map[string]string{"10.0.0.2": "g14"}) } + } + for name, steps := range map[string][]step{ + "a have failed past the apply interval is apply-failing, cleared by the clean apply": { + {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", failed) }, nil}, + {func(k *NodeKeeper, adv func(time.Duration)) { adv(ApplyEvery + time.Second); k.Have("b", failed) }, []string{"machine.ace.apply-failing"}}, + {func(k *NodeKeeper, _ func(time.Duration)) { + k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning}) + }, nil}, + }, + "a have unhealthy past a minute is module-unhealthy, cleared when it is healthy or removed": { + {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", sick) }, nil}, + {func(k *NodeKeeper, adv func(time.Duration)) { adv(time.Minute); k.Have("b", sick) }, []string{"module.b.ace.module-unhealthy"}}, + {func(k *NodeKeeper, _ func(time.Duration)) { + k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRemoved}) + }, nil}, + }, + "what a module keeps as it runs is kept-as-it-runs until its have says none": { + {func(k *NodeKeeper, _ func(time.Duration)) { + k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Kept: []string{"tool-runner"}}) + }, []string{"module.b.ace.kept-as-it-runs"}}, + {func(k *NodeKeeper, _ func(time.Duration)) { k.HaveGone("b") }, nil}, + }, + "each part of the report is its own finding and each ends with the report": { + {report(nodeengine.HealthByNodeValue{ + Network: &nodeengine.NetworkHealth{Since: at, Parts: []nodeengine.NetworkPart{ + {Part: "bus", State: "unhealthy", Reason: "no bus", Since: at}, + {Part: "tunnel", State: "unhealthy", Since: at, Targets: []string{"g14"}}, + {Part: "resolv-conf", State: "unhealthy", Owner: "dnsmasq", Since: at}}}, + Units: &nodeengine.UnitsHealth{Failed: []nodeengine.FailedUnit{{Unit: "x.service", Since: at}}}, + Relogin: []nodeengine.NodeAccount{{Account: "u", Module: "m", Since: at}}, + Found: []nodeengine.NodeDirectory{{Module: "m", Resource: "m.dir", Since: at}}, + Fallback: &nodeengine.LoginFallback{Node: "ace", From: "new", At: at}}), + []string{"machine.ace.dnsmasq.network-rewritten", "machine.ace.machine-network", "machine.ace.machine-units", + "machine.ace.node-account-fallback", "machine.g14.ace.network-unreachable", "module.m.ace.directory-used-as-found", + "module.m.ace.relogin-needed"}}, + {report(nodeengine.HealthByNodeValue{}), nil}, + }, + "a tunnel peer not seen is the machine unreachable": { + {report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}}}}), + []string{"machine.g14.ace.network-unreachable"}}, + {report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}, Seen: true}}}), nil}, + }, + } { + t.Run(name, func(t *testing.T) { + now := at + w := &fakeFindings{} + // Another node's finding and an unrelated kind stand in the state and are never touched. + w.held = []condseat.Finding{{Scope: "machine", ID: "g14", Kind: KindMachineUnits, Subject: condseat.Subject{Machine: "g14"}}, + {Scope: "machine", ID: "ace", Kind: HeldKind, Subject: condseat.Subject{Machine: "ace"}}} + k := &NodeKeeper{Node: "ace", Observer: "mesh-host", Writer: w, Now: func() time.Time { return now }} + for i, s := range steps { + s.do(k, func(d time.Duration) { now = now.Add(d) }) + var got []string + for _, f := range w.held { + if f.Observer == "mesh-host" { + got = append(got, f.Scope+"."+f.ID+"."+f.Kind) + } + } + if strings.Join(sorted(got), ",") != strings.Join(s.want, ",") { + t.Fatalf("step %d: stands %v, want %v", i, sorted(got), s.want) + } + } + if len(w.held) != 2 { + t.Fatalf("what is not the keeper's was touched: %+v", w.held) + } + }) + } +} + +func sorted(in []string) []string { + out := append([]string(nil), in...) + for i := range out { + for j := i + 1; j < len(out); j++ { + if out[j] < out[i] { + out[i], out[j] = out[j], out[i] + } + } + } + return out +}