novox/mesh-host
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

H-find: the findings keep the keys the watchman raised them under

Browse Source 
Each finding's id carries its token as a ref (A-14), so machine.<node>.network, .rewritten, .unreachable, .units and module.<module>.<node>.unhealthy and .kept
read as before.

Builds: design 47 G A-197

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
jschoubben
2026-11-12 06:02:07 +01:00
parent 4b215ba6ae
commit a3fea9106a
2 changed files with 24 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/seatwait/nodefindings.go
+19 -13
View File
@@ -224,7 +224,7 @@ func (k *NodeKeeper) owns(f condseat.Finding) bool {
return false
}
if f.Kind == KindNetworkUnreachable {
return strings.HasSuffix(f.ID, "."+k.Node)
return strings.HasSuffix(f.ID, "."+k.Node+".unreachable")
}
return f.Subject.Machine == k.Node
}
@@ -243,10 +243,16 @@ func sameFinding(a, b condseat.Finding) bool {
}
// finding is a finding of this node's: observed by the engine's own module, about a machine of the scope.
func (k *NodeKeeper) finding(scope, id, kind, machine string, severity string, since time.Time, summary, said string) condseat.Finding {
//
// The id and ref are the condition's key the watchman raised it under, `<scope>.<id>.<token>` (A-14): a token other than the kind
// goes into the id as `<id>.<ref>` and is kept in ref, as the watchman's findings were written.
func (k *NodeKeeper) finding(scope, id, kind, ref, machine string, severity string, since time.Time, summary, said string) condseat.Finding {
since = since.UTC()
if ref != "" {
id += "." + ref
}
f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: kind, Scope: scope, ID: id, Severity: severity,
Source: kind, Summary: summary, At: since, Since: since,
Ref: ref, Source: kind, Summary: summary, At: since, Since: since,
Subject: condseat.Subject{Scope: scope, ID: id, Machine: machine}}
if said != "" {
f.Evidence = []condseat.Evidence{{At: since, Said: said}}
@@ -277,7 +283,7 @@ func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) {
tool = strings.TrimSuffix(strings.SplitN(w[i+len(PackageManagerKey):], ")", 2)[0], ")")
}
}
f := k.finding(scopeMachine, k.Node, KindApplyFailing, k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; "))
f := k.finding(scopeMachine, k.Node, KindApplyFailing, "", k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; "))
f.Headline = k.Node + " is not applying its update"
if tool != "" {
f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, since %s (bound %s)",
@@ -299,14 +305,14 @@ func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) {
severity = condseat.FindingSeverityUrgent
}
h := k.haves[m]
f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, k.Node, severity, at,
f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, "unhealthy", k.Node, severity, at,
fmt.Sprintf("%s on %s is not healthy: %s", m, k.Node, orWords(h.Detail, "its check failed")), firstLine(orWords(h.Detail, "failed")))
f.Headline = fmt.Sprintf("%s not working on %s", m, k.Node)
add(f)
}
for _, m := range slices.Sorted(maps.Keys(k.keptSince)) {
words := strings.Join(k.haves[m].Kept, "; ")
f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, k.Node, condseat.FindingSeverityWarning, k.keptSince[m],
f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, "kept", k.Node, condseat.FindingSeverityWarning, k.keptSince[m],
fmt.Sprintf("%s on %s is kept as it runs: %s", m, k.Node, words), firstLine(words))
f.Headline = fmt.Sprintf("%s kept as it runs on %s", m, k.Node)
f.Needs = "a build of " + m + " whose manifest says it (tool-runner, core-credentials)"
@@ -375,7 +381,7 @@ func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) {
if k.control() {
severity = condseat.FindingSeverityUrgent
}
add(k.finding(scopeMachine, node, KindMachineNetwork, node, severity, h.Network.Since,
add(k.finding(scopeMachine, node, KindMachineNetwork, "network", node, severity, h.Network.Since,
fmt.Sprintf("%s's network is not healthy: %s", node, strings.Join(words, "; ")), strings.Join(said, " | ")))
}
// A tunnel peer this node does not see points at that peer's machine as a tunnel part toward it would.
@@ -411,7 +417,7 @@ func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) {
severity = condseat.FindingSeverityUrgent
}
}
f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, x, severity, since,
f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, "unreachable", x, severity, since,
fmt.Sprintf("%s cannot be reached from %s: %s", x, node, strings.Join(what, ", ")), strings.Join(said, " | "))
f.Subject.Also = []string{node}
add(f)
@@ -427,7 +433,7 @@ func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) {
}
if f := h.Fallback; f != nil && f.Node == node {
said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why)
g := k.finding(scopeMachine, node, KindAccountFallback, node, condseat.FindingSeverityWarning, f.At, said, said)
g := k.finding(scopeMachine, node, KindAccountFallback, "", node, condseat.FindingSeverityWarning, f.At, said, said)
g.Headline = fmt.Sprintf("%s fell back to its old bus login", node)
g.Needs = fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node)
add(g)
@@ -460,7 +466,7 @@ func (k *NodeKeeper) rewritten(p nodeengine.NetworkPart, all []nodeengine.Networ
if k.control() {
severity = condseat.FindingSeverityUrgent
}
return k.finding(scopeMachine, id, KindNetworkRewritten, m, severity, p.Since, summary,
return k.finding(scopeMachine, id, KindNetworkRewritten, "rewritten", m, severity, p.Since, summary,
fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | ")))
}
@@ -500,7 +506,7 @@ func (k *NodeKeeper) units(u *nodeengine.UnitsHealth) (condseat.Finding, bool) {
if len(u.Failed) > 1 {
failed = fmt.Sprintf("%d failed services", len(u.Failed))
}
f := k.finding(scopeMachine, node, KindMachineUnits, node, condseat.FindingSeverityWarning, since,
f := k.finding(scopeMachine, node, KindMachineUnits, "units", node, condseat.FindingSeverityWarning, since,
fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+
"mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")),
strings.Join(said, "; "))
@@ -534,7 +540,7 @@ func (k *NodeKeeper) relogin(accounts []nodeengine.NodeAccount) []condseat.Findi
sort.Strings(names)
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+
"before it was; log out of %[1]s completely and log in again, or restart it", k.Node, orModule(module), strings.Join(names, ", "))
out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, k.Node, condseat.FindingSeverityWarning, since, said, said))
out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, "", k.Node, condseat.FindingSeverityWarning, since, said, said))
}
return out
}
@@ -557,7 +563,7 @@ func (k *NodeKeeper) found(dirs []nodeengine.NodeDirectory) []condseat.Finding {
}
m := orModule(module)
said := fmt.Sprintf("%s on %s uses %s as found: %s", m, k.Node, strings.Join(ids, ", "), strings.Join(why, "; "))
f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, k.Node, condseat.FindingSeverityWarning, since, said, said)
f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, "", k.Node, condseat.FindingSeverityWarning, since, said, said)
f.Headline = fmt.Sprintf("%s waits for a directory on %s", m, k.Node)
f.Explanation = fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+
"and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, k.Node, m)
internal/seatwait/nodefindings_test.go
+5 -5
View File
@@ -33,7 +33,7 @@ func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) {
},
"a have unhealthy past a minute is module-unhealthy, cleared when it is healthy or removed": {
{func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", sick) }, nil},
{func(k *NodeKeeper, adv func(time.Duration)) { adv(time.Minute); k.Have("b", sick) }, []string{"module.b.ace.module-unhealthy"}},
{func(k *NodeKeeper, adv func(time.Duration)) { adv(time.Minute); k.Have("b", sick) }, []string{"module.b.ace.unhealthy.module-unhealthy"}},
{func(k *NodeKeeper, _ func(time.Duration)) {
k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRemoved})
}, nil},
@@ -41,7 +41,7 @@ func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) {
"what a module keeps as it runs is kept-as-it-runs until its have says none": {
{func(k *NodeKeeper, _ func(time.Duration)) {
k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Kept: []string{"tool-runner"}})
}, []string{"module.b.ace.kept-as-it-runs"}},
}, []string{"module.b.ace.kept.kept-as-it-runs"}},
{func(k *NodeKeeper, _ func(time.Duration)) { k.HaveGone("b") }, nil},
},
"each part of the report is its own finding and each ends with the report": {
@@ -54,14 +54,14 @@ func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) {
Relogin: []nodeengine.NodeAccount{{Account: "u", Module: "m", Since: at}},
Found: []nodeengine.NodeDirectory{{Module: "m", Resource: "m.dir", Since: at}},
Fallback: &nodeengine.LoginFallback{Node: "ace", From: "new", At: at}}),
[]string{"machine.ace.dnsmasq.network-rewritten", "machine.ace.machine-network", "machine.ace.machine-units",
"machine.ace.node-account-fallback", "machine.g14.ace.network-unreachable", "module.m.ace.directory-used-as-found",
[]string{"machine.ace.dnsmasq.rewritten.network-rewritten", "machine.ace.network.machine-network", "machine.ace.node-account-fallback",
"machine.ace.units.machine-units", "machine.g14.ace.unreachable.network-unreachable", "module.m.ace.directory-used-as-found",
"module.m.ace.relogin-needed"}},
{report(nodeengine.HealthByNodeValue{}), nil},
},
"a tunnel peer not seen is the machine unreachable": {
{report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}}}}),
[]string{"machine.g14.ace.network-unreachable"}},
[]string{"machine.g14.ace.unreachable.network-unreachable"}},
{report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}, Seen: true}}}), nil},
},
} {