diff --git a/internal/seatwait/nodefindings.go b/internal/seatwait/nodefindings.go index 395815cf..f3000aa4 100644 --- a/internal/seatwait/nodefindings.go +++ b/internal/seatwait/nodefindings.go @@ -224,7 +224,7 @@ func (k *NodeKeeper) owns(f condseat.Finding) bool { return false } if f.Kind == KindNetworkUnreachable { - return strings.HasSuffix(f.ID, "."+k.Node) + return strings.HasSuffix(f.ID, "."+k.Node+".unreachable") } return f.Subject.Machine == k.Node } @@ -243,10 +243,16 @@ func sameFinding(a, b condseat.Finding) bool { } // finding is a finding of this node's: observed by the engine's own module, about a machine of the scope. -func (k *NodeKeeper) finding(scope, id, kind, machine string, severity string, since time.Time, summary, said string) condseat.Finding { +// +// The id and ref are the condition's key the watchman raised it under, `..` (A-14): a token other than the kind +// goes into the id as `.` and is kept in ref, as the watchman's findings were written. +func (k *NodeKeeper) finding(scope, id, kind, ref, machine string, severity string, since time.Time, summary, said string) condseat.Finding { since = since.UTC() + if ref != "" { + id += "." + ref + } f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: kind, Scope: scope, ID: id, Severity: severity, - Source: kind, Summary: summary, At: since, Since: since, + Ref: ref, Source: kind, Summary: summary, At: since, Since: since, Subject: condseat.Subject{Scope: scope, ID: id, Machine: machine}} if said != "" { f.Evidence = []condseat.Evidence{{At: since, Said: said}} @@ -277,7 +283,7 @@ func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) { tool = strings.TrimSuffix(strings.SplitN(w[i+len(PackageManagerKey):], ")", 2)[0], ")") } } - f := k.finding(scopeMachine, k.Node, KindApplyFailing, k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; ")) + f := k.finding(scopeMachine, k.Node, KindApplyFailing, "", k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; ")) f.Headline = k.Node + " is not applying its update" if tool != "" { f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, since %s (bound %s)", @@ -299,14 +305,14 @@ func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) { severity = condseat.FindingSeverityUrgent } h := k.haves[m] - f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, k.Node, severity, at, + f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, "unhealthy", k.Node, severity, at, fmt.Sprintf("%s on %s is not healthy: %s", m, k.Node, orWords(h.Detail, "its check failed")), firstLine(orWords(h.Detail, "failed"))) f.Headline = fmt.Sprintf("%s not working on %s", m, k.Node) add(f) } for _, m := range slices.Sorted(maps.Keys(k.keptSince)) { words := strings.Join(k.haves[m].Kept, "; ") - f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, k.Node, condseat.FindingSeverityWarning, k.keptSince[m], + f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, "kept", k.Node, condseat.FindingSeverityWarning, k.keptSince[m], fmt.Sprintf("%s on %s is kept as it runs: %s", m, k.Node, words), firstLine(words)) f.Headline = fmt.Sprintf("%s kept as it runs on %s", m, k.Node) f.Needs = "a build of " + m + " whose manifest says it (tool-runner, core-credentials)" @@ -375,7 +381,7 @@ func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) { if k.control() { severity = condseat.FindingSeverityUrgent } - add(k.finding(scopeMachine, node, KindMachineNetwork, node, severity, h.Network.Since, + add(k.finding(scopeMachine, node, KindMachineNetwork, "network", node, severity, h.Network.Since, fmt.Sprintf("%s's network is not healthy: %s", node, strings.Join(words, "; ")), strings.Join(said, " | "))) } // A tunnel peer this node does not see points at that peer's machine as a tunnel part toward it would. @@ -411,7 +417,7 @@ func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) { severity = condseat.FindingSeverityUrgent } } - f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, x, severity, since, + f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, "unreachable", x, severity, since, fmt.Sprintf("%s cannot be reached from %s: %s", x, node, strings.Join(what, ", ")), strings.Join(said, " | ")) f.Subject.Also = []string{node} add(f) @@ -427,7 +433,7 @@ func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) { } if f := h.Fallback; f != nil && f.Node == node { said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why) - g := k.finding(scopeMachine, node, KindAccountFallback, node, condseat.FindingSeverityWarning, f.At, said, said) + g := k.finding(scopeMachine, node, KindAccountFallback, "", node, condseat.FindingSeverityWarning, f.At, said, said) g.Headline = fmt.Sprintf("%s fell back to its old bus login", node) g.Needs = fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node) add(g) @@ -460,7 +466,7 @@ func (k *NodeKeeper) rewritten(p nodeengine.NetworkPart, all []nodeengine.Networ if k.control() { severity = condseat.FindingSeverityUrgent } - return k.finding(scopeMachine, id, KindNetworkRewritten, m, severity, p.Since, summary, + return k.finding(scopeMachine, id, KindNetworkRewritten, "rewritten", m, severity, p.Since, summary, fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))) } @@ -500,7 +506,7 @@ func (k *NodeKeeper) units(u *nodeengine.UnitsHealth) (condseat.Finding, bool) { if len(u.Failed) > 1 { failed = fmt.Sprintf("%d failed services", len(u.Failed)) } - f := k.finding(scopeMachine, node, KindMachineUnits, node, condseat.FindingSeverityWarning, since, + f := k.finding(scopeMachine, node, KindMachineUnits, "units", node, condseat.FindingSeverityWarning, since, fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+ "mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")), strings.Join(said, "; ")) @@ -534,7 +540,7 @@ func (k *NodeKeeper) relogin(accounts []nodeengine.NodeAccount) []condseat.Findi sort.Strings(names) said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+ "before it was; log out of %[1]s completely and log in again, or restart it", k.Node, orModule(module), strings.Join(names, ", ")) - out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, k.Node, condseat.FindingSeverityWarning, since, said, said)) + out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, "", k.Node, condseat.FindingSeverityWarning, since, said, said)) } return out } @@ -557,7 +563,7 @@ func (k *NodeKeeper) found(dirs []nodeengine.NodeDirectory) []condseat.Finding { } m := orModule(module) said := fmt.Sprintf("%s on %s uses %s as found: %s", m, k.Node, strings.Join(ids, ", "), strings.Join(why, "; ")) - f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, k.Node, condseat.FindingSeverityWarning, since, said, said) + f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, "", k.Node, condseat.FindingSeverityWarning, since, said, said) f.Headline = fmt.Sprintf("%s waits for a directory on %s", m, k.Node) f.Explanation = fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+ "and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, k.Node, m) diff --git a/internal/seatwait/nodefindings_test.go b/internal/seatwait/nodefindings_test.go index 8324bc7e..78611246 100644 --- a/internal/seatwait/nodefindings_test.go +++ b/internal/seatwait/nodefindings_test.go @@ -33,7 +33,7 @@ func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) { }, "a have unhealthy past a minute is module-unhealthy, cleared when it is healthy or removed": { {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", sick) }, nil}, - {func(k *NodeKeeper, adv func(time.Duration)) { adv(time.Minute); k.Have("b", sick) }, []string{"module.b.ace.module-unhealthy"}}, + {func(k *NodeKeeper, adv func(time.Duration)) { adv(time.Minute); k.Have("b", sick) }, []string{"module.b.ace.unhealthy.module-unhealthy"}}, {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRemoved}) }, nil}, @@ -41,7 +41,7 @@ func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) { "what a module keeps as it runs is kept-as-it-runs until its have says none": { {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Kept: []string{"tool-runner"}}) - }, []string{"module.b.ace.kept-as-it-runs"}}, + }, []string{"module.b.ace.kept.kept-as-it-runs"}}, {func(k *NodeKeeper, _ func(time.Duration)) { k.HaveGone("b") }, nil}, }, "each part of the report is its own finding and each ends with the report": { @@ -54,14 +54,14 @@ func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) { Relogin: []nodeengine.NodeAccount{{Account: "u", Module: "m", Since: at}}, Found: []nodeengine.NodeDirectory{{Module: "m", Resource: "m.dir", Since: at}}, Fallback: &nodeengine.LoginFallback{Node: "ace", From: "new", At: at}}), - []string{"machine.ace.dnsmasq.network-rewritten", "machine.ace.machine-network", "machine.ace.machine-units", - "machine.ace.node-account-fallback", "machine.g14.ace.network-unreachable", "module.m.ace.directory-used-as-found", + []string{"machine.ace.dnsmasq.rewritten.network-rewritten", "machine.ace.network.machine-network", "machine.ace.node-account-fallback", + "machine.ace.units.machine-units", "machine.g14.ace.unreachable.network-unreachable", "module.m.ace.directory-used-as-found", "module.m.ace.relogin-needed"}}, {report(nodeengine.HealthByNodeValue{}), nil}, }, "a tunnel peer not seen is the machine unreachable": { {report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}}}}), - []string{"machine.g14.ace.network-unreachable"}}, + []string{"machine.g14.ace.unreachable.network-unreachable"}}, {report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}, Seen: true}}}), nil}, }, } {