diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 6a99ed7b..fea28393 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -3,6 +3,7 @@ package watch import ( "encoding/json" "errors" + "slices" "strings" "sync/atomic" "testing" @@ -602,6 +603,23 @@ func TestTheWatchmansFindings(t *testing.T) { if _, ok := o.findings["machine.hub.unreachable"]; ok { t.Error("the silent hub is said unreachable as well") } + // design 47 G §706: the silence lists the machines failing toward it in `also`, as `unreachable` does. + if f := o.findings["machine.hub.silent"]; !slices.Equal(f.Also, []string{"lap"}) { + t.Errorf("the hub's silence lists %v in also", f.Also) + } + w.NodeHealth("lap", nil, Delete) + if f := o.findings["machine.hub.silent"]; len(f.Also) != 0 { + t.Errorf("the hub's silence still lists %v once nothing fails toward it", f.Also) + } + w.NodeHealth("lap", toward("lap", "tunnel", "hub"), Put) + // the hub's own network open, two machines failing toward it: its `network` finding lists both. + w.NodeHealth("hub", toward("hub", "bus"), Put) + w.NodeHealth("ace", toward("ace", "tunnel", "hub"), Put) + if f := o.findings["machine.hub.network"]; !slices.Equal(f.Also, []string{"ace", "lap"}) { + t.Errorf("the hub's network lists %v in also: %v", f.Also, o.findings) + } + w.NodeHealth("hub", nil, Delete) + w.NodeHealth("ace", nil, Delete) w.Connect("node.hub") // back: lap's tunnel failing toward a hub up is its own again if _, own := o.findings["machine.lap.network"]; !own { t.Errorf("hub back: findings %v", o.findings) diff --git a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go index da6adfe1..7295e1b5 100644 --- a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go +++ b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go @@ -274,6 +274,34 @@ func orModule(m string) string { return m } +// silentAlso sets the machines `also` lists on the open `silent` finding of x, when there is one. +func (w *Watchman) silentAlso(x string, also []string) { + value, ok := w.findings[FindingKey(ScopeMachine, x, KindSilent)] + if !ok { + return + } + f, err := DecodeFinding(value) + if err != nil || slices.Equal(f.Also, also) { + return + } + f.Also = also + w.find(f) +} + +// keepSilentAlso gives a `silent` finding about to be written the `also` the open one holds, so a write of its other +// words (its urgency after half an hour) never drops the machines listed there. +func (w *Watchman) keepSilentAlso(f Finding) Finding { + if f.Kind != KindSilent { + return f + } + if value, ok := w.findings[f.Key()]; ok { + if cur, err := DecodeFinding(value); err == nil { + f.Also = cur.Also + } + } + return f +} + // pointed is one machine's failing part that points at another machine. type pointed struct { from string @@ -355,6 +383,13 @@ func (w *Watchman) networks() { } return len(xs) > 0 } + // A machine silent on the record holds the failures toward it under its silence, so its finding lists in `also` the + // machines whose failing parts point at it, as `unreachable` does (design 47 G §706); a change of `also` rewrites it. + for key, value := range w.findings { + if f, err := DecodeFinding(value); err == nil && f.Kind == KindSilent && key == f.Key() { + w.silentAlso(f.ID, from(f.ID)) + } + } want := map[string]bool{} saysOwn := map[string]bool{} for _, m := range machines { @@ -406,7 +441,7 @@ func (w *Watchman) networks() { severity = Urgent summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", ")) } - f := Finding{Scope: ScopeMachine, ID: m, Token: "network", Kind: KindMachineNetwork, Machine: m, Severity: severity, + f := Finding{Scope: ScopeMachine, ID: m, Token: "network", Kind: KindMachineNetwork, Machine: m, Also: from(m), Severity: severity, Source: "health-report", Since: w.health[m].Network.Since, Summary: summary, Said: strings.Join(said, " | ")} want[f.Key()] = true saysOwn[m] = true diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index e72c31da..c82d83ad 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -179,7 +179,7 @@ func (w *Watchman) signalExpired(pk string) { if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) { return } - w.find(due.Finding) + w.find(w.keepSilentAlso(due.Finding)) } } @@ -288,7 +288,7 @@ func (w *Watchman) stuckFinding(sk string, s meshwatchdogsclient.Stuck) { w.signal("S1", node, ControlNodeUrgentAfter-age+time.Second, signalDue{Do: dueFind, While: "silent:" + node, Finding: u}) } } - w.find(f) + w.find(w.keepSilentAlso(f)) w.applyCheck(node) return }