diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 91e165f..9c291e5 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -249,7 +249,7 @@ func TestTheWatchmansFindings(t *testing.T) { } return b } - w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node + w.Have("novox.nats", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node: the bus node w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox", Network: &delivery.NetworkHealth{State: "unhealthy", Parts: []delivery.NetworkPart{ {Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"}, {Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"}, @@ -341,7 +341,16 @@ func TestTheWatchmansFindings(t *testing.T) { w.Have("ace.mail", haveOf(t, delivery.PhaseRemoved, delivery.HealthUnknown, ""), Put) }, map[string]string{}}, {"WM2 S1: a control node silent is urgent after half an hour", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) + // 1126 (ADR 0590): the control node is the bus node, the one whose have holds the bus module; a have of the + // ended controller names none. + w.Have("g14.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) + w.Connect("node.g14") + w.Disconnect("node.g14") + expire(w, c, pulseKeyOf(t, presenceNS, "g14"), 120*time.Second) + if _, ok := w.signals[signalKey("S1", "g14")]; ok { + t.Fatalf("a node running the controller's have is held a control node") + } + w.Have("novox.nats", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) w.Connect("node.novox") w.Disconnect("node.novox") expire(w, c, pulseKeyOf(t, presenceNS, "novox"), 120*time.Second) @@ -349,7 +358,7 @@ func TestTheWatchmansFindings(t *testing.T) { t.Fatalf("findings = %v", o.findings) } expire(w, c, signalKey("S1", "novox"), ControlNodeUrgentAfter) - }, map[string]string{"machine.novox.silent": Urgent}}, + }, map[string]string{"machine.novox.silent": Urgent, "machine.g14.silent": Warning}}, {"WM3 S9: the bus module's closures, each counted once, ended an hour after the newest", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { cl := BusClosure{User: "ace.web", CID: 7, Reason: "Slow Consumer (Write Deadline)", Opened: t0.Add(-time.Hour), Closed: t0} w.Closed(KindSlowConsumer, cl) diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index e28ec1e..e5bc234 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -252,7 +252,7 @@ func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) { f := Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1", Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), SilentBound), Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since} - // A control node, the machine running the controller, is urgent after half an hour silent, as the controller + // A control node, the bus node (ControlNodeModule), is urgent after half an hour silent, as the controller // says it (its watchHeartbeats): the expiry of a pulse from its silence's start, never a loop. if w.control[node] { if age := w.now().Sub(s.Since); age > ControlNodeUrgentAfter { @@ -459,14 +459,14 @@ func ToolsNode(subject string) string { // ---- S21 and unhealthy, from the haves and the provisioner clients' reports ---- // haveFindings reads a have's change for the rows the haves feed: S21 (the machine's applies failing, applyCheck), -// unhealthy (two writes in a row, urgent after four hours), and where the controller runs (S1's control node). A have +// unhealthy (two writes in a row, urgent after four hours), and where the bus module runs (S1's control node). A have // removed or deleted ends its part of them. // // needs-operator is not observed from the haves: a module's part that waits for the operator is a provision in the state // `waits` of the seat mesh-provisioner (Provision, ADR 0570 §7). func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Have) { unhealthyKey := FindingKey(ScopeModule, module+"."+node, "unhealthy") - if module == ControllerModule { + if module == ControlNodeModule { w.control[node] = h != nil && h.Phase != delivery.PhaseRemoved } if h == nil || h.Phase == delivery.PhaseRemoved { @@ -531,8 +531,10 @@ func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Hav } } -// ControllerModule is the module whose have names a control node. -const ControllerModule = "mesh-controller" +// ControlNodeModule is the module whose have names a control node: the bus module, so the control node is the bus node, +// the one stably reachable node every part depends on (novox/hq ADR 0590, issue 1126: the controller that named it ended). +// The same name the bus module's events are heard under (SlowConsumerSubject). +const ControlNodeModule = "nats" // Provision is a change of a key of mesh-provisioner's `provisions` (ADR 0570 §7): a provision in `waits` is a part of its // consumer waiting for the operator, said as `module...needs-operator` naming each waiting provision and