novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Recognise the control node by the bus module, not the controller (issue 1126)

Browse Source 
The watchman held a node a control node, urgent after half an hour
silent, while its have named mesh-controller; the controller ended by
its unassignment (ADR 0590), so no node was one any more and the bus
node's silence stayed a warning. The control node is now the node whose
have holds the bus module, nats, which the watchman already hears by
that name.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-11-02 11:47:53 +01:00
parent 8d4c13427a
commit 7e358b0f1d
2 changed files with 19 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-watchman/internal/watch/findings_test.go
+12 -3
View File
@@ -249,7 +249,7 @@ func TestTheWatchmansFindings(t *testing.T) {
}
return b
}
w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node
w.Have("novox.nats", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node: the bus node
w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox", Network: &delivery.NetworkHealth{State: "unhealthy", Parts: []delivery.NetworkPart{
{Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"},
{Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"},
@@ -341,7 +341,16 @@ func TestTheWatchmansFindings(t *testing.T) {
w.Have("ace.mail", haveOf(t, delivery.PhaseRemoved, delivery.HealthUnknown, ""), Put)
}, map[string]string{}},
{"WM2 S1: a control node silent is urgent after half an hour", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put)
// 1126 (ADR 0590): the control node is the bus node, the one whose have holds the bus module; a have of the
// ended controller names none.
w.Have("g14.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put)
w.Connect("node.g14")
w.Disconnect("node.g14")
expire(w, c, pulseKeyOf(t, presenceNS, "g14"), 120*time.Second)
if _, ok := w.signals[signalKey("S1", "g14")]; ok {
t.Fatalf("a node running the controller's have is held a control node")
}
w.Have("novox.nats", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put)
w.Connect("node.novox")
w.Disconnect("node.novox")
expire(w, c, pulseKeyOf(t, presenceNS, "novox"), 120*time.Second)
@@ -349,7 +358,7 @@ func TestTheWatchmansFindings(t *testing.T) {
t.Fatalf("findings = %v", o.findings)
}
expire(w, c, signalKey("S1", "novox"), ControlNodeUrgentAfter)
}, map[string]string{"machine.novox.silent": Urgent}},
}, map[string]string{"machine.novox.silent": Urgent, "machine.g14.silent": Warning}},
{"WM3 S9: the bus module's closures, each counted once, ended an hour after the newest", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
cl := BusClosure{User: "ace.web", CID: 7, Reason: "Slow Consumer (Write Deadline)", Opened: t0.Add(-time.Hour), Closed: t0}
w.Closed(KindSlowConsumer, cl)
modules/mesh-delivery-watchman/internal/watch/signals.go
+7 -5
View File
@@ -252,7 +252,7 @@ func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) {
f := Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1",
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), SilentBound),
Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since}
// A control node, the machine running the controller, is urgent after half an hour silent, as the controller
// A control node, the bus node (ControlNodeModule), is urgent after half an hour silent, as the controller
// says it (its watchHeartbeats): the expiry of a pulse from its silence's start, never a loop.
if w.control[node] {
if age := w.now().Sub(s.Since); age > ControlNodeUrgentAfter {
@@ -459,14 +459,14 @@ func ToolsNode(subject string) string {
// ---- S21 and unhealthy, from the haves and the provisioner clients' reports ----
// haveFindings reads a have's change for the rows the haves feed: S21 (the machine's applies failing, applyCheck),
// unhealthy (two writes in a row, urgent after four hours), and where the controller runs (S1's control node). A have
// unhealthy (two writes in a row, urgent after four hours), and where the bus module runs (S1's control node). A have
// removed or deleted ends its part of them.
//
// needs-operator is not observed from the haves: a module's part that waits for the operator is a provision in the state
// `waits` of the seat mesh-provisioner (Provision, ADR 0570 §7).
func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Have) {
unhealthyKey := FindingKey(ScopeModule, module+"."+node, "unhealthy")
if module == ControllerModule {
if module == ControlNodeModule {
w.control[node] = h != nil && h.Phase != delivery.PhaseRemoved
}
if h == nil || h.Phase == delivery.PhaseRemoved {
@@ -531,8 +531,10 @@ func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Hav
}
}
// ControllerModule is the module whose have names a control node.
const ControllerModule = "mesh-controller"
// ControlNodeModule is the module whose have names a control node: the bus module, so the control node is the bus node,
// the one stably reachable node every part depends on (novox/hq ADR 0590, issue 1126: the controller that named it ended).
// The same name the bus module's events are heard under (SlowConsumerSubject).
const ControlNodeModule = "nats"
// Provision is a change of a key of mesh-provisioner's `provisions` (ADR 0570 §7): a provision in `waits` is a part of its
// consumer waiting for the operator, said as `module.<consumer>.<node>.needs-operator` naming each waiting provision and