novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

mesh-watcher stops consuming the controller's doctor-heartbeat (ADR 0564 clause 9)

Browse Source 
The watcher hears the self-check's `checked` alone: the consume of mesh-controller.doctor-heartbeat, the subscription
to the controller's seat, the follow switch (takeover.go) and its read of mesh-conditions.takeover are gone. The
manifest test now lists the one consume and the heartbeat-key test refuses the controller's event. No test added.

(issue 774, 774/S11)

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-11-02 00:18:29 +01:00
parent 432de55186
commit 204a02240c
7 changed files with 21 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-watcher/cmd/mesh-watcher/heartbeat.go
+12 -12
View File
@@ -2,33 +2,33 @@ package main
// The self-check's heartbeat, read in this one place (novox/hq to-be 45 §4, S10).
//
// **The contract this reads, and every assumption it makes**, because the controller's side was built
// **The contract this reads, and every assumption it makes**, because the self-check's side was built
// at the same time from the same design, which says only "every run ends with a heartbeat event
// carrying the run's id and counts":
//
// - The event is the mesh-controller seat's own, `doctor-heartbeat`, consumed as
// `mesh-controller.doctor-heartbeat` (subject `mesh.seat.mesh-controller.event.doctor-heartbeat`).
// - The event is the seat mesh-self-check's `checked`, consumed as `mesh-self-check.checked` (subject
// `mesh.seat.mesh-self-check.event.checked`). The controller's `doctor-heartbeat` is no longer heard (novox/hq ADR
// 0564 clause 9, issue 774).
// - The body is one JSON object: `run` (the run's id), `at` (when the run ended, RFC 3339),
// `interval-seconds` (how often doctor runs), and `counts` ({pass, fail, failed-to-run}).
// `interval-seconds` (how often the check runs), and `counts` ({pass, fail, failed-to-run}).
// `finished`/`time` are read for `at`; `interval_seconds`, `interval` (seconds, or a duration such
// as "5m") for the interval.
// - Without a time, the heartbeat is taken as made when it arrived (said in the status as such).
// Without an interval, the design's five minutes stand.
// - The counts are kept for the status only. A run that found failures is still a heartbeat: the
// watcher watches that the checker runs; what it finds is the controller's to raise.
// watcher watches that the checker runs; what it finds is the checker's to raise.
import (
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// HeartbeatEvent is the event's local name under the controller's seat.
const (
HeartbeatEvent = "doctor-heartbeat"
ControllerSeat = "mesh-controller"
)
// HeartbeatEvent is the event's local name under the seat mesh-self-check.
const HeartbeatEvent = health.EventChecked
// Beat is one heartbeat.
type Beat struct {
@@ -38,8 +38,8 @@ type Beat struct {
Counts map[string]int
}
// isHeartbeat says whether an envelope's key is the controller's heartbeat.
func isHeartbeat(key string) bool { return key == ControllerSeat+"."+HeartbeatEvent }
// isHeartbeat says whether an envelope's key is the self-check's heartbeat.
func isHeartbeat(key string) bool { return key == health.SelfCheckModule+"."+HeartbeatEvent }
// DecodeBeat reads one heartbeat's body.
func DecodeBeat(body []byte) (Beat, error) {
modules/mesh-watcher/cmd/mesh-watcher/main.go
+5 -10
View File
@@ -1,6 +1,6 @@
// mesh-watcher: the watcher's watcher (novox/hq to-be 45 §5, S10, ADR 0227 rule 6). A Go bundle the
// node's runtime launches on one machine that is not the control node. It hears the controller's
// self-check heartbeat and makes a round trip through the bus every minute; when either goes silent
// node's runtime launches on one machine that is not the control node. It hears the self-check's
// heartbeat and makes a round trip through the bus every minute; when either goes silent
// past its bound, it tells the operator on Telegram directly over HTTPS — the one sender that does not
// pass through the control node — and tells them again when it returns. stdout is the MCP channel;
// what this module says, it says on stderr.
@@ -100,10 +100,8 @@ func run(w *Watcher, l *listening, node string) {
time.Sleep(10 * time.Minute)
}
}()
follow := newTakeover()
go followTakeover(follow)
handle := func(e stdio.Envelope) error {
if !follow.heard(e.Key) {
if !isHeartbeat(e.Key) {
return nil
}
b, err := DecodeBeat(e.Body)
@@ -116,10 +114,7 @@ func run(w *Watcher, l *listening, node string) {
}
<-stdio.Served() // subscribed as soon as the runtime serves, never after a guessed wait: a heartbeat sent meanwhile is heard
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.Subscribe(ControllerSeat+".*", handle)
if err == nil {
err = stdio.Subscribe(health.SelfCheckModule+".*", handle)
}
err := stdio.Subscribe(health.SelfCheckModule+".*", handle)
if err == nil {
l.set("listening")
logf("[mesh-watcher] listening for the self-check's heartbeat; watching from %s", node)
@@ -199,7 +194,7 @@ func tools(w *Watcher, l *listening) []stdio.Tool {
{Name: "watcher_status",
Description: "The watcher's own health first — whether it can tell the operator anything (the Telegram token and " +
"chat id), whether it runs on the machine it watches, whether heartbeats reach it — then each watched " +
"signal (the controller's self-check heartbeat, a round trip through the bus): last heard, how long ago, " +
"signal (the self-check's heartbeat, a round trip through the bus): last heard, how long ago, " +
"its bound, whether it is said silent, and any message not sent yet.",
Run: func(map[string]any) (any, error) { return w.Status(l.get()), nil }},
{Name: "watcher_last_heard",
modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go
+1 -1
View File
@@ -34,7 +34,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(m.Consumes, []string{ControllerSeat + "." + HeartbeatEvent, "mesh-self-check.checked"}) {
if !reflect.DeepEqual(m.Consumes, []string{"mesh-self-check.checked"}) {
t.Fatalf("consumes %v", m.Consumes)
}
if !reflect.DeepEqual(m.Invokes, []string{m.Module + ".watcher_ping"}) {
modules/mesh-watcher/cmd/mesh-watcher/takeover.go
-59
View File
@@ -1,59 +0,0 @@
package main
import (
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
// takeover is which self-check run the watcher hears: the controller's `doctor-heartbeat` until the keeper's
// take-over key names the keeper, then mesh-self-check's `checked`, whose body is the same run (novox/hq ADR 0564
// clause 9, issue 1952 W1). An absent key follows the controller.
type takeover struct {
mu sync.Mutex
t health.Takeover
}
func newTakeover() *takeover {
return &takeover{t: health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}}
}
// heard says whether key is the followed self-check event.
func (f *takeover) heard(key string) bool {
f.mu.Lock()
defer f.mu.Unlock()
seat, event := f.t.SelfCheckEvent()
return key == seat+"."+event
}
func (f *takeover) set(value []byte, found bool) error {
t, err := health.DecodeTakeover(value, found)
if err != nil {
return err
}
f.mu.Lock()
f.t = t
f.mu.Unlock()
return nil
}
// followTakeover watches the key for the life of the process; one it cannot read leaves the event it heard.
func followTakeover(f *takeover) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.State(health.KeeperModule+"."+health.TakeoverState).Watch(health.TakeoverKey, func(c stdio.StateChange) error {
if err := f.set(c.Value, c.Op != "delete" && len(c.Value) > 0); err != nil {
logf("[mesh-watcher] the take-over key is unreadable (%v); no change in what it hears", err)
} else {
logf("[mesh-watcher] the take-over key now follows %s", f.t.Follow)
}
return nil
})
if err == nil {
return
}
logf("[mesh-watcher] not watching the take-over key yet (%v)", err)
time.Sleep(wait)
}
}
modules/mesh-watcher/cmd/mesh-watcher/watcher.go
+2 -2
View File
@@ -7,7 +7,7 @@ package main
//
// Two signals:
//
// - self-check: the controller's `doctor` heartbeat. Bound: twice the interval the heartbeat says
// - self-check: the self-check's heartbeat (mesh-self-check.checked). Bound: twice the interval the heartbeat says
// it runs at, else twice five minutes (to-be 45 §3, S10). Heard by the time the heartbeat says it
// was made, not by when it arrived: a backlog delivered after a restart is old news, not a sign of
// life.
@@ -224,7 +224,7 @@ func (w *Watcher) Tick() {
func (w *Watcher) what(s *signal) string {
if s.name == SelfCheck {
return "the controller's self-check (doctor)"
return "the self-check"
}
return "the bus"
}
modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go
+1 -1
View File
@@ -253,7 +253,7 @@ func TestTheHeartbeatAsTheDesignSaysIt(t *testing.T) {
t.Errorf("%s: %v", body, err)
}
}
if !isHeartbeat("mesh-controller.doctor-heartbeat") || isHeartbeat("mesh-controller.applied") {
if !isHeartbeat("mesh-self-check.checked") || isHeartbeat("mesh-controller.doctor-heartbeat") {
t.Fatalf("the heartbeat's key")
}
}
modules/mesh-watcher/module.json
-2
View File
@@ -3,11 +3,9 @@
"version": "1",
"slug": "watch",
"consumes": [
"mesh-controller.doctor-heartbeat",
"mesh-self-check.checked"
],
"reads": [
"mesh-conditions.takeover",
"mesh-delivery-assigner.seat-holders"
],
"invokes": [