Raise the bus module's watch when it falls silent or says it cannot see (hq issue 270, review 274)
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head d24132d54f01 merged into trunk 04730b54a8f2 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head d24132d54f01 merged into trunk 04730b54a8f2 — pass: its merge-check.sh passed

A dead watch of slow consumers reads as none. The watch now beats after every
look; S9 reads the newest heartbeat and raises bus-watch-dead when it is older
than two minutes or says the watch is not healthy, on the bus's machine. A
stream with no heartbeat at all is a bus module older than its watch.
This commit is contained in:
jochen
2026-10-14 09:19:57 +02:00
parent 04730b54a8
commit d24132d54f
6 changed files with 136 additions and 0 deletions
+33
View File
@@ -32,6 +32,9 @@ const BusModule = "nats"
const (
BusSlowConsumerEvent = "bus.slow-consumer"
BusConnectionClosedEvent = "bus.connection-closed"
// BusWatchEvent is the bus module's watch saying it is there after every look, and whether it is healthy
// (novox/hq ADR 0357, review 274): a watch that stops saying it is a watch nobody would otherwise hear.
BusWatchEvent = "bus.watch"
)
// AdvisoryConnectionClosed is a connection the bus closed for breaking its rules, as conditions name it.
@@ -233,3 +236,33 @@ func MergeAdvisories(a, b []Advisory) []Advisory {
}
return out
}
// BusWatchBeat is the bus module's watch's newest heartbeat.
type BusWatchBeat struct {
At time.Time `json:"at"`
Healthy bool `json:"healthy"`
Why string `json:"why,omitempty"`
// Machine is the machine the runtime stamped on it (`x-node`).
Machine string `json:"-"`
}
// BusWatchLast is the newest heartbeat of the bus module's watch, read directly from the events stream; false
// when the stream holds none — a bus module older than its watch, which says nothing either way.
func (s *Server) BusWatchLast(ctx context.Context) (BusWatchBeat, bool, error) {
if s.js == nil {
return BusWatchBeat{}, false, errors.New("this control plane is not on the bus, so it cannot read the bus module's watch")
}
msg, err := s.js.Context().GetLastMsg(broker.EventsStream, EventSubject(BusModule, BusWatchEvent), nats.Context(ctx))
if errors.Is(err, nats.ErrMsgNotFound) {
return BusWatchBeat{}, false, nil
}
if err != nil {
return BusWatchBeat{}, false, fmt.Errorf("reading the bus module's watch's heartbeat: %w", err)
}
var b BusWatchBeat
if err := json.Unmarshal(msg.Data, &b); err != nil || b.At.IsZero() {
b = BusWatchBeat{At: msg.Time, Why: "its heartbeat could not be read"}
}
b.Machine = msg.Header.Get("x-node")
return b, true, nil
}