Raise the bus module's watch when it falls silent or says it cannot see (hq issue 270, review 274)
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head d24132d54f01 merged into trunk 04730b54a8f2 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head d24132d54f01 merged into trunk 04730b54a8f2 — pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head d24132d54f01 merged into trunk 04730b54a8f2 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head d24132d54f01 merged into trunk 04730b54a8f2 — pass: its merge-check.sh passed
A dead watch of slow consumers reads as none. The watch now beats after every look; S9 reads the newest heartbeat and raises bus-watch-dead when it is older than two minutes or says the watch is not healthy, on the bus's machine. A stream with no heartbeat at all is a bus module older than its watch.
This commit is contained in:
@@ -32,6 +32,9 @@ const BusModule = "nats"
|
||||
const (
|
||||
BusSlowConsumerEvent = "bus.slow-consumer"
|
||||
BusConnectionClosedEvent = "bus.connection-closed"
|
||||
// BusWatchEvent is the bus module's watch saying it is there after every look, and whether it is healthy
|
||||
// (novox/hq ADR 0357, review 274): a watch that stops saying it is a watch nobody would otherwise hear.
|
||||
BusWatchEvent = "bus.watch"
|
||||
)
|
||||
|
||||
// AdvisoryConnectionClosed is a connection the bus closed for breaking its rules, as conditions name it.
|
||||
@@ -233,3 +236,33 @@ func MergeAdvisories(a, b []Advisory) []Advisory {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BusWatchBeat is the bus module's watch's newest heartbeat.
|
||||
type BusWatchBeat struct {
|
||||
At time.Time `json:"at"`
|
||||
Healthy bool `json:"healthy"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Machine is the machine the runtime stamped on it (`x-node`).
|
||||
Machine string `json:"-"`
|
||||
}
|
||||
|
||||
// BusWatchLast is the newest heartbeat of the bus module's watch, read directly from the events stream; false
|
||||
// when the stream holds none — a bus module older than its watch, which says nothing either way.
|
||||
func (s *Server) BusWatchLast(ctx context.Context) (BusWatchBeat, bool, error) {
|
||||
if s.js == nil {
|
||||
return BusWatchBeat{}, false, errors.New("this control plane is not on the bus, so it cannot read the bus module's watch")
|
||||
}
|
||||
msg, err := s.js.Context().GetLastMsg(broker.EventsStream, EventSubject(BusModule, BusWatchEvent), nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
return BusWatchBeat{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return BusWatchBeat{}, false, fmt.Errorf("reading the bus module's watch's heartbeat: %w", err)
|
||||
}
|
||||
var b BusWatchBeat
|
||||
if err := json.Unmarshal(msg.Data, &b); err != nil || b.At.IsZero() {
|
||||
b = BusWatchBeat{At: msg.Time, Why: "its heartbeat could not be read"}
|
||||
}
|
||||
b.Machine = msg.Header.Get("x-node")
|
||||
return b, true, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user