Raise the bus module's watch when it falls silent or says it cannot see (hq issue 270, review 274)
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head d24132d54f01 merged into trunk 04730b54a8f2 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head d24132d54f01 merged into trunk 04730b54a8f2 — pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head d24132d54f01 merged into trunk 04730b54a8f2 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head d24132d54f01 merged into trunk 04730b54a8f2 — pass: its merge-check.sh passed
A dead watch of slow consumers reads as none. The watch now beats after every look; S9 reads the newest heartbeat and raises bus-watch-dead when it is older than two minutes or says the watch is not healthy, on the bus's machine. A stream with no heartbeat at all is a bus module older than its watch.
This commit is contained in:
@@ -6,6 +6,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
@@ -119,3 +121,27 @@ func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) {
|
||||
t.Fatalf("a read past its bound read as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The bus module's watch is heard by its heartbeat** (review 274): the newest is read directly, with the
|
||||
// machine the runtime stamped on it; none at all is a bus module older than its watch.
|
||||
func TestNatsTheBusWatchsNewestHeartbeatIsRead(t *testing.T) {
|
||||
s := aBusWithEvents(t)
|
||||
if _, heard, err := s.BusWatchLast(t.Context()); err != nil || heard {
|
||||
t.Fatalf("a stream with no heartbeat read as heard %v, %v", heard, err)
|
||||
}
|
||||
at := time.Now().UTC().Truncate(time.Second)
|
||||
for i, healthy := range []bool{true, false} {
|
||||
msg := nats.NewMsg(EventSubject(BusModule, BusWatchEvent))
|
||||
msg.Header.Set("x-node", "anchor")
|
||||
msg.Data, _ = json.Marshal(map[string]any{"at": at.Add(time.Duration(i) * time.Second), "healthy": healthy,
|
||||
"why": "the watch of slow consumers last read the bus's closed connections 95s ago"})
|
||||
if _, err := s.js.Context().PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
b, heard, err := s.BusWatchLast(t.Context())
|
||||
if err != nil || !heard || b.Healthy || b.Machine != "anchor" || !b.At.Equal(at.Add(time.Second)) ||
|
||||
!strings.Contains(b.Why, "95s ago") {
|
||||
t.Fatalf("read %+v, %v, %v", b, heard, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user