Raise the bus module's watch when it falls silent or says it cannot see (hq issue 270, review 274)

A dead watch of slow consumers reads as none. The watch now beats after every
look; S9 reads the newest heartbeat and raises bus-watch-dead when it is older
than two minutes or says the watch is not healthy, on the bus's machine. A
stream with no heartbeat at all is a bus module older than its watch.

internal/link/busfaults.go

@@ -32,6 +32,9 @@ const BusModule = "nats"
 const (
 	BusSlowConsumerEvent     = "bus.slow-consumer"
 	BusConnectionClosedEvent = "bus.connection-closed"
+	// BusWatchEvent is the bus module's watch saying it is there after every look, and whether it is healthy
+	// (novox/hq ADR 0357, review 274): a watch that stops saying it is a watch nobody would otherwise hear.
+	BusWatchEvent = "bus.watch"
 )
 
 // AdvisoryConnectionClosed is a connection the bus closed for breaking its rules, as conditions name it.
@@ -233,3 +236,33 @@ func MergeAdvisories(a, b []Advisory) []Advisory {
 	}
 	return out
 }
+
+// BusWatchBeat is the bus module's watch's newest heartbeat.
+type BusWatchBeat struct {
+	At      time.Time `json:"at"`
+	Healthy bool      `json:"healthy"`
+	Why     string    `json:"why,omitempty"`
+	// Machine is the machine the runtime stamped on it (`x-node`).
+	Machine string `json:"-"`
+}
+
+// BusWatchLast is the newest heartbeat of the bus module's watch, read directly from the events stream; false
+// when the stream holds none — a bus module older than its watch, which says nothing either way.
+func (s *Server) BusWatchLast(ctx context.Context) (BusWatchBeat, bool, error) {
+	if s.js == nil {
+		return BusWatchBeat{}, false, errors.New("this control plane is not on the bus, so it cannot read the bus module's watch")
+	}
+	msg, err := s.js.Context().GetLastMsg(broker.EventsStream, EventSubject(BusModule, BusWatchEvent), nats.Context(ctx))
+	if errors.Is(err, nats.ErrMsgNotFound) {
+		return BusWatchBeat{}, false, nil
+	}
+	if err != nil {
+		return BusWatchBeat{}, false, fmt.Errorf("reading the bus module's watch's heartbeat: %w", err)
+	}
+	var b BusWatchBeat
+	if err := json.Unmarshal(msg.Data, &b); err != nil || b.At.IsZero() {
+		b = BusWatchBeat{At: msg.Time, Why: "its heartbeat could not be read"}
+	}
+	b.Machine = msg.Header.Get("x-node")
+	return b, true, nil
+}

internal/link/busfaults_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/nats-io/nats.go"
+
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/testbus"
 )
@@ -119,3 +121,27 @@ func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) {
 		t.Fatalf("a read past its bound read as %+v", got)
 	}
 }
+
+// **The bus module's watch is heard by its heartbeat** (review 274): the newest is read directly, with the
+// machine the runtime stamped on it; none at all is a bus module older than its watch.
+func TestNatsTheBusWatchsNewestHeartbeatIsRead(t *testing.T) {
+	s := aBusWithEvents(t)
+	if _, heard, err := s.BusWatchLast(t.Context()); err != nil || heard {
+		t.Fatalf("a stream with no heartbeat read as heard %v, %v", heard, err)
+	}
+	at := time.Now().UTC().Truncate(time.Second)
+	for i, healthy := range []bool{true, false} {
+		msg := nats.NewMsg(EventSubject(BusModule, BusWatchEvent))
+		msg.Header.Set("x-node", "anchor")
+		msg.Data, _ = json.Marshal(map[string]any{"at": at.Add(time.Duration(i) * time.Second), "healthy": healthy,
+			"why": "the watch of slow consumers last read the bus's closed connections 95s ago"})
+		if _, err := s.js.Context().PublishMsg(msg); err != nil {
+			t.Fatal(err)
+		}
+	}
+	b, heard, err := s.BusWatchLast(t.Context())
+	if err != nil || !heard || b.Healthy || b.Machine != "anchor" || !b.At.Equal(at.Add(time.Second)) ||
+		!strings.Contains(b.Why, "95s ago") {
+		t.Fatalf("read %+v, %v, %v", b, heard, err)
+	}
+}