mesh-catalog/modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go

package main

import (
	"errors"
	"strings"
	"testing"
	"time"
)

type fakeTelegram struct {
	notReady error
	fail     error
	sent     []Message
}

func (f *fakeTelegram) Ready() error { return f.notReady }
func (f *fakeTelegram) Send(m Message) (string, error) {
	if f.fail != nil {
		return "", f.fail
	}
	f.sent = append(f.sent, m)
	return "1", nil
}

type clock struct{ t time.Time }

func (c *clock) now() time.Time       { return c.t }
func (c *clock) pass(d time.Duration) { c.t = c.t.Add(d) }

func watcher(t *testing.T) (*Watcher, *fakeTelegram, *clock) {
	c := &clock{t: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)}
	tg := &fakeTelegram{}
	return NewWatcher(tg, c.now, t.Logf, "ace"), tg, c
}

// beatEvery has the controller's doctor run, and the bus answer, every five minutes up to d.
func beatEvery(w *Watcher, c *clock, d time.Duration) {
	for end := c.t.Add(d); c.t.Before(end); {
		c.pass(time.Minute)
		if c.t.Minute()%5 == 0 {
			w.Heartbeat(Beat{Run: "r", At: c.t, Interval: 5 * time.Minute})
		}
		w.BusAnswered(nil)
		w.Tick()
	}
}

func TestHeardSignalsSayNothing(t *testing.T) {
	w, tg, c := watcher(t)
	beatEvery(w, c, 3*time.Hour)
	if len(tg.sent) != 0 {
		t.Fatalf("sent %v", tg.sent)
	}
	if st := w.Status("listening"); st.Verdict != "ok: watching" {
		t.Fatalf("%+v", st)
	}
}

func TestTheSelfCheckSilentPastTwiceItsIntervalIsSentOnceThenOnceMoreThenItsReturn(t *testing.T) {
	w, tg, c := watcher(t)
	beatEvery(w, c, 30*time.Minute)
	last := c.t
	// The controller stops; the bus still answers.
	for i := 0; i < 9; i++ {
		c.pass(time.Minute)
		w.BusAnswered(nil)
		w.Tick()
	}
	if len(tg.sent) != 0 {
		t.Fatalf("said silent within its bound (9 min of 10): %v", tg.sent)
	}
	for i := 0; i < 2; i++ {
		c.pass(time.Minute)
		w.BusAnswered(nil)
		w.Tick()
	}
	if len(tg.sent) != 1 || !strings.Contains(tg.sent[0].Title, "self-check-silent") {
		t.Fatalf("not said silent past twice the interval: %v", tg.sent)
	}
	if !strings.Contains(tg.sent[0].Body, "directly over HTTPS") || !strings.Contains(tg.sent[0].Body, last.Format("15:04")) {
		t.Fatalf("body: %q", tg.sent[0].Body)
	}
	if st := w.Status("listening"); !strings.HasPrefix(st.Verdict, "ALARM") {
		t.Fatalf("%+v", st)
	}
	for i := 0; i < 70; i++ {
		c.pass(time.Minute)
		w.BusAnswered(nil)
		w.Tick()
	}
	if len(tg.sent) != 2 || !strings.HasPrefix(tg.sent[1].Title, "STILL SILENT") {
		t.Fatalf("not said once more after an hour: %v", tg.sent)
	}
	beatEvery(w, c, 10*time.Minute)
	if len(tg.sent) != 3 || !strings.HasPrefix(tg.sent[2].Title, "CLEARED") {
		t.Fatalf("its return not said: %v", tg.sent)
	}
	beatEvery(w, c, time.Hour)
	if len(tg.sent) != 3 {
		t.Fatalf("said again after it returned: %v", tg.sent)
	}
}

func TestAHeartbeatNeverHeardIsSilenceFromTheStart(t *testing.T) {
	w, tg, c := watcher(t)
	for i := 0; i < 11; i++ {
		c.pass(time.Minute)
		w.BusAnswered(nil)
		w.Tick()
	}
	if len(tg.sent) != 1 || !strings.Contains(tg.sent[0].Body, "not heard once since the watcher started") {
		t.Fatalf("%v", tg.sent)
	}
}

func TestAReplayedOldHeartbeatIsNotASignOfLife(t *testing.T) {
	w, tg, c := watcher(t)
	beatEvery(w, c, 10*time.Minute)
	old := c.t
	for i := 0; i < 11; i++ {
		c.pass(time.Minute)
		w.BusAnswered(nil)
		w.Tick()
	}
	// The watcher's consumer delivers a backlog: heartbeats made before the silence.
	w.Heartbeat(Beat{Run: "old", At: old.Add(-time.Minute)})
	w.Heartbeat(Beat{Run: "older", At: old.Add(-10 * time.Minute)})
	c.pass(time.Minute)
	w.Tick()
	if len(tg.sent) != 1 {
		t.Fatalf("a replay read as a return: %v", tg.sent)
	}
	w.Heartbeat(Beat{Run: "future", At: c.t.Add(time.Hour)})
	c.pass(time.Minute)
	w.Tick()
	if len(tg.sent) != 1 || w.Status("listening").BadBeats != 1 {
		t.Fatalf("a heartbeat from the future believed: %v", tg.sent)
	}
}

func TestTheBoundFollowsTheIntervalTheHeartbeatSays(t *testing.T) {
	w, tg, c := watcher(t)
	w.Heartbeat(Beat{At: c.t, Interval: 15 * time.Minute})
	for i := 0; i < 29; i++ {
		c.pass(time.Minute)
		w.BusAnswered(nil)
		w.Tick()
	}
	if len(tg.sent) != 0 {
		t.Fatalf("said silent inside 2 × 15 min: %v", tg.sent)
	}
	c.pass(2 * time.Minute)
	w.BusAnswered(nil)
	w.Tick()
	if len(tg.sent) != 1 {
		t.Fatalf("not said past 2 × 15 min")
	}
}

func TestTheBusSilentPastThreeMinutesIsSent(t *testing.T) {
	w, tg, c := watcher(t)
	beatEvery(w, c, 10*time.Minute)
	for i := 0; i < 4; i++ {
		c.pass(time.Minute)
		w.BusAnswered(errors.New("timeout"))
		w.Tick()
	}
	if len(tg.sent) != 1 || !strings.Contains(tg.sent[0].Title, "bus-silent") {
		t.Fatalf("%v", tg.sent)
	}
	if st := w.Status("listening"); st.Signals[1].LastFailure != "timeout" {
		t.Fatalf("%+v", st.Signals[1])
	}
}

func TestATelegramThatCannotSendIsSaidAndTheMessageIsOwed(t *testing.T) {
	w, tg, c := watcher(t)
	tg.fail = errors.New("telegram sendMessage: cannot connect (dial)")
	for i := 0; i < 11; i++ {
		c.pass(time.Minute)
		w.BusAnswered(nil)
		w.Tick()
	}
	st := w.Status("listening")
	if !strings.HasPrefix(st.Verdict, "BLIND") || st.Signals[0].Owed == "" {
		t.Fatalf("%+v", st)
	}
	tg.fail = nil
	c.pass(time.Minute)
	w.Tick()
	if len(tg.sent) != 1 || w.Status("listening").Signals[0].Owed != "" {
		t.Fatalf("the owed message was not sent: %v", tg.sent)
	}
}

func TestNotGivenATokenIsBlind(t *testing.T) {
	w, tg, _ := watcher(t)
	tg.notReady = errors.New("no bot token")
	if st := w.Status("listening"); st.Verdict != "BLIND: the watcher cannot tell the operator anything — CANNOT SEND: no bot token" {
		t.Fatalf("%q", st.Verdict)
	}
}

func TestOnTheControlNodeItSaysItIsMisplaced(t *testing.T) {
	w, _, _ := watcher(t)
	w.Placed([]string{"novox"})
	if st := w.Status("listening"); st.Verdict != "ok: watching" || st.Control != "novox" {
		t.Fatalf("%+v", st)
	}
	w.Placed([]string{"ace"})
	if st := w.Status("listening"); !strings.HasPrefix(st.Verdict, "MISPLACED") {
		t.Fatalf("%+v", st)
	}
}

func TestTheHoldersAreReadFromTheAssignmentsSeatsState(t *testing.T) {
	keys := map[string][]byte{
		"mesh-broker": []byte(`{"v":1,"seat":"mesh-broker","scope":"mesh","holders":[{"module":"nats","node":"n1"},{"module":"nats","node":"n2"},{"module":"nats","node":"n1"}]}`),
		"git":         []byte(`{"v":1,"seat":"git","scope":"mesh","holders":[{"module":"gitea","node":"n3"}]}`),
	}
	got, err := holderNodes(keys, "mesh-broker")
	if err != nil || strings.Join(got, ",") != "n1,n2" {
		t.Errorf("%v %v", got, err)
	}
	if _, err := holderNodes(map[string][]byte{}, "mesh-broker"); err == nil {
		t.Error("no holder named, and no error")
	}
}

func TestTheHeartbeatAsTheDesignSaysIt(t *testing.T) {
	b, err := DecodeBeat([]byte(`{"run":"doctor-41","at":"2026-10-06T12:05:00Z","interval-seconds":300,
		"counts":{"pass":10,"fail":1,"failed-to-run":0}}`))
	if err != nil || b.Run != "doctor-41" || b.Interval != 5*time.Minute || b.Counts["fail"] != 1 ||
		!b.At.Equal(time.Date(2026, 10, 6, 12, 5, 0, 0, time.UTC)) {
		t.Fatalf("%+v %v", b, err)
	}
	b, err = DecodeBeat([]byte(`{"run":7,"finished":"2026-10-06T12:05:00Z","interval":"5m"}`))
	if err != nil || b.Run != "7" || b.Interval != 5*time.Minute || b.At.IsZero() {
		t.Fatalf("%+v %v", b, err)
	}
	if b, err := DecodeBeat([]byte(`{}`)); err != nil || !b.At.IsZero() {
		t.Fatalf("an empty heartbeat: %+v %v", b, err)
	}
	for body, says := range map[string]string{
		`[]`:                      "not a JSON object",
		`{"at":"yesterday"}`:      "not an RFC 3339 time",
		`{"interval":"often"}`:    "not a duration",
		`{"interval-seconds":-5}`: "not one doctor runs at",
		`{"counts":"many"}`:       "counts",
	} {
		if _, err := DecodeBeat([]byte(body)); err == nil || !strings.Contains(err.Error(), says) {
			t.Errorf("%s: %v", body, err)
		}
	}
	if !isHeartbeat("mesh-self-check.checked") || isHeartbeat("mesh-controller.doctor-heartbeat") {
		t.Fatalf("the heartbeat's key")
	}
}