jochen
0ae7933d54
The mesh noticed 48 core failures in six days and told nobody (ADR 0227). messenger holds the operator-channel seat: it consumes the controller's condition events and sends them to Telegram and the desktop notifier, deduplicated by key, reminded once, edited on clear, capped at 20 an hour with the rest folded, and refusing anything carrying an address, a path or a secret. mesh-watcher, on a machine other than the control node, sends to Telegram directly when the self-check heartbeat or the bus goes silent.
259 lines
7.7 KiB
Go
259 lines
7.7 KiB
Go
package main
|
||
|
||
import (
|
||
"encoding/json"
|
||
"errors"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
type fakeTelegram struct {
|
||
notReady error
|
||
fail error
|
||
sent []Message
|
||
}
|
||
|
||
func (f *fakeTelegram) Ready() error { return f.notReady }
|
||
func (f *fakeTelegram) Send(m Message) (string, error) {
|
||
if f.fail != nil {
|
||
return "", f.fail
|
||
}
|
||
f.sent = append(f.sent, m)
|
||
return "1", nil
|
||
}
|
||
|
||
type clock struct{ t time.Time }
|
||
|
||
func (c *clock) now() time.Time { return c.t }
|
||
func (c *clock) pass(d time.Duration) { c.t = c.t.Add(d) }
|
||
|
||
func watcher(t *testing.T) (*Watcher, *fakeTelegram, *clock) {
|
||
c := &clock{t: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)}
|
||
tg := &fakeTelegram{}
|
||
return NewWatcher(tg, c.now, t.Logf, "ace"), tg, c
|
||
}
|
||
|
||
// beatEvery has the controller's doctor run, and the bus answer, every five minutes up to d.
|
||
func beatEvery(w *Watcher, c *clock, d time.Duration) {
|
||
for end := c.t.Add(d); c.t.Before(end); {
|
||
c.pass(time.Minute)
|
||
if c.t.Minute()%5 == 0 {
|
||
w.Heartbeat(Beat{Run: "r", At: c.t, Interval: 5 * time.Minute})
|
||
}
|
||
w.BusAnswered(nil)
|
||
w.Tick()
|
||
}
|
||
}
|
||
|
||
func TestHeardSignalsSayNothing(t *testing.T) {
|
||
w, tg, c := watcher(t)
|
||
beatEvery(w, c, 3*time.Hour)
|
||
if len(tg.sent) != 0 {
|
||
t.Fatalf("sent %v", tg.sent)
|
||
}
|
||
if st := w.Status("listening"); st.Verdict != "ok: watching" {
|
||
t.Fatalf("%+v", st)
|
||
}
|
||
}
|
||
|
||
func TestTheSelfCheckSilentPastTwiceItsIntervalIsSentOnceThenOnceMoreThenItsReturn(t *testing.T) {
|
||
w, tg, c := watcher(t)
|
||
beatEvery(w, c, 30*time.Minute)
|
||
last := c.t
|
||
// The controller stops; the bus still answers.
|
||
for i := 0; i < 9; i++ {
|
||
c.pass(time.Minute)
|
||
w.BusAnswered(nil)
|
||
w.Tick()
|
||
}
|
||
if len(tg.sent) != 0 {
|
||
t.Fatalf("said silent within its bound (9 min of 10): %v", tg.sent)
|
||
}
|
||
for i := 0; i < 2; i++ {
|
||
c.pass(time.Minute)
|
||
w.BusAnswered(nil)
|
||
w.Tick()
|
||
}
|
||
if len(tg.sent) != 1 || !strings.Contains(tg.sent[0].Title, "self-check-silent") {
|
||
t.Fatalf("not said silent past twice the interval: %v", tg.sent)
|
||
}
|
||
if !strings.Contains(tg.sent[0].Body, "directly over HTTPS") || !strings.Contains(tg.sent[0].Body, last.Format("15:04")) {
|
||
t.Fatalf("body: %q", tg.sent[0].Body)
|
||
}
|
||
if st := w.Status("listening"); !strings.HasPrefix(st.Verdict, "ALARM") {
|
||
t.Fatalf("%+v", st)
|
||
}
|
||
for i := 0; i < 70; i++ {
|
||
c.pass(time.Minute)
|
||
w.BusAnswered(nil)
|
||
w.Tick()
|
||
}
|
||
if len(tg.sent) != 2 || !strings.HasPrefix(tg.sent[1].Title, "STILL SILENT") {
|
||
t.Fatalf("not said once more after an hour: %v", tg.sent)
|
||
}
|
||
beatEvery(w, c, 10*time.Minute)
|
||
if len(tg.sent) != 3 || !strings.HasPrefix(tg.sent[2].Title, "CLEARED") {
|
||
t.Fatalf("its return not said: %v", tg.sent)
|
||
}
|
||
beatEvery(w, c, time.Hour)
|
||
if len(tg.sent) != 3 {
|
||
t.Fatalf("said again after it returned: %v", tg.sent)
|
||
}
|
||
}
|
||
|
||
func TestAHeartbeatNeverHeardIsSilenceFromTheStart(t *testing.T) {
|
||
w, tg, c := watcher(t)
|
||
for i := 0; i < 11; i++ {
|
||
c.pass(time.Minute)
|
||
w.BusAnswered(nil)
|
||
w.Tick()
|
||
}
|
||
if len(tg.sent) != 1 || !strings.Contains(tg.sent[0].Body, "not heard once since the watcher started") {
|
||
t.Fatalf("%v", tg.sent)
|
||
}
|
||
}
|
||
|
||
func TestAReplayedOldHeartbeatIsNotASignOfLife(t *testing.T) {
|
||
w, tg, c := watcher(t)
|
||
beatEvery(w, c, 10*time.Minute)
|
||
old := c.t
|
||
for i := 0; i < 11; i++ {
|
||
c.pass(time.Minute)
|
||
w.BusAnswered(nil)
|
||
w.Tick()
|
||
}
|
||
// The watcher's consumer delivers a backlog: heartbeats made before the silence.
|
||
w.Heartbeat(Beat{Run: "old", At: old.Add(-time.Minute)})
|
||
w.Heartbeat(Beat{Run: "older", At: old.Add(-10 * time.Minute)})
|
||
c.pass(time.Minute)
|
||
w.Tick()
|
||
if len(tg.sent) != 1 {
|
||
t.Fatalf("a replay read as a return: %v", tg.sent)
|
||
}
|
||
w.Heartbeat(Beat{Run: "future", At: c.t.Add(time.Hour)})
|
||
c.pass(time.Minute)
|
||
w.Tick()
|
||
if len(tg.sent) != 1 || w.Status("listening").BadBeats != 1 {
|
||
t.Fatalf("a heartbeat from the future believed: %v", tg.sent)
|
||
}
|
||
}
|
||
|
||
func TestTheBoundFollowsTheIntervalTheHeartbeatSays(t *testing.T) {
|
||
w, tg, c := watcher(t)
|
||
w.Heartbeat(Beat{At: c.t, Interval: 15 * time.Minute})
|
||
for i := 0; i < 29; i++ {
|
||
c.pass(time.Minute)
|
||
w.BusAnswered(nil)
|
||
w.Tick()
|
||
}
|
||
if len(tg.sent) != 0 {
|
||
t.Fatalf("said silent inside 2 × 15 min: %v", tg.sent)
|
||
}
|
||
c.pass(2 * time.Minute)
|
||
w.BusAnswered(nil)
|
||
w.Tick()
|
||
if len(tg.sent) != 1 {
|
||
t.Fatalf("not said past 2 × 15 min")
|
||
}
|
||
}
|
||
|
||
func TestTheBusSilentPastThreeMinutesIsSent(t *testing.T) {
|
||
w, tg, c := watcher(t)
|
||
beatEvery(w, c, 10*time.Minute)
|
||
for i := 0; i < 4; i++ {
|
||
c.pass(time.Minute)
|
||
w.BusAnswered(errors.New("timeout"))
|
||
w.Tick()
|
||
}
|
||
if len(tg.sent) != 1 || !strings.Contains(tg.sent[0].Title, "bus-silent") {
|
||
t.Fatalf("%v", tg.sent)
|
||
}
|
||
if st := w.Status("listening"); st.Signals[1].LastFailure != "timeout" {
|
||
t.Fatalf("%+v", st.Signals[1])
|
||
}
|
||
}
|
||
|
||
func TestATelegramThatCannotSendIsSaidAndTheMessageIsOwed(t *testing.T) {
|
||
w, tg, c := watcher(t)
|
||
tg.fail = errors.New("telegram sendMessage: cannot connect (dial)")
|
||
for i := 0; i < 11; i++ {
|
||
c.pass(time.Minute)
|
||
w.BusAnswered(nil)
|
||
w.Tick()
|
||
}
|
||
st := w.Status("listening")
|
||
if !strings.HasPrefix(st.Verdict, "BLIND") || st.Signals[0].Owed == "" {
|
||
t.Fatalf("%+v", st)
|
||
}
|
||
tg.fail = nil
|
||
c.pass(time.Minute)
|
||
w.Tick()
|
||
if len(tg.sent) != 1 || w.Status("listening").Signals[0].Owed != "" {
|
||
t.Fatalf("the owed message was not sent: %v", tg.sent)
|
||
}
|
||
}
|
||
|
||
func TestNotGivenATokenIsBlind(t *testing.T) {
|
||
w, tg, _ := watcher(t)
|
||
tg.notReady = errors.New("no bot token")
|
||
if st := w.Status("listening"); st.Verdict != "BLIND: the watcher cannot tell the operator anything — CANNOT SEND: no bot token" {
|
||
t.Fatalf("%q", st.Verdict)
|
||
}
|
||
}
|
||
|
||
func TestOnTheControlNodeItSaysItIsMisplaced(t *testing.T) {
|
||
w, _, _ := watcher(t)
|
||
w.Placed([]string{"novox"})
|
||
if st := w.Status("listening"); st.Verdict != "ok: watching" || st.Control != "novox" {
|
||
t.Fatalf("%+v", st)
|
||
}
|
||
w.Placed([]string{"ace"})
|
||
if st := w.Status("listening"); !strings.HasPrefix(st.Verdict, "MISPLACED") {
|
||
t.Fatalf("%+v", st)
|
||
}
|
||
}
|
||
|
||
func TestTheSeatsAnswerIsReadInEitherShape(t *testing.T) {
|
||
bare := `{"seats":[{"seat":"mesh-controller","holders":[{"node":"n1","module":"mesh-controller"}]},
|
||
{"seat":"mesh-broker","holders":[{"node":"n1"},{"node":"n2"}]},{"seat":"git","holders":[{"node":"n3"}]}]}`
|
||
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]string{{"type": "text", "text": bare}}})
|
||
output, _ := json.Marshal(map[string]any{"ok": true, "output": bare})
|
||
for _, raw := range []string{bare, string(wrapped), string(output)} {
|
||
got, err := holdersOf(json.RawMessage(raw), "mesh-controller", "mesh-broker")
|
||
if err != nil || strings.Join(got, ",") != "n1,n2" {
|
||
t.Errorf("%s: %v %v", raw, got, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestTheHeartbeatAsTheDesignSaysIt(t *testing.T) {
|
||
b, err := DecodeBeat([]byte(`{"run":"doctor-41","at":"2026-10-06T12:05:00Z","interval-seconds":300,
|
||
"counts":{"pass":10,"fail":1,"failed-to-run":0}}`))
|
||
if err != nil || b.Run != "doctor-41" || b.Interval != 5*time.Minute || b.Counts["fail"] != 1 ||
|
||
!b.At.Equal(time.Date(2026, 10, 6, 12, 5, 0, 0, time.UTC)) {
|
||
t.Fatalf("%+v %v", b, err)
|
||
}
|
||
b, err = DecodeBeat([]byte(`{"run":7,"finished":"2026-10-06T12:05:00Z","interval":"5m"}`))
|
||
if err != nil || b.Run != "7" || b.Interval != 5*time.Minute || b.At.IsZero() {
|
||
t.Fatalf("%+v %v", b, err)
|
||
}
|
||
if b, err := DecodeBeat([]byte(`{}`)); err != nil || !b.At.IsZero() {
|
||
t.Fatalf("an empty heartbeat: %+v %v", b, err)
|
||
}
|
||
for body, says := range map[string]string{
|
||
`[]`: "not a JSON object",
|
||
`{"at":"yesterday"}`: "not an RFC 3339 time",
|
||
`{"interval":"often"}`: "not a duration",
|
||
`{"interval-seconds":-5}`: "not one doctor runs at",
|
||
`{"counts":"many"}`: "counts",
|
||
} {
|
||
if _, err := DecodeBeat([]byte(body)); err == nil || !strings.Contains(err.Error(), says) {
|
||
t.Errorf("%s: %v", body, err)
|
||
}
|
||
}
|
||
if !isHeartbeat("mesh-controller.doctor-heartbeat") || isHeartbeat("mesh-controller.applied") {
|
||
t.Fatalf("the heartbeat's key")
|
||
}
|
||
}
|