jochen
f00b5a19b1
mesh/merge-gate pass: builds dunst, messenger → g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group feat/plain-notifications delivered: every member is delivered
The operator could not tell from a notification whether to act, nor act without opening a session. Each message now opens with a verdict, a warning that needs nothing is not sent, and a notification offers answers the messenger performs through their seat verbs; dunst's send gains actions and says the chosen one (hq ADR 0253).
462 lines
15 KiB
Go
462 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type fakeChannel struct {
|
|
name string
|
|
notReady error
|
|
fail error
|
|
sends []Message
|
|
edits map[string]Message
|
|
n int
|
|
silent bool // edits notify nobody, as Telegram's
|
|
}
|
|
|
|
func (f *fakeChannel) SilentEdit() bool { return f.silent }
|
|
|
|
func (f *fakeChannel) Name() string { return f.name }
|
|
func (f *fakeChannel) Ready() error { return f.notReady }
|
|
func (f *fakeChannel) CanEdit() bool { return true }
|
|
func (f *fakeChannel) Send(m Message) (string, error) {
|
|
if f.fail != nil {
|
|
return "", f.fail
|
|
}
|
|
f.n++
|
|
f.sends = append(f.sends, m)
|
|
return fmt.Sprint(f.n), nil
|
|
}
|
|
func (f *fakeChannel) Edit(id string, m Message) error {
|
|
if f.fail != nil {
|
|
return f.fail
|
|
}
|
|
if f.edits == nil {
|
|
f.edits = map[string]Message{}
|
|
}
|
|
f.edits[id] = m
|
|
return nil
|
|
}
|
|
|
|
type memStore struct {
|
|
recs map[string]Record
|
|
recent []Sent
|
|
}
|
|
|
|
func (m *memStore) Put(r Record) error {
|
|
if m.recs == nil {
|
|
m.recs = map[string]Record{}
|
|
}
|
|
m.recs[r.Key] = r
|
|
return nil
|
|
}
|
|
func (m *memStore) Delete(k string) error { delete(m.recs, k); return nil }
|
|
func (m *memStore) All() ([]Record, error) {
|
|
var out []Record
|
|
for _, r := range m.recs {
|
|
out = append(out, r)
|
|
}
|
|
return out, nil
|
|
}
|
|
func (m *memStore) PutRecent(s []Sent) error { m.recent = s; return nil }
|
|
func (m *memStore) Recent() ([]Sent, error) { return m.recent, nil }
|
|
|
|
type clock struct{ t time.Time }
|
|
|
|
func (c *clock) now() time.Time { return c.t }
|
|
func (c *clock) pass(d time.Duration) { c.t = c.t.Add(d) }
|
|
func start() *clock { return &clock{t: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)} }
|
|
func cond(key, sev, summary string) Condition {
|
|
parts := strings.Split(key, ".")
|
|
return Condition{Key: key, Scope: parts[0], ID: parts[1], Kind: parts[len(parts)-1], Severity: sev, Summary: summary,
|
|
Headline: headlineOf(key), Explanation: "Needs you: look at it. What it means: " + summary + ".",
|
|
Needs: "look at it."}
|
|
}
|
|
|
|
// headlineOf is the headline a test condition carries: its key in words, so a message can be told by it
|
|
// and never carries the key itself (novox/hq ADR 0253).
|
|
func headlineOf(key string) string { return strings.ReplaceAll(key, ".", " ") }
|
|
|
|
// keyless fails a message that carries a condition's key: the operator reads words, never keys.
|
|
func keyless(t *testing.T, ms ...Message) {
|
|
t.Helper()
|
|
for _, m := range ms {
|
|
for _, w := range strings.Fields(m.Text()) {
|
|
if strings.Count(w, ".") >= 2 && !strings.HasSuffix(w, ".") {
|
|
t.Errorf("a message carries a key %q: %q", w, m.Text())
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// settle lets a burst window pass and sends what is due.
|
|
func settle(h *Holder, c *clock) {
|
|
c.pass(BurstWindow)
|
|
h.Flush()
|
|
}
|
|
|
|
func newHolder(t *testing.T) (*Holder, *fakeChannel, *fakeChannel, *clock, *memStore) {
|
|
t.Helper()
|
|
c := start()
|
|
tg, dt := &fakeChannel{name: "telegram", silent: true}, &fakeChannel{name: "desktop"}
|
|
st := &memStore{}
|
|
var emitted []string
|
|
h := &Holder{Telegram: tg, Desktop: dt, Store: st, Now: c.now, Logf: t.Logf,
|
|
Emit: func(e string, _ any) error { emitted = append(emitted, e); return nil }}
|
|
h.init()
|
|
return h, tg, dt, c, st
|
|
}
|
|
|
|
func TestARaisedConditionIsSentOnceByItsKey(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
k := cond("machine.ace.silent", Urgent, "the home server has not been heard for 15 min")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
h.Condition(EventRaised, k) // a redelivery
|
|
c.pass(time.Minute)
|
|
h.Condition(EventRaised, k) // said again by the controller
|
|
if len(tg.sends) != 1 || len(dt.sends) != 1 {
|
|
t.Fatalf("urgent: telegram %d, desktop %d; want one each", len(tg.sends), len(dt.sends))
|
|
}
|
|
keyless(t, tg.sends...)
|
|
if tg.sends[0].Title != "Urgent: machine ace silent" || !strings.HasPrefix(tg.sends[0].Body, "Needs you: look at it. What it means: ") ||
|
|
!strings.Contains(tg.sends[0].Body, "\nSince ") {
|
|
t.Fatalf("message: %q", tg.sends[0].Text())
|
|
}
|
|
if open := h.Open(); len(open) != 1 || open[0].Count != 1 {
|
|
t.Fatalf("open: %+v", open)
|
|
}
|
|
}
|
|
|
|
func TestAWarningGoesToTheDesktopWhenASessionAnswersElseTelegram(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
h.Condition(EventRaised, cond("plan.41.stalled", Warning, "plan 41 waits on a build"))
|
|
if len(dt.sends)+len(tg.sends) != 0 {
|
|
t.Fatalf("a warning went out before its burst window")
|
|
}
|
|
settle(h, c)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 0 {
|
|
t.Fatalf("desktop answered: desktop %d telegram %d", len(dt.sends), len(tg.sends))
|
|
}
|
|
dt.fail = errors.New("the account is not logged in")
|
|
c.pass(DesktopWarningEvery)
|
|
h.Condition(EventRaised, cond("plan.42.stalled", Warning, "plan 42 waits on a build"))
|
|
settle(h, c) // the desktop fails: handed to Telegram
|
|
h.Flush()
|
|
if len(tg.sends) != 1 {
|
|
t.Fatalf("no session: telegram %d", len(tg.sends))
|
|
}
|
|
dt.fail, dt.notReady = nil, errors.New("not configured")
|
|
h.Condition(EventRaised, cond("plan.43.stalled", Warning, "plan 43 waits on a build"))
|
|
settle(h, c)
|
|
if len(tg.sends) != 2 {
|
|
t.Fatalf("no desktop configured: telegram %d", len(tg.sends))
|
|
}
|
|
}
|
|
|
|
func TestTheCapIsALastLineAndIsSaidOnce(t *testing.T) {
|
|
h, tg, _, c, _ := newHolder(t)
|
|
h.Desktop = nil
|
|
// Urgent conditions, each alone and after a quiet half minute: each is its own message — until
|
|
// the cap, which is said once, in a message of its own, and holds the rest.
|
|
for i := 0; i < 25; i++ {
|
|
h.Condition(EventRaised, cond(fmt.Sprintf("machine.m%d.silent", i), Urgent, "a machine is silent"))
|
|
c.pass(BurstWindow + time.Second)
|
|
h.Flush()
|
|
}
|
|
if len(tg.sends) != CapPerHour+1 {
|
|
t.Fatalf("sent %d, want the cap of %d and one saying so", len(tg.sends), CapPerHour)
|
|
}
|
|
if last := tg.sends[CapPerHour]; !strings.HasPrefix(last.Title, "Mesh messages paused for this hour") {
|
|
t.Fatalf("the cap was not said: %q", last.Title)
|
|
}
|
|
st := h.Status("listening")
|
|
if st.Channels[0].Held != 5 || st.Channels[0].SentLastHour != CapPerHour {
|
|
t.Fatalf("status: %+v", st.Channels[0])
|
|
}
|
|
for i := 0; i < 10; i++ {
|
|
c.pass(time.Minute)
|
|
h.Tick()
|
|
}
|
|
if len(tg.sends) != CapPerHour+1 {
|
|
t.Fatalf("said the cap again or sent past it: %d", len(tg.sends))
|
|
}
|
|
// The hour frees the channel: what was held goes out as one digest.
|
|
c.pass(time.Hour)
|
|
h.Tick()
|
|
if len(tg.sends) != CapPerHour+2 {
|
|
t.Fatalf("sends %d", len(tg.sends))
|
|
}
|
|
d := tg.sends[len(tg.sends)-1]
|
|
if !strings.HasPrefix(d.Title, "Urgent: 5 urgent problems") {
|
|
t.Fatalf("digest: %q", d.Title)
|
|
}
|
|
for i := 20; i < 25; i++ {
|
|
if !strings.Contains(d.Body, fmt.Sprintf("machine m%d silent", i)) {
|
|
t.Fatalf("the digest does not name m%d: %q", i, d.Body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAMessageCarryingAnAddressIsRefusedAndSaidWithItsWordsWithheld(t *testing.T) {
|
|
h, tg, _, _, _ := newHolder(t)
|
|
var emitted []string
|
|
h.Emit = func(e string, _ any) error { emitted = append(emitted, e); return nil }
|
|
h.Condition(EventRaised, cond("provider.keycloak.ace.failing", Urgent, "cannot reach 192.168.1.20:8443 with token=abc"))
|
|
if len(tg.sends) != 1 {
|
|
t.Fatalf("sends: %d", len(tg.sends))
|
|
}
|
|
text := tg.sends[0].Text()
|
|
if strings.Contains(text, "192.168") || strings.Contains(text, "token=") {
|
|
t.Fatalf("the address left: %q", text)
|
|
}
|
|
if !strings.Contains(text, "withheld, because it carried an IPv4 address") || !strings.Contains(text, "provider keycloak ace failing") {
|
|
t.Fatalf("not said as refused, by key: %q", text)
|
|
}
|
|
hist := h.History(10)["refusals"].([]RefusalNote)
|
|
if len(hist) != 1 || hist[0].Class != "address" {
|
|
t.Fatalf("refusals: %+v", hist)
|
|
}
|
|
if len(emitted) != 1 || emitted[0] != "refused" {
|
|
t.Fatalf("emitted: %v", emitted)
|
|
}
|
|
// Its clearing keeps the words withheld.
|
|
h.Condition(EventCleared, Condition{Key: "provider.keycloak.ace.failing"})
|
|
for _, m := range tg.edits {
|
|
if strings.Contains(m.Text(), "192.168") {
|
|
t.Fatalf("the clearing carried it: %q", m.Text())
|
|
}
|
|
}
|
|
if len(tg.edits) != 1 {
|
|
t.Fatalf("edits: %d", len(tg.edits))
|
|
}
|
|
}
|
|
|
|
func TestAKeyThatCarriesAnAddressIsWithheldToo(t *testing.T) {
|
|
h, tg, _, _, _ := newHolder(t)
|
|
h.Desktop = nil
|
|
h.Condition(EventRaised, cond("machine.10.0.0.7.silent", Urgent, "a machine is silent"))
|
|
if len(tg.sends) != 1 || strings.Contains(tg.sends[0].Text(), "10.0.0.7") {
|
|
t.Fatalf("sends: %+v", tg.sends)
|
|
}
|
|
}
|
|
|
|
func TestStillOpenPastItsBoundItIsSaidOnceMore(t *testing.T) {
|
|
h, tg, _, c, _ := newHolder(t)
|
|
h.Desktop = nil
|
|
u := cond("bus.controller.slow-consumer", Urgent, "the controller's consumer is far behind")
|
|
u.Raised = c.now()
|
|
w := cond("plan.41.stalled", Warning, "plan 41 waits")
|
|
w.Raised = c.now()
|
|
h.Condition(EventRaised, u)
|
|
h.Condition(EventRaised, w)
|
|
settle(h, c)
|
|
c.pass(59 * time.Minute)
|
|
h.Tick()
|
|
if len(tg.sends) != 2 {
|
|
t.Fatalf("reminded before the hour: %d", len(tg.sends))
|
|
}
|
|
c.pass(2 * time.Minute)
|
|
h.Tick()
|
|
if len(tg.sends) != 3 || (tg.sends[2].Title != "Still open: bus controller slow-consumer" || !strings.Contains(tg.sends[2].Body, "for 61 min")) {
|
|
t.Fatalf("urgent reminder: %d %q", len(tg.sends), tg.sends[len(tg.sends)-1].Title)
|
|
}
|
|
c.pass(3 * time.Hour)
|
|
h.Tick()
|
|
if len(tg.sends) != 3 {
|
|
t.Fatalf("reminded twice")
|
|
}
|
|
c.pass(9 * time.Hour) // the warning is now 13 h old
|
|
h.Tick()
|
|
settle(h, c)
|
|
if len(tg.sends) != 4 || tg.sends[3].Title != "Still open: plan 41 stalled" {
|
|
t.Fatalf("warning reminder: %d", len(tg.sends))
|
|
}
|
|
}
|
|
|
|
func TestClearedEditsTheFirstMessageAndReopenedWithinTenMinutesIsNotNew(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
k := cond("machine.ace.silent", Urgent, "the home server is silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
c.pass(14 * time.Minute)
|
|
h.Condition(EventCleared, Condition{Key: k.Key})
|
|
if len(tg.sends) != 1 || len(tg.edits) != 1 || len(dt.edits) != 1 {
|
|
t.Fatalf("telegram sends %d edits %d, desktop edits %d", len(tg.sends), len(tg.edits), len(dt.edits))
|
|
}
|
|
if m := tg.edits["1"]; m.Title != "Resolved: machine ace silent, after 14 min" || m.Body != "" {
|
|
t.Fatalf("edit: %q", m.Title)
|
|
}
|
|
if len(h.Open()) != 0 {
|
|
t.Fatalf("still open")
|
|
}
|
|
c.pass(5 * time.Minute)
|
|
k.At = c.now() // a reopening keeps its first raising time; the event says when it happened
|
|
h.Condition(EventRaised, k)
|
|
if len(tg.sends) != 1 || !strings.Contains(tg.edits["1"].Title, "(again, 2nd time)") {
|
|
t.Fatalf("reopened as new: sends %d, edit %q", len(tg.sends), tg.edits["1"].Title)
|
|
}
|
|
h.Condition(EventCleared, Condition{Key: k.Key})
|
|
c.pass(11 * time.Minute)
|
|
h.Tick()
|
|
if _, kept := st.recs[k.Key]; kept {
|
|
t.Fatalf("a cleared message kept past the reopen window")
|
|
}
|
|
k.At = c.now()
|
|
h.Condition(EventRaised, k)
|
|
if len(tg.sends) != 2 {
|
|
t.Fatalf("a raise after the window is a new message: %d", len(tg.sends))
|
|
}
|
|
}
|
|
|
|
func TestASilencedConditionSendsNothing(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
k := cond("machine.ace.silent", Urgent, "silent")
|
|
k.SilencedTill = c.now().Add(2 * time.Hour)
|
|
h.Condition(EventRaised, k)
|
|
c.pass(90 * time.Minute)
|
|
h.Tick()
|
|
h.Condition(EventCleared, Condition{Key: k.Key})
|
|
if len(tg.sends)+len(dt.sends)+len(tg.edits)+len(dt.edits) != 0 {
|
|
t.Fatalf("a silenced condition said something")
|
|
}
|
|
// Silenced after it was sent: no reminder.
|
|
k2 := cond("machine.shanks.silent", Urgent, "silent")
|
|
k2.Raised = c.now()
|
|
h.Condition(EventRaised, k2)
|
|
k2.SilencedTill = c.now().Add(3 * time.Hour)
|
|
h.Condition(EventChanged, k2)
|
|
c.pass(2 * time.Hour)
|
|
h.Tick()
|
|
if len(tg.sends) != 1 {
|
|
t.Fatalf("reminded while silenced: %d", len(tg.sends))
|
|
}
|
|
}
|
|
|
|
func TestEscalationIsSaidOnce(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
k := cond("machine.novox.silent", Warning, "the anchor is silent")
|
|
h.Condition(EventRaised, k)
|
|
settle(h, c)
|
|
k.Severity = Urgent
|
|
h.Condition(EventChanged, k)
|
|
h.Condition(EventChanged, k)
|
|
settle(h, c)
|
|
if len(dt.sends) != 2 || len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "Now urgent: ") {
|
|
t.Fatalf("desktop %d telegram %d", len(dt.sends), len(tg.sends))
|
|
}
|
|
// Escalated while the warning still waited its window: one message, the urgent one.
|
|
k2 := cond("machine.ace.silent", Warning, "the home server is silent")
|
|
h.Condition(EventRaised, k2)
|
|
k2.Severity = Urgent
|
|
h.Condition(EventChanged, k2)
|
|
settle(h, c)
|
|
if len(dt.sends) != 3 || !strings.HasPrefix(dt.sends[2].Title, "Now urgent: ") {
|
|
t.Fatalf("desktop %d: %q", len(dt.sends), dt.sends[len(dt.sends)-1].Title)
|
|
}
|
|
}
|
|
|
|
func TestAChannelThatCannotSendSaysSoAndIsTriedAgain(t *testing.T) {
|
|
h, tg, _, c, _ := newHolder(t)
|
|
h.Desktop = nil
|
|
tg.fail = errors.New("telegram sendMessage: cannot connect (dial)")
|
|
h.Condition(EventRaised, cond("machine.ace.silent", Urgent, "silent"))
|
|
st := h.Status("listening")
|
|
if !strings.HasPrefix(st.Verdict, "CANNOT SEND") || len(st.Unsent) != 1 || st.Channels[0].LastError == "" {
|
|
t.Fatalf("status: %+v", st)
|
|
}
|
|
tg.fail = nil
|
|
h.Flush()
|
|
if len(tg.sends) != 0 {
|
|
t.Fatalf("tried again before a minute")
|
|
}
|
|
c.pass(RetryEvery)
|
|
h.Tick()
|
|
if len(tg.sends) != 1 || h.Status("listening").Verdict != "ok" {
|
|
t.Fatalf("not retried: %d, %s", len(tg.sends), h.Status("listening").Verdict)
|
|
}
|
|
}
|
|
|
|
func TestNothingConfiguredIsSaidInTheStatus(t *testing.T) {
|
|
h, tg, dt, _, _ := newHolder(t)
|
|
tg.notReady = errors.New("no bot token")
|
|
dt.notReady = errors.New("no machine")
|
|
st := h.Status("listening")
|
|
if !strings.HasPrefix(st.Verdict, "CANNOT SEND") || st.Channels[0].NotReady != "no bot token" {
|
|
t.Fatalf("status: %+v", st)
|
|
}
|
|
}
|
|
|
|
func TestARestartForgetsNothing(t *testing.T) {
|
|
h, tg, _, c, st := newHolder(t)
|
|
h.Desktop = nil
|
|
k := cond("machine.ace.silent", Urgent, "silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
again := &Holder{Telegram: tg, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Condition(EventRaised, k)
|
|
if len(tg.sends) != 1 {
|
|
t.Fatalf("a restart sent it again")
|
|
}
|
|
again.Condition(EventCleared, Condition{Key: k.Key})
|
|
if len(tg.edits) != 1 {
|
|
t.Fatalf("a restart lost the message to edit")
|
|
}
|
|
}
|
|
|
|
func TestAnUnreadableEventIsToldOnceAnHour(t *testing.T) {
|
|
h, tg, _, c, _ := newHolder(t)
|
|
h.Desktop = nil
|
|
h.Unreadable("mesh-controller.condition-raised", errors.New("no severity"))
|
|
h.Unreadable("mesh-controller.condition-raised", errors.New("no severity"))
|
|
settle(h, c)
|
|
if len(tg.sends) != 1 || h.Status("listening").Unreadable != 2 {
|
|
t.Fatalf("sends %d", len(tg.sends))
|
|
}
|
|
c.pass(61 * time.Minute)
|
|
h.Unreadable("mesh-controller.condition-raised", errors.New("no severity"))
|
|
settle(h, c)
|
|
if len(tg.sends) != 2 {
|
|
t.Fatalf("not said again after an hour")
|
|
}
|
|
}
|
|
|
|
func TestNotifyIsKeptApartFromConditions(t *testing.T) {
|
|
h, tg, _, c, _ := newHolder(t)
|
|
h.Desktop = nil
|
|
out, err := h.Notify("backup.ace.failed", Warning, "last night's backup of the home server failed", "backup", false)
|
|
if err != nil || out["held"] != true {
|
|
t.Fatalf("%v %v", out, err)
|
|
}
|
|
settle(h, c)
|
|
if err != nil || out["key"] != "notify.backup.ace.failed" || len(tg.sends) != 1 {
|
|
t.Fatalf("%v %v %d", out, err, len(tg.sends))
|
|
}
|
|
if _, err := h.Notify("x", "", "s", "", false); err == nil {
|
|
t.Fatalf("no severity was not refused")
|
|
}
|
|
if _, err := h.Notify("x", Urgent, "see /var/lib/x", "", false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(tg.sends[len(tg.sends)-1].Text(), "/var/lib") {
|
|
t.Fatalf("a path left")
|
|
}
|
|
}
|
|
|
|
func TestATestMessageRespectsTheRule(t *testing.T) {
|
|
h, tg, dt, _, _ := newHolder(t)
|
|
if out := h.Test("telegram", "see https://example.org"); out["refused"] == "" || len(tg.sends) != 0 {
|
|
t.Fatalf("%v", out)
|
|
}
|
|
if out := h.Test("", ""); out["telegram"] != "sent" || out["desktop"] != "sent" || len(dt.sends) != 1 {
|
|
t.Fatalf("%v", out)
|
|
}
|
|
}
|