mesh-catalog/modules/messenger/cmd/messenger/onemessage_test.go

package main

// Issue 373: the operator no longer trusted the mesh's notifications — one problem shown twice, a unit that
// failed again at each apply said again each time, a question still offered after it ended. One message per
// problem at any time, updated in place.

import (
	occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
	"strings"
	"testing"
	"time"
)

func saidOn(chs ...*fakeChannel) int {
	n := 0
	for _, ch := range chs {
		n += len(ch.sends) + len(ch.edits) + len(ch.replaced)
	}
	return n
}

// A unit that fails again seconds after each apply clears it: the message continues without a word — no
// "again, 22nd time", no "Resolved" — and once it stays cleared, its clearing is said once.
func TestAFlappingConditionIsSaidOnceAndItsClearingOnceWhenItStops(t *testing.T) {
	h, tg, dt, c, _ := newHolder(t)
	k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
	k.Raised = c.now()
	h.Condition(EventRaised, k)
	settle(h, c)
	if len(dt.sends) != 1 || saidOn(tg) != 0 {
		t.Fatalf("raised: desktop %d, telegram %d", len(dt.sends), saidOn(tg))
	}
	for i := 0; i < 22; i++ {
		c.pass(3 * time.Minute)
		h.Tick()
		cl := Condition{Key: k.Key, At: c.now()}
		h.Condition(EventCleared, cl)
		c.pass(15 * time.Second)
		h.Flush()
		k.At = c.now()
		h.Condition(EventRaised, k)
		h.Flush()
	}
	runFor(h, c, 20*time.Minute)
	if got := saidOn(tg, dt); got != 1 {
		t.Fatalf("a flapping condition was said %d times; want once: desktop %+v %+v %+v", got, dt.sends, dt.edits, dt.replaced)
	}
	// Once it has come back, a longer gap — past ClearSettle, inside the reopen window — is still a blip.
	h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
	runFor(h, c, 5*time.Minute)
	k.At = c.now()
	h.Condition(EventRaised, k)
	runFor(h, c, time.Minute)
	if got := saidOn(tg, dt); got != 1 {
		t.Fatalf("a condition that flaps slower was said again: %d", got)
	}
	// Its comebacks were never said: the count the operator reads ("again, Nth time") does not grow with them.
	if open := h.Open(); len(open) != 1 || open[0].Count != 1 {
		t.Fatalf("open: %+v", open)
	}
	// A genuinely new problem meanwhile is said.
	n := cond("machine.ace.disk-full", Urgent, "the home server's disk is full")
	n.Raised = c.now()
	h.Condition(EventRaised, n)
	if len(tg.sends) != 1 || len(dt.sends) != 2 {
		t.Fatalf("a new problem was swallowed: telegram %d desktop %d", len(tg.sends), len(dt.sends))
	}
	// The unit stays mended: its clearing is said once, in place of its message.
	h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
	runFor(h, c, 30*time.Minute)
	if len(dt.edits) != 1 || !strings.HasPrefix(dt.edits["g14=1"].Title, "Resolved: ") || len(dt.sends) != 2 {
		t.Fatalf("its clearing: edits %+v, sends %d", dt.edits, len(dt.sends))
	}
}

// A clearing a few seconds long is never said, and the message continues; an urgent problem rang once.
func TestAnUrgentBlipRingsOnceAndItsBlipsSayNothing(t *testing.T) {
	h, tg, dt, c, _ := newHolder(t)
	k := cond("machine.ace.silent", Urgent, "the home server is silent")
	k.Raised = c.now()
	h.Condition(EventRaised, k)
	for i := 0; i < 5; i++ {
		c.pass(time.Minute)
		h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
		c.pass(20 * time.Second)
		h.Tick()
		k.At = c.now()
		h.Condition(EventRaised, k)
	}
	runFor(h, c, 10*time.Minute)
	if len(tg.sends) != 1 || len(dt.sends) != 1 || len(tg.edits)+len(dt.edits)+len(tg.replaced)+len(dt.replaced) != 0 {
		t.Fatalf("telegram %d %d %d, desktop %d %d %d", len(tg.sends), len(tg.edits), len(tg.replaced), len(dt.sends),
			len(dt.edits), len(dt.replaced))
	}
}

// Cleared before it was ever said, and back within the reopen window: a problem the operator never heard of is
// said, not swallowed as the same message.
func TestAConditionClearedBeforeItWasSaidAndBackIsSaid(t *testing.T) {
	h, _, dt, c, _ := newHolder(t)
	k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
	k.Raised = c.now()
	h.Condition(EventRaised, k)
	c.pass(10 * time.Second)
	h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
	c.pass(time.Minute)
	k.At = c.now()
	h.Condition(EventRaised, k)
	runFor(h, c, 2*time.Minute)
	if len(dt.sends) != 1 || !strings.Contains(dt.sends[0].Title, "machine shanks units") {
		t.Fatalf("desktop %+v", dt.sends)
	}
}

// Said first inside a digest, a problem's next message is one of its own, and the word after that replaces or
// edits that one: never a second message beside it.
func TestAProblemSaidInADigestGetsAMessageOfItsOwnThatLaterWordsReplace(t *testing.T) {
	h, tg, _, c, _ := newHolder(t)
	h.Desk = nil
	a := cond("seat.s1.silent", Warning, "s1 is silent")
	b := cond("seat.s2.silent", Warning, "s2 is silent")
	a.Raised, b.Raised = c.now(), c.now()
	h.Condition(EventRaised, a)
	h.Condition(EventRaised, b)
	settle(h, c)
	if len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "Mesh: 2 new problems") {
		t.Fatalf("sends %+v", tg.sends)
	}
	b.Severity = Urgent
	h.Condition(EventChanged, b)
	settle(h, c)
	if len(tg.sends) != 2 || !strings.HasPrefix(tg.sends[1].Title, "Now urgent: ") {
		t.Fatalf("escalation: %+v", tg.sends)
	}
	h.Condition(EventCleared, Condition{Key: b.Key, At: c.now()})
	c.pass(ClearSettle)
	h.Tick()
	settle(h, c)
	if len(tg.sends) != 2 || !strings.HasPrefix(tg.edits["2"].Title, "Resolved: seat s2") {
		t.Fatalf("its clearing was not an edit of its own message: sends %d, edits %+v", len(tg.sends), tg.edits)
	}
}

// An ask's expiry is said with its day when it is not today: an ask lasting a week read as ending tonight.
func TestAnAsksExpiryIsSaidWithItsDay(t *testing.T) {
	now := time.Date(2026, 10, 9, 21, 33, 0, 0, time.UTC)
	for at, want := range map[time.Time]string{
		now.Add(time.Hour):          "22:33",
		now.Add(24 * time.Hour):     "tomorrow 21:33",
		now.Add(7 * 24 * time.Hour): "Fri 16 Oct 21:33",
	} {
		if got := by(at, now, time.UTC); got != want {
			t.Errorf("%s: %q, want %q", at, got, want)
		}
	}
	h, _, _, c, _ := newHolder(t)
	h.Zone = func() *time.Location { return time.UTC }
	c.t = now
	rec := &AskRecord{Asker: ConditionOwner}
	rec.Ask.Expires = occ.FormatTime(now.Add(7 * 24 * time.Hour))
	rec.Ask.OnExpiry = "nothing is done"
	lines := strings.Join(h.askLines(rec, &Record{Raised: now}), "\n")
	if !strings.Contains(lines, "Unanswered by Fri 16 Oct 21:33: nothing is done.") {
		t.Fatalf("%s", lines)
	}
}

// A question on a problem's message, then its reminder: when the question ends, the one message the operator
// has loses its buttons and says so — none is left offering an answer nobody takes.
func TestAnEndedQuestionLeavesNoMessageOfferingItsAnswers(t *testing.T) {
	h, tg, dt, c, store, _ := newRouter(t)
	link(h, store, c, "42", c.now().Add(-2*time.Hour))
	k := cond("machine.novox.secret-given", Urgent, "a secret was given at the terminal")
	k.Raised = c.now()
	h.Condition(EventRaised, k)
	settle(h, c)
	a := ackAsk(c, "q-1")
	a.About = k.Key
	a.Expires = occ.FormatTime(c.now().Add(3 * time.Hour))
	submit(t, h, ConditionOwner, a)
	settle(h, c)
	for i := 0; i < 4*60; i++ {
		c.pass(time.Minute)
		stand(h, c)
		h.Tick()
		h.Flush()
	}
	if rec := store.asks[ConditionOwner+".q-1"]; rec.State == StateOpen {
		t.Fatalf("the ask did not expire: %+v", rec.State)
	}
	for _, ch := range []*fakeChannel{tg, dt} {
		if len(ch.sends) != 1 {
			t.Fatalf("%s: %d messages for one problem", ch.name, len(ch.sends))
		}
		last := ch.edits[map[string]string{"telegram": "1", "desktop": "g14=1"}[ch.name]]
		if len(ch.replaced) == 0 || len(last.Actions) > 1 || !strings.Contains(last.Body, "No longer answerable") {
			t.Fatalf("%s: replaced %d, the message now: %q %v", ch.name, len(ch.replaced), last.Text(), labelsOf(last))
		}
	}
}

// The review of PR 176: a comeback before the clearing was said must not swallow what changed meanwhile — a
// warning back as urgent rings, and a reminder waiting to go out when it cleared still goes out.
func TestABlipSwallowsNeitherAnEscalationNorAWaitingReminder(t *testing.T) {
	h, tg, dt, c, _ := newHolder(t)
	k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
	k.Raised = c.now()
	h.Condition(EventRaised, k)
	settle(h, c)
	if len(dt.sends) != 1 || len(tg.sends) != 0 {
		t.Fatalf("raised: desktop %d telegram %d", len(dt.sends), len(tg.sends))
	}
	h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
	c.pass(20 * time.Second)
	k.Severity, k.At = Urgent, c.now()
	h.Condition(EventRaised, k)
	settle(h, c)
	if len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "Now urgent: ") {
		t.Fatalf("back as urgent, and Telegram was not rung: %+v", tg.sends)
	}

	// A warning's reminder held by the desk's gentle rhythm when it cleared: said when it comes back.
	w := cond("plan.41.stalled", Warning, "plan 41 waits")
	w.Raised = c.now()
	h.Condition(EventRaised, w)
	runFor(h, c, DesktopWarningEvery+time.Minute)
	before := len(dt.sends) + len(dt.replaced)
	c.pass(RemindWarning)
	h.Tick() // the reminder is due, and the desk holds it: a warning went out there less than 15 min ago
	h.Condition(EventRaised, cond("seat.s9.silent", Warning, "s9 is silent"))
	h.Flush()
	h.Condition(EventCleared, Condition{Key: w.Key, At: c.now()})
	c.pass(20 * time.Second)
	w.At = c.now()
	h.Condition(EventRaised, w)
	runFor(h, c, DesktopWarningEvery+time.Minute)
	found := false
	for _, r := range dt.replaced {
		found = found || strings.HasPrefix(r.m.Title, "Still open: plan 41")
	}
	for _, m := range dt.sends {
		found = found || strings.Contains(m.Body, "plan 41 stalled") || strings.HasPrefix(m.Title, "Still open: plan 41")
	}
	if !found || len(dt.sends)+len(dt.replaced) == before {
		t.Fatalf("the reminder waiting when it blipped was lost: sends %+v replaced %+v", dt.sends, dt.replaced)
	}
}

// The review of PR 176: a reminder for one problem and a new problem in the same burst — the reminder goes
// alone in place of its message, never as a digest line beside it.
func TestAReminderInABurstReplacesItsMessageAndIsNoDigestLine(t *testing.T) {
	h, tg, _, c, _ := newHolder(t)
	h.Desk = nil
	a := cond("plan.41.stalled", Warning, "plan 41 waits")
	a.Raised = c.now()
	h.Condition(EventRaised, a)
	settle(h, c)
	c.pass(RemindWarning)
	h.Tick()
	b := cond("seat.s9.silent", Warning, "s9 is silent")
	b.Raised = c.now()
	h.Condition(EventRaised, b)
	settle(h, c)
	for _, m := range tg.sends {
		if strings.HasPrefix(m.Title, "Mesh: ") {
			t.Fatalf("a reminder went into a digest: %q", m.Text())
		}
	}
	if len(tg.replaced) != 1 || tg.replaced[0].id != "1" || len(tg.sends) != 2 {
		t.Fatalf("sends %+v, replaced %+v", tg.sends, tg.replaced)
	}
}

// The review of PR 176: a clearing the channel could not take is kept until it is said, past the reopen window.
func TestAClearingTheChannelCouldNotTakeIsSaidLater(t *testing.T) {
	h, tg, _, c, _ := newHolder(t)
	h.Desk = nil
	k := cond("machine.ace.silent", Urgent, "the home server is silent")
	k.Raised = c.now()
	h.Condition(EventRaised, k)
	tg.fail = errorf("telegram is away")
	h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
	runFor(h, c, ReopenWindow+2*time.Minute)
	tg.fail = nil
	runFor(h, c, 2*time.Minute)
	for _, m := range tg.sends {
		if strings.HasPrefix(m.Title, "Resolved: ") {
			return
		}
	}
	t.Fatalf("the clearing was lost: %+v", tg.sends)
}

// Issue 373, seen at 14:40: a condition silenced for a week with a reason left its critical popup standing on
// the desk as if still open, and the router logged nothing. A silence edits the shown message in place to say
// so — until when, and why — with no answers and no urgency, so the desk's popup times out; nothing more is
// said while it lasts; when the silence ends and the condition is still open, the message is open again.
func TestASilenceIsSaidOnTheShownMessageAndTheMessageIsOpenAgainWhenItEnds(t *testing.T) {
	h, tg, dt, c, _ := newHolder(t)
	h.Zone = func() *time.Location { return time.UTC }
	k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
	k.Raised = c.now()
	h.Condition(EventRaised, k)
	settle(h, c)
	if len(dt.sends) != 1 || len(dt.sends[0].Actions) == 0 {
		t.Fatalf("raised: %+v", dt.sends)
	}
	c.pass(time.Hour)
	k.SilencedTill, k.SilencedWhy = c.now().Add(7*24*time.Hour), "the mounts are gone for good"
	h.Condition(EventChanged, k)
	m, edited := dt.edits["g14=1"]
	if !edited || m.Title != "Silenced until Tue 13 Oct 13:00: machine shanks units" || !m.Quiet || m.Urgent ||
		len(m.Actions) != 0 || !strings.Contains(m.Body, "Because: the mounts are gone for good.") {
		t.Fatalf("silenced, and the desk's message says %q %q quiet=%v urgent=%v actions=%v", m.Title, m.Body, m.Quiet, m.Urgent, m.Actions)
	}
	if len(tg.sends)+len(tg.edits)+len(tg.replaced) != 0 {
		t.Fatalf("a silence reached a channel the message was not on")
	}
	// Nothing more while it lasts: no reminder, and the same silence said again edits nothing more.
	h.Condition(EventChanged, k)
	runFor(h, c, 13*time.Hour)
	if len(dt.sends) != 1 || len(dt.replaced) != 0 || len(dt.edits) != 1 {
		t.Fatalf("said again while silenced: sends %d replaced %d edits %d", len(dt.sends), len(dt.replaced), len(dt.edits))
	}
	// The silence runs out and the condition is still open: its message is open again, on the same notification.
	c.pass(7 * 24 * time.Hour)
	h.Tick()
	if m := dt.edits["g14=1"]; m.Title != "machine shanks units" || m.Quiet || len(m.Actions) == 0 {
		t.Fatalf("the silence ended, and the message says %q quiet=%v actions=%v", m.Title, m.Quiet, m.Actions)
	}
	if len(dt.sends) != 1 {
		t.Fatalf("the silence ending made a new message: %d", len(dt.sends))
	}
}

// Silenced on both channels and then resolved: Telegram's message, whose edits notify nobody, says so; the
// desk's popup, which timed out, is not brought back.
func TestAConditionResolvedWhileSilencedSaysSoOnlyWhereAnEditNotifiesNobody(t *testing.T) {
	h, tg, dt, c, _ := newHolder(t)
	k := cond("machine.ace.silent", Urgent, "the home server is silent")
	k.Raised = c.now()
	h.Condition(EventRaised, k)
	k.SilencedTill, k.SilencedWhy = c.now().Add(24*time.Hour), "ace is moving house"
	h.Condition(EventChanged, k)
	if !strings.HasPrefix(tg.edits["1"].Title, "Silenced until ") || !strings.HasPrefix(dt.edits["g14=1"].Title, "Silenced until ") {
		t.Fatalf("telegram %q desk %q", tg.edits["1"].Title, dt.edits["g14=1"].Title)
	}
	h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
	runFor(h, c, ClearSettle+time.Minute)
	if !strings.HasPrefix(tg.edits["1"].Title, "Resolved: ") || !strings.HasPrefix(dt.edits["g14=1"].Title, "Silenced until ") {
		t.Fatalf("resolved while silenced: telegram %q desk %q", tg.edits["1"].Title, dt.edits["g14=1"].Title)
	}
	if len(tg.sends)+len(dt.sends) != 2 || len(tg.replaced)+len(dt.replaced) != 0 {
		t.Fatalf("a new message was made: sends %d %d", len(tg.sends), len(dt.sends))
	}
}

// A silence given while this holder was not told (the 10-min reading of the open set carries it) is said the
// same way, and the reading that says it ended makes the message open again.
func TestASilenceReadFromTheOpenSetIsSaidOnTheShownMessage(t *testing.T) {
	h, _, dt, c, _ := newHolder(t)
	k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
	k.Raised = c.now()
	h.Condition(EventRaised, k)
	settle(h, c)
	c.pass(10 * time.Minute)
	s := k
	s.SilencedTill, s.SilencedWhy = c.now().Add(7*24*time.Hour), "the mounts are gone for good"
	h.Sync([]Condition{s}, c.now())
	if m := dt.edits["g14=1"]; !strings.HasPrefix(m.Title, "Silenced until ") || !m.Quiet || len(m.Actions) != 0 {
		t.Fatalf("read silenced, and the desk's message says %q quiet=%v actions=%v", m.Title, m.Quiet, m.Actions)
	}
	c.pass(10 * time.Minute)
	h.Sync([]Condition{k}, c.now()) // the silence was lifted: the reading carries none
	if m := dt.edits["g14=1"]; m.Title != "machine shanks units" || len(dt.sends) != 1 {
		t.Fatalf("read unsilenced, and the desk's message says %q (sends %d)", m.Title, len(dt.sends))
	}
}