package main // Issue 373: the operator no longer trusted the mesh's notifications — one problem shown twice, a unit that // failed again at each apply said again each time, a question still offered after it ended. One message per // problem at any time, updated in place. import ( "strings" "testing" "time" ) func saidOn(chs ...*fakeChannel) int { n := 0 for _, ch := range chs { n += len(ch.sends) + len(ch.edits) + len(ch.replaced) } return n } // A unit that fails again seconds after each apply clears it: the message continues without a word — no // "again, 22nd time", no "Resolved" — and once it stays cleared, its clearing is said once. func TestAFlappingConditionIsSaidOnceAndItsClearingOnceWhenItStops(t *testing.T) { h, tg, dt, c, _ := newHolder(t) k := cond("machine.shanks.units", Warning, "2 failed services on shanks") k.Raised = c.now() h.Condition(EventRaised, k) settle(h, c) if len(dt.sends) != 1 || saidOn(tg) != 0 { t.Fatalf("raised: desktop %d, telegram %d", len(dt.sends), saidOn(tg)) } for i := 0; i < 22; i++ { c.pass(3 * time.Minute) h.Tick() cl := Condition{Key: k.Key, At: c.now()} h.Condition(EventCleared, cl) c.pass(15 * time.Second) h.Flush() k.At = c.now() h.Condition(EventRaised, k) h.Flush() } runFor(h, c, 20*time.Minute) if got := saidOn(tg, dt); got != 1 { t.Fatalf("a flapping condition was said %d times; want once: desktop %+v %+v %+v", got, dt.sends, dt.edits, dt.replaced) } // Once it has come back, a longer gap — past ClearSettle, inside the reopen window — is still a blip. h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()}) runFor(h, c, 5*time.Minute) k.At = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) if got := saidOn(tg, dt); got != 1 { t.Fatalf("a condition that flaps slower was said again: %d", got) } // Its comebacks were never said: the count the operator reads ("again, Nth time") does not grow with them. if open := h.Open(); len(open) != 1 || open[0].Count != 1 { t.Fatalf("open: %+v", open) } // A genuinely new problem meanwhile is said. n := cond("machine.ace.disk-full", Urgent, "the home server's disk is full") n.Raised = c.now() h.Condition(EventRaised, n) if len(tg.sends) != 1 || len(dt.sends) != 2 { t.Fatalf("a new problem was swallowed: telegram %d desktop %d", len(tg.sends), len(dt.sends)) } // The unit stays mended: its clearing is said once, in place of its message. h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()}) runFor(h, c, 30*time.Minute) if len(dt.edits) != 1 || !strings.HasPrefix(dt.edits["g14=1"].Title, "Resolved: ") || len(dt.sends) != 2 { t.Fatalf("its clearing: edits %+v, sends %d", dt.edits, len(dt.sends)) } } // A clearing a few seconds long is never said, and the message continues; an urgent problem rang once. func TestAnUrgentBlipRingsOnceAndItsBlipsSayNothing(t *testing.T) { h, tg, dt, c, _ := newHolder(t) k := cond("machine.ace.silent", Urgent, "the home server is silent") k.Raised = c.now() h.Condition(EventRaised, k) for i := 0; i < 5; i++ { c.pass(time.Minute) h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()}) c.pass(20 * time.Second) h.Tick() k.At = c.now() h.Condition(EventRaised, k) } runFor(h, c, 10*time.Minute) if len(tg.sends) != 1 || len(dt.sends) != 1 || len(tg.edits)+len(dt.edits)+len(tg.replaced)+len(dt.replaced) != 0 { t.Fatalf("telegram %d %d %d, desktop %d %d %d", len(tg.sends), len(tg.edits), len(tg.replaced), len(dt.sends), len(dt.edits), len(dt.replaced)) } } // Cleared before it was ever said, and back within the reopen window: a problem the operator never heard of is // said, not swallowed as the same message. func TestAConditionClearedBeforeItWasSaidAndBackIsSaid(t *testing.T) { h, _, dt, c, _ := newHolder(t) k := cond("machine.shanks.units", Warning, "2 failed services on shanks") k.Raised = c.now() h.Condition(EventRaised, k) c.pass(10 * time.Second) h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()}) c.pass(time.Minute) k.At = c.now() h.Condition(EventRaised, k) runFor(h, c, 2*time.Minute) if len(dt.sends) != 1 || !strings.Contains(dt.sends[0].Title, "machine shanks units") { t.Fatalf("desktop %+v", dt.sends) } } // Said first inside a digest, a problem's next message is one of its own, and the word after that replaces or // edits that one: never a second message beside it. func TestAProblemSaidInADigestGetsAMessageOfItsOwnThatLaterWordsReplace(t *testing.T) { h, tg, _, c, _ := newHolder(t) h.Desk = nil a := cond("seat.s1.silent", Warning, "s1 is silent") b := cond("seat.s2.silent", Warning, "s2 is silent") a.Raised, b.Raised = c.now(), c.now() h.Condition(EventRaised, a) h.Condition(EventRaised, b) settle(h, c) if len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "Mesh: 2 new problems") { t.Fatalf("sends %+v", tg.sends) } b.Severity = Urgent h.Condition(EventChanged, b) settle(h, c) if len(tg.sends) != 2 || !strings.HasPrefix(tg.sends[1].Title, "Now urgent: ") { t.Fatalf("escalation: %+v", tg.sends) } h.Condition(EventCleared, Condition{Key: b.Key, At: c.now()}) c.pass(ClearSettle) h.Tick() settle(h, c) if len(tg.sends) != 2 || !strings.HasPrefix(tg.edits["2"].Title, "Resolved: seat s2") { t.Fatalf("its clearing was not an edit of its own message: sends %d, edits %+v", len(tg.sends), tg.edits) } } // An ask's expiry is said with its day when it is not today: an ask lasting a week read as ending tonight. func TestAnAsksExpiryIsSaidWithItsDay(t *testing.T) { now := time.Date(2026, 10, 9, 21, 33, 0, 0, time.UTC) for at, want := range map[time.Time]string{ now.Add(time.Hour): "22:33", now.Add(24 * time.Hour): "tomorrow 21:33", now.Add(7 * 24 * time.Hour): "Fri 16 Oct 21:33", } { if got := by(at, now, time.UTC); got != want { t.Errorf("%s: %q, want %q", at, got, want) } } h, _, _, c, _ := newHolder(t) h.Zone = func() *time.Location { return time.UTC } c.t = now rec := &AskRecord{Asker: ConditionOwner} rec.Ask.Expires = now.Add(7 * 24 * time.Hour) rec.Ask.OnExpiry = "nothing is done" lines := strings.Join(h.askLines(rec, &Record{Raised: now}), "\n") if !strings.Contains(lines, "Unanswered by Fri 16 Oct 21:33: nothing is done.") { t.Fatalf("%s", lines) } } // A question on a problem's message, then its reminder: when the question ends, the one message the operator // has loses its buttons and says so — none is left offering an answer nobody takes. func TestAnEndedQuestionLeavesNoMessageOfferingItsAnswers(t *testing.T) { h, tg, dt, c, store, _ := newRouter(t) link(h, store, c, "42", c.now().Add(-2*time.Hour)) k := cond("machine.novox.secret-given", Urgent, "a secret was given at the terminal") k.Raised = c.now() h.Condition(EventRaised, k) settle(h, c) a := ackAsk(c, "q-1") a.About = k.Key a.Expires = c.now().Add(3 * time.Hour) submit(t, h, ConditionOwner, a) settle(h, c) for i := 0; i < 4*60; i++ { c.pass(time.Minute) stand(h, c) h.Tick() h.Flush() } if rec := store.asks[ConditionOwner+".q-1"]; rec.State == StateOpen { t.Fatalf("the ask did not expire: %+v", rec.State) } for _, ch := range []*fakeChannel{tg, dt} { if len(ch.sends) != 1 { t.Fatalf("%s: %d messages for one problem", ch.name, len(ch.sends)) } last := ch.edits[map[string]string{"telegram": "1", "desktop": "g14=1"}[ch.name]] if len(ch.replaced) == 0 || len(last.Actions) > 1 || !strings.Contains(last.Body, "No longer answerable") { t.Fatalf("%s: replaced %d, the message now: %q %v", ch.name, len(ch.replaced), last.Text(), labelsOf(last)) } } } // The review of PR 176: a comeback before the clearing was said must not swallow what changed meanwhile — a // warning back as urgent rings, and a reminder waiting to go out when it cleared still goes out. func TestABlipSwallowsNeitherAnEscalationNorAWaitingReminder(t *testing.T) { h, tg, dt, c, _ := newHolder(t) k := cond("machine.shanks.units", Warning, "2 failed services on shanks") k.Raised = c.now() h.Condition(EventRaised, k) settle(h, c) if len(dt.sends) != 1 || len(tg.sends) != 0 { t.Fatalf("raised: desktop %d telegram %d", len(dt.sends), len(tg.sends)) } h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()}) c.pass(20 * time.Second) k.Severity, k.At = Urgent, c.now() h.Condition(EventRaised, k) settle(h, c) if len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "Now urgent: ") { t.Fatalf("back as urgent, and Telegram was not rung: %+v", tg.sends) } // A warning's reminder held by the desk's gentle rhythm when it cleared: said when it comes back. w := cond("plan.41.stalled", Warning, "plan 41 waits") w.Raised = c.now() h.Condition(EventRaised, w) runFor(h, c, DesktopWarningEvery+time.Minute) before := len(dt.sends) + len(dt.replaced) c.pass(RemindWarning) h.Tick() // the reminder is due, and the desk holds it: a warning went out there less than 15 min ago h.Condition(EventRaised, cond("seat.s9.silent", Warning, "s9 is silent")) h.Flush() h.Condition(EventCleared, Condition{Key: w.Key, At: c.now()}) c.pass(20 * time.Second) w.At = c.now() h.Condition(EventRaised, w) runFor(h, c, DesktopWarningEvery+time.Minute) found := false for _, r := range dt.replaced { found = found || strings.HasPrefix(r.m.Title, "Still open: plan 41") } for _, m := range dt.sends { found = found || strings.Contains(m.Body, "plan 41 stalled") || strings.HasPrefix(m.Title, "Still open: plan 41") } if !found || len(dt.sends)+len(dt.replaced) == before { t.Fatalf("the reminder waiting when it blipped was lost: sends %+v replaced %+v", dt.sends, dt.replaced) } } // The review of PR 176: a reminder for one problem and a new problem in the same burst — the reminder goes // alone in place of its message, never as a digest line beside it. func TestAReminderInABurstReplacesItsMessageAndIsNoDigestLine(t *testing.T) { h, tg, _, c, _ := newHolder(t) h.Desk = nil a := cond("plan.41.stalled", Warning, "plan 41 waits") a.Raised = c.now() h.Condition(EventRaised, a) settle(h, c) c.pass(RemindWarning) h.Tick() b := cond("seat.s9.silent", Warning, "s9 is silent") b.Raised = c.now() h.Condition(EventRaised, b) settle(h, c) for _, m := range tg.sends { if strings.HasPrefix(m.Title, "Mesh: ") { t.Fatalf("a reminder went into a digest: %q", m.Text()) } } if len(tg.replaced) != 1 || tg.replaced[0].id != "1" || len(tg.sends) != 2 { t.Fatalf("sends %+v, replaced %+v", tg.sends, tg.replaced) } } // The review of PR 176: a clearing the channel could not take is kept until it is said, past the reopen window. func TestAClearingTheChannelCouldNotTakeIsSaidLater(t *testing.T) { h, tg, _, c, _ := newHolder(t) h.Desk = nil k := cond("machine.ace.silent", Urgent, "the home server is silent") k.Raised = c.now() h.Condition(EventRaised, k) tg.fail = errorf("telegram is away") h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()}) runFor(h, c, ReopenWindow+2*time.Minute) tg.fail = nil runFor(h, c, 2*time.Minute) for _, m := range tg.sends { if strings.HasPrefix(m.Title, "Resolved: ") { return } } t.Fatalf("the clearing was lost: %+v", tg.sends) } // Issue 373, seen at 14:40: a condition silenced for a week with a reason left its critical popup standing on // the desk as if still open, and the router logged nothing. A silence edits the shown message in place to say // so — until when, and why — with no answers and no urgency, so the desk's popup times out; nothing more is // said while it lasts; when the silence ends and the condition is still open, the message is open again. func TestASilenceIsSaidOnTheShownMessageAndTheMessageIsOpenAgainWhenItEnds(t *testing.T) { h, tg, dt, c, _ := newHolder(t) h.Zone = func() *time.Location { return time.UTC } k := cond("machine.shanks.units", Warning, "2 failed services on shanks") k.Raised = c.now() h.Condition(EventRaised, k) settle(h, c) if len(dt.sends) != 1 || len(dt.sends[0].Actions) == 0 { t.Fatalf("raised: %+v", dt.sends) } c.pass(time.Hour) k.SilencedTill, k.SilencedWhy = c.now().Add(7*24*time.Hour), "the mounts are gone for good" h.Condition(EventChanged, k) m, edited := dt.edits["g14=1"] if !edited || m.Title != "Silenced until Tue 13 Oct 13:00: machine shanks units" || !m.Quiet || m.Urgent || len(m.Actions) != 0 || !strings.Contains(m.Body, "Because: the mounts are gone for good.") { t.Fatalf("silenced, and the desk's message says %q %q quiet=%v urgent=%v actions=%v", m.Title, m.Body, m.Quiet, m.Urgent, m.Actions) } if len(tg.sends)+len(tg.edits)+len(tg.replaced) != 0 { t.Fatalf("a silence reached a channel the message was not on") } // Nothing more while it lasts: no reminder, and the same silence said again edits nothing more. h.Condition(EventChanged, k) runFor(h, c, 13*time.Hour) if len(dt.sends) != 1 || len(dt.replaced) != 0 || len(dt.edits) != 1 { t.Fatalf("said again while silenced: sends %d replaced %d edits %d", len(dt.sends), len(dt.replaced), len(dt.edits)) } // The silence runs out and the condition is still open: its message is open again, on the same notification. c.pass(7 * 24 * time.Hour) h.Tick() if m := dt.edits["g14=1"]; m.Title != "machine shanks units" || m.Quiet || len(m.Actions) == 0 { t.Fatalf("the silence ended, and the message says %q quiet=%v actions=%v", m.Title, m.Quiet, m.Actions) } if len(dt.sends) != 1 { t.Fatalf("the silence ending made a new message: %d", len(dt.sends)) } } // Silenced on both channels and then resolved: Telegram's message, whose edits notify nobody, says so; the // desk's popup, which timed out, is not brought back. func TestAConditionResolvedWhileSilencedSaysSoOnlyWhereAnEditNotifiesNobody(t *testing.T) { h, tg, dt, c, _ := newHolder(t) k := cond("machine.ace.silent", Urgent, "the home server is silent") k.Raised = c.now() h.Condition(EventRaised, k) k.SilencedTill, k.SilencedWhy = c.now().Add(24*time.Hour), "ace is moving house" h.Condition(EventChanged, k) if !strings.HasPrefix(tg.edits["1"].Title, "Silenced until ") || !strings.HasPrefix(dt.edits["g14=1"].Title, "Silenced until ") { t.Fatalf("telegram %q desk %q", tg.edits["1"].Title, dt.edits["g14=1"].Title) } h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()}) runFor(h, c, ClearSettle+time.Minute) if !strings.HasPrefix(tg.edits["1"].Title, "Resolved: ") || !strings.HasPrefix(dt.edits["g14=1"].Title, "Silenced until ") { t.Fatalf("resolved while silenced: telegram %q desk %q", tg.edits["1"].Title, dt.edits["g14=1"].Title) } if len(tg.sends)+len(dt.sends) != 2 || len(tg.replaced)+len(dt.replaced) != 0 { t.Fatalf("a new message was made: sends %d %d", len(tg.sends), len(dt.sends)) } } // A silence given while this holder was not told (the 10-min reading of the open set carries it) is said the // same way, and the reading that says it ended makes the message open again. func TestASilenceReadFromTheOpenSetIsSaidOnTheShownMessage(t *testing.T) { h, _, dt, c, _ := newHolder(t) k := cond("machine.shanks.units", Warning, "2 failed services on shanks") k.Raised = c.now() h.Condition(EventRaised, k) settle(h, c) c.pass(10 * time.Minute) s := k s.SilencedTill, s.SilencedWhy = c.now().Add(7*24*time.Hour), "the mounts are gone for good" h.Sync([]Condition{s}, c.now()) if m := dt.edits["g14=1"]; !strings.HasPrefix(m.Title, "Silenced until ") || !m.Quiet || len(m.Actions) != 0 { t.Fatalf("read silenced, and the desk's message says %q quiet=%v actions=%v", m.Title, m.Quiet, m.Actions) } c.pass(10 * time.Minute) h.Sync([]Condition{k}, c.now()) // the silence was lifted: the reading carries none if m := dt.edits["g14=1"]; m.Title != "machine shanks units" || len(dt.sends) != 1 { t.Fatalf("read unsilenced, and the desk's message says %q (sends %d)", m.Title, len(dt.sends)) } }