One message per problem: a reminder replaces, a blip says nothing (issues 373 and 369)

The operator stopped believing the mesh's notifications: one problem stood
twice on the desk (a reminder beside the message it followed, an escalation
then a reminder 31 s later), a unit failing again at each apply was said
again 22 times, and a question's buttons outlived it on a reminder the
ask's end never reached.

- messenger: a reminder or escalation is said in place of the problem's
  message (Replace under its handle), alone, never as a digest line; a
  problem first said in a digest gets its own message next time. A
  reminder counts from when it was last said. A clearing is said once it
  stays cleared 2 min (10 min once it came back); back before that, the
  message continues without a word. Cleared before ever said and back:
  said. An ask's expiry is said with its day.
- desk-channel: a send or show under a handle already shown replaces its
  notification by the notifier's replace id.
- telegram: the same deletes the earlier message and sends the new one,
  ringing as its kind rings; one it cannot delete loses its buttons.
This commit is contained in:
jochen
2026-10-10 14:22:42 +02:00
parent 6ec02b6eb1
commit 3345ff9317
17 changed files with 569 additions and 55 deletions
+4
View File
@@ -11,6 +11,10 @@ each machine the operator sits at (the `node-notifier` seat), and says which but
was made. A message with buttons stays until it is dismissed.
- **A notifier that started again** (`dunst.started`) is shown again every open message with buttons it showed.
- A message no machine took is said to the router as failed; the router tries it again.
- **One notification per message** (issue 373): an `edit`, and a `send` or `show` under a handle already shown (a
reminder, an escalation), replace that message's notification on each machine by its id (the notifier's
`replace_id`), so a problem is one notification updated in place. An edit goes only where it was shown; a
message said again is shown anew on a machine it was not.
## What the operator gives
@@ -141,8 +141,13 @@ func (d *Desk) Take(subject string, body json.RawMessage) error {
machines = []string{on}
}
}
// **One notification per message** (issue 373): an edit, and a message said again under a handle already
// shown (a reminder, an escalation), replace the notification it had on each machine — dunst's replace id —
// so a problem is one notification updated in place, never one more beside it. An edit stays quiet where
// it was shown; a message said again is shown anew where it was not.
edit := parts[4] == asks.Edit
var before *Shown
if parts[4] == asks.Edit {
if edit || !m.Secret {
var err error
if before, err = d.Store.Get(m.Handle); err != nil {
return err
@@ -154,7 +159,7 @@ func (d *Desk) Take(subject string, body json.RawMessage) error {
replace := 0
if before != nil {
replace = before.On[machine]
if replace == 0 {
if edit && replace == 0 {
continue // an edit goes where the message was shown, and nowhere new
}
}
@@ -259,3 +259,42 @@ func TestItRunsAsItsOwnAccount(t *testing.T) {
t.Errorf("the account %v, its state %v, its settings %v", account, state, settings)
}
}
// Issue 373: a message said again under its handle (a reminder, an escalation) replaces its notification on
// each machine — the notifier is called with that notification's replace id — and is shown anew only where it
// was not shown; a message under a new handle replaces nothing.
func TestAMessageSaidAgainUnderItsHandleReplacesItsNotification(t *testing.T) {
d, shown, _, store := newDesk(t, "laptop", "desktop")
take(t, d, asks.Send, asks.Message{Handle: "m1", Title: "2 failed services on shanks"})
take(t, d, asks.Send, asks.Message{Handle: "m2", Title: "another problem"})
if len(*shown) != 4 {
t.Fatalf("shown %d", len(*shown))
}
for _, n := range *shown {
if _, has := n.args["replace_id"]; has {
t.Fatalf("a new message replaced a notification: %+v", n)
}
}
first := store["m1"].On
// The desk now names a third machine as well.
d.Machines = func() []string { return []string{"laptop", "desktop", "third"} }
take(t, d, asks.Show, asks.Message{Handle: "m1", Title: "Still open: 2 failed services on shanks", Urgent: true,
Buttons: []asks.Button{{Label: "Silence for a week", Ticket: "t1"}}})
again := (*shown)[4:]
if len(again) != 3 {
t.Fatalf("said again on %d machines", len(again))
}
for _, n := range again {
want, was := first[n.machine]
got, replaces := n.args["replace_id"]
switch {
case was && (!replaces || got != want):
t.Errorf("on %s the reminder did not replace notification %d: %v", n.machine, want, n.args)
case !was && replaces:
t.Errorf("on %s, where it was not shown, it replaced %v", n.machine, got)
}
}
if s := store["m1"]; len(s.On) != 3 || len(s.Message.Buttons) != 1 {
t.Fatalf("what is shown was not kept: %+v", s)
}
}
+13 -4
View File
@@ -114,12 +114,21 @@ warning kept quiet is also said once after a day.
| When | What |
|---|---|
| `condition-raised` | one message, deduplicated by the condition's key |
| still open after 1 h (urgent) or 12 h (warning) | once more — never sooner than that after it was first said |
| `condition-changed` from warning to urgent | once more, to both channels |
| `condition-cleared` | the first message edited to say so; said inside a digest, a line in the next one |
| cleared and raised again within 10 min | the same message, edited back to open — not a new one |
| still open after 1 h (urgent) or 12 h (warning) | once more, in place of its message — never sooner than that after it was last said |
| `condition-changed` from warning to urgent | once more, in place of its message, to both channels |
| `condition-cleared` | once it stays cleared 2 min (10 min once it came back): its message edited to say so; said inside a digest, a line in the next one |
| cleared and raised again before its clearing was said | nothing: to the operator it never ended — a unit failing again after each apply is one message, not one per apply |
| cleared, said so, and raised again within 10 min | the same message, edited back to open — not a new one |
| silenced | nothing, its clearing included |
**One message per problem** (issue 373). A record keeps the id of its message on each channel; a reminder or
an escalation is said **in place of it** — the channel takes the earlier one away and shows the new one as a
new message is shown, so it rings once and no second message stands beside the first (the desk replaces the
notification by its id; Telegram deletes the earlier message). A message said again goes alone, never as a
line of a digest beside the message it follows; one first said inside a digest gets a message of its own the
next time, and later words replace or edit that one. An ask's buttons live on that one message, so an ask that
ends takes them away everywhere; its expiry is said with its day when it is not today.
- **Routing:** urgent to every kind that can send; a warning to the desk where it can send, otherwise to
the channel that reaches the operator away; an ask to every kind that carries one of its answers.
- **Bursts are one message.** What is to be said on a channel is held **30 s** from the first and goes
@@ -11,6 +11,15 @@ import (
// open message", because each new message replaced the ticket of the one before it. Every message still
// shown keeps a Details of its own, used once, and Details never acts.
// shownOn is every message a channel was given: sent, or said again in place of an earlier one (issue 373).
func shownOn(ch *fakeChannel) []Message {
out := append([]Message(nil), ch.sends...)
for _, r := range ch.replaced {
out = append(out, r.m)
}
return out
}
func detailsOf(t *testing.T, m Message) string {
t.Helper()
tok := ticketFor(m, DetailsLabel)
@@ -47,10 +56,13 @@ func TestDetailsWorksOnEachOfThreeWarningsOnceEach(t *testing.T) {
settle(h, c)
}
}
if len(tg.sends) != 3 {
t.Fatalf("%d messages, want 3", len(tg.sends))
// Issue 373: one problem said again replaces its message rather than standing beside it; each of
// the three messages still has a Details of its own.
msgs := shownOn(tg)
if len(msgs) != 3 {
t.Fatalf("%d messages, want 3", len(msgs))
}
for _, m := range tg.sends {
for _, m := range msgs {
toks = append(toks, detailsOf(t, m))
}
if toks[0] == toks[1] || toks[1] == toks[2] || toks[0] == toks[2] {
@@ -188,10 +200,11 @@ func TestAnUnreadyChannelPushesOutNoShownDetails(t *testing.T) {
tg.notReady = nil
c.pass(2 * time.Minute)
h.Flush()
if len(tg.sends) != 2 {
t.Fatalf("%d messages, want 2", len(tg.sends))
msgs := shownOn(tg)
if len(msgs) != 2 {
t.Fatalf("%d messages, want 2", len(msgs))
}
for _, tk := range []string{tok, detailsOf(t, tg.sends[1])} {
for _, tk := range []string{tok, detailsOf(t, msgs[1])} {
if out, err := h.Chosen("telegram", choiceOf(tk, "42", c.now())); err != nil || out != "shown" {
t.Errorf("Details: %q %v", out, err)
}
@@ -292,7 +292,7 @@ func TestABurstIsOneMessage(t *testing.T) {
for i := 0; i < 3; i++ {
h.Condition(EventCleared, Condition{Key: fmt.Sprintf("seat.s%d.silent", i)})
}
runFor(h, c, time.Minute)
runFor(h, c, ClearSettle+time.Minute) // a clearing is said once it stays cleared (issue 373)
if len(tg.sends) != 2 || len(tg.edits) != 0 || !strings.HasPrefix(tg.sends[1].Title, "Mesh: 3 resolved") {
t.Fatalf("sends %d edits %d: %q", len(tg.sends), len(tg.edits), tg.sends[len(tg.sends)-1].Title)
}
+90 -19
View File
@@ -25,9 +25,11 @@ package main
//
// - On `condition-raised`: one message, deduplicated by the condition's key. Said again with the
// same key while open, it is the same message, not a second.
// - Once more if still open after 1 hour (urgent) or 12 hours (warning).
// - On `condition-cleared`: the first message is edited where the channel can (both can).
// Cleared and raised again within ten minutes, it is the same message, edited back to open.
// - Once more if still open after 1 hour (urgent) or 12 hours (warning), counted from when it was last
// said; a reminder or an escalation replaces its message, never stands beside it (issue 373).
// - On `condition-cleared`: once it stays cleared ClearSettle, its message is edited where the channel can
// (both can). Raised again before that, nothing is said: to the operator it never ended. Cleared, said
// so, and raised again within ten minutes, it is the same message, edited back to open.
// - A silenced condition sends nothing.
// - Urgent to both channels; warning to the desktop when the operator's session is there,
// otherwise to Telegram.
@@ -59,9 +61,13 @@ const (
// QuietRemindAfter: a warning kept quiet as needing nothing is said once after this.
QuietRemindAfter = 24 * time.Hour
ReopenWindow = 10 * time.Minute
CapPerHour = 20
KeptSends = 200
KeptRefusals = 50
// ClearSettle: a clearing is said once the condition stayed cleared this long. One raised again sooner
// (a unit failing again after each apply, issue 373) continues its message without a word, and its
// clearing is never said. Once it has come back, its next clearing waits the whole ReopenWindow.
ClearSettle = 2 * time.Minute
CapPerHour = 20
KeptSends = 200
KeptRefusals = 50
// FreshFor: an event older than this by its own time is history — state, never a message.
FreshFor = 10 * time.Minute
@@ -120,6 +126,18 @@ type Channel interface {
CanEdit() bool
}
// replacer is a channel that can say a message again in place of the one it showed under that id: the earlier
// one goes, the new one is shown as a new message is — it rings as its kind rings (issue 373). A reminder or
// an escalation is then one message per problem, never one more beside it.
type replacer interface {
Replace(id string, m Message) error
}
func replaces(ch Channel) bool {
_, ok := ch.(replacer)
return ok
}
// silentEditor is a channel whose edits notify nobody (Telegram's are): what is over is said there
// even when it ended while the holder was away, since saying it costs the operator nothing.
type silentEditor interface{ SilentEdit() bool }
@@ -160,14 +178,22 @@ type Record struct {
More string `json:"more"`
Raised time.Time `json:"raised"`
SilencedTill time.Time `json:"silenced_till,omitempty"`
Sent map[string]string `json:"sent,omitempty"` // channel -> the first message's id; "" when said in a digest
Sent map[string]string `json:"sent,omitempty"` // channel -> its message's id; "" when said in a digest
FirstSent time.Time `json:"first_sent,omitempty"`
Reminded bool `json:"reminded,omitempty"`
Pending string `json:"pending,omitempty"` // what could not be said yet: raised, reminder, …
Folded bool `json:"folded,omitempty"` // held by the cap
Cleared time.Time `json:"cleared,omitempty"`
Count int `json:"count"`
Refused string `json:"refused,omitempty"`
// LastSaid is when a message about it last went out: a reminder is due its bound after it, never sooner
// (an escalation followed by a reminder 31 s later, issue 373).
LastSaid time.Time `json:"last_said,omitempty"`
// ClearUnsaid: cleared, and the clearing not said yet — it is said once it stays cleared ClearSettle;
// raised again before, the message continues as if it never cleared. ClearSilent: said only where an edit
// notifies nobody (it ended while this holder was away).
ClearUnsaid bool `json:"clear_unsaid,omitempty"`
ClearSilent bool `json:"clear_silent,omitempty"`
Reminded bool `json:"reminded,omitempty"`
Pending string `json:"pending,omitempty"` // what could not be said yet: raised, reminder, …
Folded bool `json:"folded,omitempty"` // held by the cap
Cleared time.Time `json:"cleared,omitempty"`
Count int `json:"count"`
Refused string `json:"refused,omitempty"`
// History: known from the state or from an old event, and never said — not news.
History bool `json:"history,omitempty"`
// Seen is when this holder last took word of it; the open set read after it is authoritative.
@@ -454,14 +480,27 @@ func (h *Holder) raised(rec Record, quiet bool) {
if old != nil && now.Sub(old.Cleared) < ReopenWindow {
// Cleared and raised again within ten minutes: the same message, back to open (to-be 45 §2).
h.mu.Lock()
blip := old.ClearUnsaid
gone := now.Sub(old.Cleared)
old.Cleared = time.Time{}
old.ClearUnsaid, old.ClearSilent = false, false
old.Count++
old.Summary, old.Severity, old.SilencedTill = rec.Summary, rec.Severity, rec.SilencedTill
old.takeWords(rec)
old.Seen = now
h.mu.Unlock()
if !quiet && !old.silenced(now) && old.told() {
switch {
case quiet || old.silenced(now):
case blip:
// Its clearing was never said (issue 373): to the operator it never ended, so nothing is said now.
h.Logf("[messenger] %s back %s after it cleared, which was not said yet: the same message continues, nothing said",
old.Key, roughly(gone))
h.record(Sent{At: now, Channel: "-", Key: old.Key, What: "reopened", Outcome: "quiet: back before its clearing was said"})
case old.told():
h.edit(old, "reopened", false)
case !needsNothing(old) && !h.queued(old.Key):
// Cleared before it was ever said, and back: a problem the operator has not heard of is said.
h.deliver(old, "raised")
}
h.persist(old)
return
@@ -553,11 +592,32 @@ func (h *Holder) cleared(key string, quiet bool) {
h.Logf("[messenger] %s cleared before it was said: nothing to unsay", key)
}
default:
h.edit(old, "cleared", quiet)
// Said once it stays cleared (ClearSettle; a condition that came back, ReopenWindow): a blip of a few
// seconds is not said at all (issue 373).
h.mu.Lock()
old.ClearUnsaid, old.ClearSilent = true, quiet
h.mu.Unlock()
}
h.persist(old)
}
// clearSettle is how long a record stays cleared before its clearing is said: one that came back before
// waits the whole reopen window, so a condition that flaps is said cleared once, when it stopped flapping.
func clearSettle(r *Record) time.Duration {
if r.Count > 1 {
return ReopenWindow - time.Minute
}
return ClearSettle
}
// lastSaid is when a message about a record last went out.
func lastSaid(r *Record) time.Time {
if !r.LastSaid.IsZero() {
return r.LastSaid
}
return r.FirstSent
}
// Tick does what time asks: reminders, retries, what is held, and forgetting what cleared long
// enough ago that a new raise is a new message.
func (h *Holder) Tick() {
@@ -578,7 +638,17 @@ func (h *Holder) Tick() {
for _, r := range recs {
switch {
case !r.Cleared.IsZero():
if now.Sub(r.Cleared) >= ReopenWindow {
if r.ClearUnsaid && now.Sub(r.Cleared) >= clearSettle(r) {
h.mu.Lock()
r.ClearUnsaid = false
silent := r.ClearSilent
h.mu.Unlock()
if !r.silenced(now) {
h.edit(r, "cleared", silent)
}
h.persist(r)
}
if !r.ClearUnsaid && now.Sub(r.Cleared) >= ReopenWindow && !h.queued(r.Key) {
h.mu.Lock()
delete(h.open, r.Key)
h.mu.Unlock()
@@ -601,9 +671,10 @@ func (h *Holder) Tick() {
h.deliver(r, "quiet-reminder")
h.persist(r)
case !r.Reminded && !r.FirstSent.IsZero() && now.Sub(r.Raised) >= remindAfter(r.Severity) &&
now.Sub(r.FirstSent) >= remindAfter(r.Severity):
now.Sub(lastSaid(r)) >= remindAfter(r.Severity):
// Once more after its bound — counted from its raising, and never sooner after it was
// first said (an old condition said at a reading of the state is not reminded at once).
// last said (an old condition said at a reading of the state is not reminded at once, nor one
// just escalated: issue 373).
h.mu.Lock()
r.Reminded = true
h.mu.Unlock()
@@ -714,7 +785,7 @@ func (h *Holder) askLines(rec *AskRecord) []string {
}
}
if !rec.Ask.Expires.IsZero() && rec.Ask.OnExpiry != "" {
lines = append(lines, "If nobody answers by "+rec.Ask.Expires.In(h.zone()).Format("15:04")+": "+rec.Ask.OnExpiry+".")
lines = append(lines, "If nobody answers by "+by(rec.Ask.Expires, h.Now(), h.zone())+": "+rec.Ask.OnExpiry+".")
}
return lines
}
+56 -12
View File
@@ -17,6 +17,22 @@ type fakeChannel struct {
edits map[string]Message
n int
silent bool // edits notify nobody, as Telegram's
// replaced is each message said again in place of an earlier one, as the id it replaced and the message.
replaced []replacement
}
type replacement struct {
id string
m Message
}
// Replace says a message again in place of an earlier one: shown as a new message is, under the same id.
func (f *fakeChannel) Replace(id string, m Message) error {
if f.fail != nil {
return f.fail
}
f.replaced = append(f.replaced, replacement{id, m})
return nil
}
func (f *fakeChannel) SilentEdit() bool { return f.silent }
@@ -208,7 +224,7 @@ func TestTheCapIsALastLineAndIsSaidOnce(t *testing.T) {
}
func TestAMessageCarryingAnAddressIsRefusedAndSaidWithItsWordsWithheld(t *testing.T) {
h, tg, _, _, _ := newHolder(t)
h, tg, _, c, _ := newHolder(t)
var emitted []string
h.Emit = func(e string, _ any) error { emitted = append(emitted, e); return nil }
h.Condition(EventRaised, cond("provider.keycloak.ace.failing", Urgent, "cannot reach 192.168.1.20:8443 with token=abc"))
@@ -231,6 +247,8 @@ func TestAMessageCarryingAnAddressIsRefusedAndSaidWithItsWordsWithheld(t *testin
}
// Its clearing keeps the words withheld.
h.Condition(EventCleared, Condition{Key: "provider.keycloak.ace.failing"})
c.pass(ClearSettle)
h.Tick()
for _, m := range tg.edits {
if strings.Contains(m.Text(), "192.168") {
t.Fatalf("the clearing carried it: %q", m.Text())
@@ -262,24 +280,26 @@ func TestStillOpenPastItsBoundItIsSaidOnceMore(t *testing.T) {
settle(h, c)
c.pass(59 * time.Minute)
h.Tick()
if len(tg.sends) != 2 {
t.Fatalf("reminded before the hour: %d", len(tg.sends))
if len(tg.sends) != 2 || len(tg.replaced) != 0 {
t.Fatalf("reminded before the hour: %d %d", len(tg.sends), len(tg.replaced))
}
c.pass(2 * time.Minute)
h.Tick()
if len(tg.sends) != 3 || (tg.sends[2].Title != "Still open: bus controller slow-consumer" || !strings.Contains(tg.sends[2].Body, "for 61 min")) {
t.Fatalf("urgent reminder: %d %q", len(tg.sends), tg.sends[len(tg.sends)-1].Title)
// Said once more in place of its first message (issue 373): one message per problem.
if len(tg.sends) != 2 || len(tg.replaced) != 1 || tg.replaced[0].id != "1" ||
tg.replaced[0].m.Title != "Still open: bus controller slow-consumer" || !strings.Contains(tg.replaced[0].m.Body, "for 61 min") {
t.Fatalf("urgent reminder: %d sends, %+v", len(tg.sends), tg.replaced)
}
c.pass(3 * time.Hour)
h.Tick()
if len(tg.sends) != 3 {
if len(tg.replaced) != 1 {
t.Fatalf("reminded twice")
}
c.pass(9 * time.Hour) // the warning is now 13 h old
h.Tick()
settle(h, c)
if len(tg.sends) != 4 || tg.sends[3].Title != "Still open: plan 41 stalled" {
t.Fatalf("warning reminder: %d", len(tg.sends))
if len(tg.sends) != 2 || len(tg.replaced) != 2 || tg.replaced[1].id != "2" || tg.replaced[1].m.Title != "Still open: plan 41 stalled" {
t.Fatalf("warning reminder: %d sends, %+v", len(tg.sends), tg.replaced)
}
}
@@ -290,6 +310,8 @@ func TestClearedEditsTheFirstMessageAndReopenedWithinTenMinutesIsNotNew(t *testi
h.Condition(EventRaised, k)
c.pass(14 * time.Minute)
h.Condition(EventCleared, Condition{Key: k.Key})
c.pass(ClearSettle) // said once it stays cleared (issue 373)
h.Tick()
if len(tg.sends) != 1 || len(tg.edits) != 1 || len(dt.edits) != 1 {
t.Fatalf("telegram sends %d edits %d, desktop edits %d", len(tg.sends), len(tg.edits), len(dt.edits))
}
@@ -299,7 +321,7 @@ func TestClearedEditsTheFirstMessageAndReopenedWithinTenMinutesIsNotNew(t *testi
if len(h.Open()) != 0 {
t.Fatalf("still open")
}
c.pass(5 * time.Minute)
c.pass(3 * time.Minute)
k.At = c.now() // a reopening keeps its first raising time; the event says when it happened
h.Condition(EventRaised, k)
if len(tg.sends) != 1 || !strings.Contains(tg.edits["1"].Title, "(again, 2nd time)") {
@@ -351,8 +373,10 @@ func TestEscalationIsSaidOnce(t *testing.T) {
h.Condition(EventChanged, k)
h.Condition(EventChanged, k)
settle(h, c)
if len(dt.sends) != 2 || len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "Now urgent: ") {
t.Fatalf("desktop %d telegram %d", len(dt.sends), len(tg.sends))
// The desktop's warning is replaced by the urgent message (issue 373); Telegram, which had none, is sent one.
if len(dt.sends) != 1 || len(dt.replaced) != 1 || dt.replaced[0].id != "g14=1" ||
!strings.HasPrefix(dt.replaced[0].m.Title, "Now urgent: ") || len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "Now urgent: ") {
t.Fatalf("desktop %d sends %d replaced, telegram %d", len(dt.sends), len(dt.replaced), len(tg.sends))
}
// Escalated while the warning still waited its window: one message, the urgent one.
k2 := cond("machine.ace.silent", Warning, "the home server is silent")
@@ -360,9 +384,23 @@ func TestEscalationIsSaidOnce(t *testing.T) {
k2.Severity = Urgent
h.Condition(EventChanged, k2)
settle(h, c)
if len(dt.sends) != 3 || !strings.HasPrefix(dt.sends[2].Title, "Now urgent: ") {
if len(dt.sends) != 2 || !strings.HasPrefix(dt.sends[1].Title, "Now urgent: ") || len(dt.replaced) != 1 {
t.Fatalf("desktop %d: %q", len(dt.sends), dt.sends[len(dt.sends)-1].Title)
}
// Escalated, then the hour of an urgent one passes since it was raised: no reminder until an hour after the
// escalation was said (issue 373: "Now urgent" and "Still open" 31 s apart).
c.pass(59 * time.Minute)
h.Tick()
settle(h, c)
if len(dt.replaced) != 1 || len(tg.replaced) != 0 {
t.Fatalf("reminded within the hour of the escalation: %+v %+v", dt.replaced, tg.replaced)
}
c.pass(2 * time.Minute)
h.Tick()
settle(h, c)
if len(tg.replaced) != 2 || len(tg.sends) != 2 {
t.Fatalf("not reminded an hour after the escalation, in place: telegram %d sends %+v", len(tg.sends), tg.replaced)
}
}
func TestAChannelThatCannotSendSaysSoAndIsTriedAgain(t *testing.T) {
@@ -411,6 +449,12 @@ func TestARestartForgetsNothing(t *testing.T) {
t.Fatalf("a restart sent it again")
}
again.Condition(EventCleared, Condition{Key: k.Key})
third := &Holder{Away: tg, Store: st, Now: c.now, Logf: t.Logf}
if err := third.Load(); err != nil {
t.Fatal(err)
}
c.pass(ClearSettle)
third.Tick() // a clearing not said yet before a restart is said after it
if len(tg.edits) != 1 {
t.Fatalf("a restart lost the message to edit")
}
@@ -0,0 +1,198 @@
package main
// Issue 373: the operator no longer trusted the mesh's notifications — one problem shown twice, a unit that
// failed again at each apply said again each time, a question still offered after it ended. One message per
// problem at any time, updated in place.
import (
"strings"
"testing"
"time"
)
func saidOn(chs ...*fakeChannel) int {
n := 0
for _, ch := range chs {
n += len(ch.sends) + len(ch.edits) + len(ch.replaced)
}
return n
}
// A unit that fails again seconds after each apply clears it: the message continues without a word — no
// "again, 22nd time", no "Resolved" — and once it stays cleared, its clearing is said once.
func TestAFlappingConditionIsSaidOnceAndItsClearingOnceWhenItStops(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
if len(dt.sends) != 1 || saidOn(tg) != 0 {
t.Fatalf("raised: desktop %d, telegram %d", len(dt.sends), saidOn(tg))
}
for i := 0; i < 22; i++ {
c.pass(3 * time.Minute)
h.Tick()
cl := Condition{Key: k.Key, At: c.now()}
h.Condition(EventCleared, cl)
c.pass(15 * time.Second)
h.Flush()
k.At = c.now()
h.Condition(EventRaised, k)
h.Flush()
}
runFor(h, c, 20*time.Minute)
if got := saidOn(tg, dt); got != 1 {
t.Fatalf("a flapping condition was said %d times; want once: desktop %+v %+v %+v", got, dt.sends, dt.edits, dt.replaced)
}
// Once it has come back, a longer gap — past ClearSettle, inside the reopen window — is still a blip.
h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
runFor(h, c, 5*time.Minute)
k.At = c.now()
h.Condition(EventRaised, k)
runFor(h, c, time.Minute)
if got := saidOn(tg, dt); got != 1 {
t.Fatalf("a condition that flaps slower was said again: %d", got)
}
if open := h.Open(); len(open) != 1 || open[0].Count != 24 {
t.Fatalf("open: %+v", open)
}
// A genuinely new problem meanwhile is said.
n := cond("machine.ace.disk-full", Urgent, "the home server's disk is full")
n.Raised = c.now()
h.Condition(EventRaised, n)
if len(tg.sends) != 1 || len(dt.sends) != 2 {
t.Fatalf("a new problem was swallowed: telegram %d desktop %d", len(tg.sends), len(dt.sends))
}
// The unit stays mended: its clearing is said once, in place of its message.
h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
runFor(h, c, 30*time.Minute)
if len(dt.edits) != 1 || !strings.HasPrefix(dt.edits["g14=1"].Title, "Resolved: ") || len(dt.sends) != 2 {
t.Fatalf("its clearing: edits %+v, sends %d", dt.edits, len(dt.sends))
}
}
// A clearing a few seconds long is never said, and the message continues; an urgent problem rang once.
func TestAnUrgentBlipRingsOnceAndItsBlipsSayNothing(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
k := cond("machine.ace.silent", Urgent, "the home server is silent")
k.Raised = c.now()
h.Condition(EventRaised, k)
for i := 0; i < 5; i++ {
c.pass(time.Minute)
h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
c.pass(20 * time.Second)
h.Tick()
k.At = c.now()
h.Condition(EventRaised, k)
}
runFor(h, c, 10*time.Minute)
if len(tg.sends) != 1 || len(dt.sends) != 1 || len(tg.edits)+len(dt.edits)+len(tg.replaced)+len(dt.replaced) != 0 {
t.Fatalf("telegram %d %d %d, desktop %d %d %d", len(tg.sends), len(tg.edits), len(tg.replaced), len(dt.sends),
len(dt.edits), len(dt.replaced))
}
}
// Cleared before it was ever said, and back within the reopen window: a problem the operator never heard of is
// said, not swallowed as the same message.
func TestAConditionClearedBeforeItWasSaidAndBackIsSaid(t *testing.T) {
h, _, dt, c, _ := newHolder(t)
k := cond("machine.shanks.units", Warning, "2 failed services on shanks")
k.Raised = c.now()
h.Condition(EventRaised, k)
c.pass(10 * time.Second)
h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()})
c.pass(time.Minute)
k.At = c.now()
h.Condition(EventRaised, k)
runFor(h, c, 2*time.Minute)
if len(dt.sends) != 1 || !strings.Contains(dt.sends[0].Title, "machine shanks units") {
t.Fatalf("desktop %+v", dt.sends)
}
}
// Said first inside a digest, a problem's next message is one of its own, and the word after that replaces or
// edits that one: never a second message beside it.
func TestAProblemSaidInADigestGetsAMessageOfItsOwnThatLaterWordsReplace(t *testing.T) {
h, tg, _, c, _ := newHolder(t)
h.Desk = nil
a := cond("seat.s1.silent", Warning, "s1 is silent")
b := cond("seat.s2.silent", Warning, "s2 is silent")
a.Raised, b.Raised = c.now(), c.now()
h.Condition(EventRaised, a)
h.Condition(EventRaised, b)
settle(h, c)
if len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "Mesh: 2 new problems") {
t.Fatalf("sends %+v", tg.sends)
}
b.Severity = Urgent
h.Condition(EventChanged, b)
settle(h, c)
if len(tg.sends) != 2 || !strings.HasPrefix(tg.sends[1].Title, "Now urgent: ") {
t.Fatalf("escalation: %+v", tg.sends)
}
h.Condition(EventCleared, Condition{Key: b.Key, At: c.now()})
c.pass(ClearSettle)
h.Tick()
settle(h, c)
if len(tg.sends) != 2 || !strings.HasPrefix(tg.edits["2"].Title, "Resolved: seat s2") {
t.Fatalf("its clearing was not an edit of its own message: sends %d, edits %+v", len(tg.sends), tg.edits)
}
}
// An ask's expiry is said with its day when it is not today: an ask lasting a week read as ending tonight.
func TestAnAsksExpiryIsSaidWithItsDay(t *testing.T) {
now := time.Date(2026, 10, 9, 21, 33, 0, 0, time.UTC)
for at, want := range map[time.Time]string{
now.Add(time.Hour): "22:33",
now.Add(24 * time.Hour): "tomorrow 21:33",
now.Add(7 * 24 * time.Hour): "Fri 16 Oct 21:33",
} {
if got := by(at, now, time.UTC); got != want {
t.Errorf("%s: %q, want %q", at, got, want)
}
}
h, _, _, c, _ := newHolder(t)
h.Zone = func() *time.Location { return time.UTC }
c.t = now
rec := &AskRecord{Asker: ConditionOwner}
rec.Ask.Expires = now.Add(7 * 24 * time.Hour)
rec.Ask.OnExpiry = "nothing is done"
lines := strings.Join(h.askLines(rec), "\n")
if !strings.Contains(lines, "If nobody answers by Fri 16 Oct 21:33: nothing is done.") {
t.Fatalf("%s", lines)
}
}
// A question on a problem's message, then its reminder: when the question ends, the one message the operator
// has loses its buttons and says so — none is left offering an answer nobody takes.
func TestAnEndedQuestionLeavesNoMessageOfferingItsAnswers(t *testing.T) {
h, tg, dt, c, store, _ := newRouter(t)
link(h, store, c, "42", c.now().Add(-2*time.Hour))
k := cond("machine.novox.secret-given", Urgent, "a secret was given at the terminal")
k.Raised = c.now()
h.Condition(EventRaised, k)
settle(h, c)
a := ackAsk(c, "q-1")
a.About = k.Key
a.Expires = c.now().Add(3 * time.Hour)
submit(t, h, ConditionOwner, a)
settle(h, c)
for i := 0; i < 4*60; i++ {
c.pass(time.Minute)
stand(h, c)
h.Tick()
h.Flush()
}
if rec := store.asks[ConditionOwner+".q-1"]; rec.State == StateOpen {
t.Fatalf("the ask did not expire: %+v", rec.State)
}
for _, ch := range []*fakeChannel{tg, dt} {
if len(ch.sends) != 1 {
t.Fatalf("%s: %d messages for one problem", ch.name, len(ch.sends))
}
last := ch.edits[map[string]string{"telegram": "1", "desktop": "g14=1"}[ch.name]]
if len(ch.replaced) == 0 || len(last.Actions) > 1 || !strings.Contains(last.Body, "No longer answerable") {
t.Fatalf("%s: replaced %d, the message now: %q %v", ch.name, len(ch.replaced), last.Text(), labelsOf(last))
}
}
}
+34 -9
View File
@@ -294,7 +294,9 @@ func (h *Holder) flushChannel(ch Channel, now time.Time, force bool) {
// an ask whose approval was not offered folded into a digest, and its one answer lost with it).
var alone, rest []entry
for _, e := range entries {
if len(e.msg.Actions) > 1 || (len(e.msg.Actions) > 0 && e.r != nil && e.r.Ask != "") {
// A message said again about a record shown here goes alone too: it replaces that record's message,
// so a reminder is never a digest line beside the message it follows (issue 373).
if len(e.msg.Actions) > 1 || (len(e.msg.Actions) > 0 && e.r != nil && e.r.Ask != "") || h.replacing(ch, e) != "" {
alone = append(alone, e)
} else {
rest = append(rest, e)
@@ -329,12 +331,26 @@ type entry struct {
msg Message
}
// sendEntries sends entries as one message — itself when alone, a digest when several — and records it.
// replacing is the id of the message an entry replaces on a channel: the record's own message there, when the
// entry says it again (not its clearing) and the channel can replace; "" for a new message.
func (h *Holder) replacing(ch Channel, e entry) string {
if e.r == nil || e.it.What == "cleared" || !replaces(ch) {
return ""
}
h.mu.Lock()
defer h.mu.Unlock()
return e.r.Sent[ch.Name()]
}
// sendEntries sends entries as one message — itself when alone, a digest when several — and records it. A
// record said again where its message is shown replaces that message (issue 373): one message per problem.
func (h *Holder) sendEntries(ch Channel, entries []entry, now time.Time) bool {
name := ch.Name()
var m Message
prior := ""
if len(entries) == 1 {
m = entries[0].msg
prior = h.replacing(ch, entries[0])
} else {
items := make([]item, len(entries))
msgs := make([]Message, len(entries))
@@ -343,7 +359,13 @@ func (h *Holder) sendEntries(ch Channel, entries []entry, now time.Time) bool {
}
m = digest(items, msgs)
}
id, err := ch.Send(m)
var id string
var err error
if prior != "" {
id, err = prior, ch.(replacer).Replace(prior, m)
} else {
id, err = ch.Send(m)
}
h.noteChannel(name, err)
if err != nil {
return false
@@ -367,22 +389,25 @@ func (h *Holder) sendEntries(ch Channel, entries []entry, now time.Time) bool {
if r.Sent == nil {
r.Sent = map[string]string{}
}
if _, has := r.Sent[name]; !has {
own := ""
if len(entries) == 1 {
own = id
}
r.Sent[name] = own
if len(entries) == 1 {
// Its own message: the one later words replace or edit — also after it was said in a digest.
r.Sent[name] = id
} else if _, has := r.Sent[name]; !has {
r.Sent[name] = ""
}
if r.FirstSent.IsZero() {
r.FirstSent = now
}
r.LastSaid = now
}
}
h.mu.Unlock()
key, what := "(digest)", fmt.Sprintf("digest of %d", len(entries))
if len(entries) == 1 {
key, what = entries[0].it.Key, entries[0].it.What
if prior != "" {
what += ", in place of its earlier message"
}
}
h.record(Sent{At: now, Channel: name, Key: key, What: what, Outcome: "sent"})
for _, e := range entries {
@@ -243,6 +243,15 @@ func (c *SeatChannel) Send(m Message) (string, error) {
return handle, c.put(handle, m, false)
}
// Replace says a message again under the handle of one shown before: the channel takes the earlier one away
// and shows this one as a new message is shown — ringing as its kind rings (issue 373). An edit is quiet.
func (c *SeatChannel) Replace(handle string, m Message) error {
if handle == "" {
return errors.New("no message on record to replace")
}
return c.put(handle, m, false)
}
// Edit replaces the message a handle names.
func (c *SeatChannel) Edit(handle string, m Message) error {
if handle == "" {
@@ -311,6 +320,13 @@ func (c *KindChannel) Send(m Message) (string, error) {
handle := newHandle()
return handle, s.putOn(c.Kind, handle, m, false)
}
func (c *KindChannel) Replace(handle string, m Message) error {
if handle == "" {
return errors.New("no message on record to replace")
}
s := &SeatChannel{Bench: c.Bench, Publish: c.Publish}
return s.putOn(c.Kind, handle, m, false)
}
func (c *KindChannel) Edit(handle string, m Message) error {
s := &SeatChannel{Bench: c.Bench, Publish: c.Publish}
return s.putOn(c.Kind, handle, m, true)
+1
View File
@@ -68,6 +68,7 @@ func (h *Holder) Sync(open []Condition, asked time.Time) SyncResult {
if !old.Cleared.IsZero() {
// Ended here, open there: back to open, quietly — the reading is not an event.
old.Cleared = time.Time{}
old.ClearUnsaid, old.ClearSilent = false, false
old.Count++
}
old.Summary, old.Subject, old.Kind, old.SilencedTill = rec.Summary, rec.Subject, rec.Kind, rec.SilencedTill
+17
View File
@@ -198,6 +198,23 @@ func when(t, now time.Time, loc *time.Location) string {
return lt.Format("2 Jan 15:04")
}
// by is a moment ahead as the operator reads it, in their zone: "23:33" today, "tomorrow 23:33", "Fri 16 Oct
// 23:33" later. An ask's expiry was said as a clock time alone, so an ask lasting a week read as one ending
// tonight (issue 373).
func by(t, now time.Time, loc *time.Location) string {
lt, ln := t.In(loc), now.In(loc)
y1, m1, d1 := lt.Date()
y2, m2, d2 := ln.Date()
if y1 == y2 && m1 == m2 && d1 == d2 {
return lt.Format("15:04")
}
ty, tm, td := ln.AddDate(0, 0, 1).Date()
if y1 == ty && m1 == tm && d1 == td {
return "tomorrow " + lt.Format("15:04")
}
return lt.Format("Mon 2 Jan 15:04")
}
func capital(s string) string {
if s == "" {
return s
@@ -225,6 +225,8 @@ func TestRealNotificationsReadInOneGlanceAndAreAnsweredFromTheNotification(t *te
t.Fatal(err)
}
h.Condition(EventCleared, cleared)
c.pass(ClearSettle)
h.Tick()
end := dt.edits["g14=1"]
if end.Title != e.cleared || end.Body != "" || len(end.Actions) != 0 {
t.Errorf("clearance %q / %q / %v, want one line %q", end.Title, end.Body, end.Actions, e.cleared)
+3
View File
@@ -8,6 +8,9 @@ as the operator.
- **Shows what the router sends** (its kind's worker on the `channel` queue): a message, a question with
buttons — each button carries only the router's one-time ticket — or an edit of a message shown before.
It sends to the account the router names as linked, or, before one is, to the chat a `/start` came from.
A message sent under a handle already shown (a reminder, an escalation) **replaces** it (issue 373): the
earlier message is deleted and the new one sent, ringing as its kind rings, since an edit here notifies
nobody; one the service will not delete loses its buttons and says a newer message follows.
- **Says what the operator chose** on `intake` under its own kind: a tapped button (`choice`), with the
sender as Telegram authenticated it; `/start` (`link`); a message it could not deliver (`failed`); and
whether it can send (`standing`). Only a private chat with the account itself counts.
+24 -2
View File
@@ -10,7 +10,8 @@ package main
// handle). Up to 4096 characters arrive whole; longer is cut and the cut said (D1). A warning or a
// clearing arrives without sound, an urgent message or a question rings (D4). A rate limit is waited out
// as told (D5); a refusal is a permanent failure, said to the router, never retried; an edit to the same
// text is a success (D6). A message that cannot be delivered after three tries is said as failed.
// text is a success (D6). A message that cannot be delivered after three tries is said as failed. A
// `send` or `show` under a handle already shown replaces that message: deleted, and the new one sent.
// - **What arrives** is long-polled: outbound only. A tapped button is said as a `choice` with the sender
// the service authenticated, and acknowledged on the phone with the router's word on it — what was
// recorded, or why it was not — which comes as a reply to the tap; without one in TapWait, with words that
@@ -232,11 +233,19 @@ func (c *Channel) Take(subject string, body json.RawMessage) error {
words, buttons := text(m), c.buttons(m)
silent := m.Silent || (!m.Urgent && len(buttons) == 0)
var place *Placed
if verb == asks.Edit {
if verb == asks.Edit || !m.Secret {
if place, err = c.Store.Placed(m.Handle); err != nil {
return err // the state is away: offered again
}
}
if verb != asks.Edit && place != nil {
// **A message said again under a handle already shown replaces it** (issue 373): a reminder or an
// escalation is one message per problem, not one more beside it. An edit here notifies nobody, so the
// earlier message is deleted and this one sent, ringing as its kind rings; one the service will not
// delete loses its buttons and says where the newer one is.
c.replaced(*place)
place = nil
}
for try := 1; ; try++ {
var id int64
switch {
@@ -274,6 +283,19 @@ func (c *Channel) Take(subject string, body json.RawMessage) error {
}
}
// replaced takes an earlier message out of the chat: deleted, or where the service refuses that (a message
// older than two days), its buttons taken away and the newer message named.
func (c *Channel) replaced(p Placed) {
err := c.Bot.Delete(p.Chat, p.ID)
if err == nil {
return
}
c.Logf("[telegram] an earlier message could not be deleted (%v); its buttons are taken away instead", err)
if err := c.Bot.Edit(p.Chat, p.ID, "Replaced by a newer message below.", nil); err != nil && !gone(err) {
c.Logf("[telegram] an earlier message could not be marked replaced either: %v", err)
}
}
func (c *Channel) newMessage(handle string, chat int64, words string, buttons [][2]string, silent bool, id *int64) error {
got, err := c.Bot.Send(chat, words, buttons, silent)
if err != nil {
@@ -580,3 +580,48 @@ func TestAToastIsCutAtTelegramsBound(t *testing.T) {
t.Errorf("toast %q", got)
}
}
// Issue 373: a reminder or an escalation is sent under the handle of the message it follows, and replaces it:
// the earlier one leaves the chat and the new one rings as its kind rings. One the service will not delete
// loses its buttons. A new handle is a new message, and nothing is deleted.
func TestAMessageSaidAgainUnderItsHandleReplacesTheEarlierOne(t *testing.T) {
c, f, _, st := newChannel(t, goodToken)
subject, body := work(asks.Show, asks.Message{Handle: "m1", Title: "Urgent: ace silent", Urgent: true, To: "42",
Buttons: []asks.Button{{Label: "Silence for a week", Ticket: "0123456789abcdef0123456789abcdef"}}})
if err := c.Take(subject, body); err != nil {
t.Fatal(err)
}
first := st.placed["m1"].ID
subject, body = work(asks.Send, asks.Message{Handle: "m2", Title: "another problem", To: "42"})
_ = c.Take(subject, body)
if len(f.of("deleteMessage")) != 0 {
t.Fatal("a new handle deleted a message")
}
subject, body = work(asks.Show, asks.Message{Handle: "m1", Title: "Still open: ace silent", Urgent: true, To: "42",
Buttons: []asks.Button{{Label: "Silence for a week", Ticket: "fedcba9876543210fedcba9876543210"}}})
if err := c.Take(subject, body); err != nil {
t.Fatal(err)
}
del := f.of("deleteMessage")
if len(del) != 1 || del[0].Body["message_id"] != float64(first) {
t.Fatalf("the earlier message was not deleted: %+v", del)
}
sent := f.of("sendMessage")
if len(sent) != 3 || sent[2].Body["disable_notification"] != false || st.placed["m1"].ID == first {
t.Fatalf("the reminder was not sent ringing in its place: %d sends, placed %d", len(sent), st.placed["m1"].ID)
}
// The service refuses the delete: the earlier message loses its buttons instead.
f.answer["deleteMessage"] = func(map[string]any) (int, string) {
return 400, `{"ok":false,"error_code":400,"description":"Bad Request: message can't be deleted"}`
}
second := st.placed["m1"].ID
subject, body = work(asks.Send, asks.Message{Handle: "m1", Title: "Now urgent: ace silent", Urgent: true, To: "42"})
_ = c.Take(subject, body)
ed := f.of("editMessageText")
if len(ed) != 1 || ed[0].Body["message_id"] != float64(second) || !strings.Contains(ed[0].Body["text"].(string), "Replaced") {
t.Fatalf("an undeletable earlier message kept its buttons: %+v", ed)
}
if kb, _ := json.Marshal(ed[0].Body["reply_markup"]); string(kb) != `{"inline_keyboard":[]}` {
t.Errorf("its buttons stayed: %s", kb)
}
}