Files
mesh-catalog/modules/messenger/cmd/messenger/holder.go
T
jochen f8515219b1
mesh/merge-gate pass: builds desk-channel, messenger, telegram → novox; no bus step; every machine composes with the change as it did without (4 of 4 compo…
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery-group group fix/373-one-message-per-condition ready: every member ready, and composed together they pass
mesh/delivery superseded: a newer head of the same pull request
Review: a blip swallows no escalation or waiting reminder; replace safely (issue 373)
- A comeback before the clearing was said rings when it is urgent now, and
  says the reminder or escalation that was waiting when it cleared.
- Comebacks never said are counted apart, so "again, Nth time" counts only
  what the operator was told; an urgent comeback after a said clearing rings
  again in place of its message.
- telegram sends the new message before deleting the earlier one, so a
  failed send leaves the operator a message.
- desk-channel keeps a machine's notification on record when its replace
  fails, and says a message shown nowhere as failed.
- A store that cannot be read sends a new message instead of holding it.
- Tests for each, and for a reminder in a burst and a clearing the channel
  could not take.
2026-10-10 14:22:53 +02:00

965 lines
34 KiB
Go

package main
// The holder of the operator-channel seat (novox/hq to-be 45 §5, ADR 0227): what is sent, to whom,
// when, and how often. The controller decides what is wrong; this decides what is said.
//
// **History is never news** (novox/hq issue 271). The events stream keeps what it was given and hands
// all of it to a consumer that is new, recreated or was away; on 2026-10-06 a newly made consumer
// replayed three hours of raised-and-cleared conditions and each was said as if it had just happened.
// So what is said is decided by the state now and by when a thing happened, never by when it arrived:
//
// - On start, and whenever it may have missed something (every ten minutes, and at once when an
// old event arrives), the holder reads what is open now from the controller (`conditions`) and
// takes it as state: nothing it says is sent for what it already said, what cleared meanwhile is
// forgotten without a word (a Telegram message is edited, which notifies nobody), an urgent
// condition open and never said is said — all of them in one message — and a warning raised in
// the last ten minutes is said like any other; an older warning is state.
// - An event older than that reading is already in it, and is not acted on. An event older than
// FreshFor (ten minutes) by its own time is state only: it is recorded, never said.
// - A clearing of something never said says nothing; a held message whose condition cleared before
// it went out is dropped.
// - Nothing is sent in the first minute after start (StartGrace) but the one message of urgent
// conditions open in the controller's state; what arrives meanwhile waits for the minute's end.
//
// What is said, and how often:
//
// - On `condition-raised`: one message, deduplicated by the condition's key. Said again with the
// same key while open, it is the same message, not a second.
// - Once more if still open after 1 hour (urgent) or 12 hours (warning), counted from when it was last
// said; a reminder or an escalation replaces its message, never stands beside it (issue 373).
// - On `condition-cleared`: once it stays cleared ClearSettle, its message is edited where the channel can
// (both can). Raised again before that, nothing is said: to the operator it never ended. Cleared, said
// so, and raised again within ten minutes, it is the same message, edited back to open.
// - A silenced condition sends nothing.
// - Urgent to both channels; warning to the desktop when the operator's session is there,
// otherwise to Telegram.
// - **Bursts are one message.** Messages for a channel are held for BurstWindow (30 s) from the
// first and go out together: one alone is sent as itself, several as one digest ("5 new
// warnings: …"). An urgent message is sent at once when nothing went out on its channel in the
// last 30 s.
// - **The desktop is gentle:** warnings reach it as at most one message every DesktopWarningEvery
// (15 min); urgent ones are not held to that.
// - At most twenty messages an hour per channel — a last line of defence, since the above keeps a
// channel far below it. Reached, it is said once in a message of its own and the rest is held,
// to go out as one digest when the hour allows.
// - A message carrying an address, a path or a secret is refused (content.go); what is sent in its
// place says `channel-refused`, with the offending part withheld.
// - A channel that cannot send says so in the status and the log, and is tried again every
// minute while what it holds is still open.
import (
"fmt"
"sort"
"strings"
"sync"
"time"
)
const (
RemindUrgent = time.Hour
RemindWarning = 12 * time.Hour
// QuietRemindAfter: a warning kept quiet as needing nothing is said once after this.
QuietRemindAfter = 24 * time.Hour
ReopenWindow = 10 * time.Minute
// ClearSettle: a clearing is said once the condition stayed cleared this long. One raised again sooner
// (a unit failing again after each apply, issue 373) continues its message without a word, and its
// clearing is never said. Once it has come back, its next clearing waits the reopen window less a minute.
ClearSettle = 2 * time.Minute
CapPerHour = 20
KeptSends = 200
KeptRefusals = 50
// FreshFor: an event older than this by its own time is history — state, never a message.
FreshFor = 10 * time.Minute
// BurstWindow: what arrives for a channel within it goes out as one message.
BurstWindow = 30 * time.Second
// DesktopWarningEvery: warnings reach the desktop at most this often.
DesktopWarningEvery = 15 * time.Minute
// StartGrace: after start, nothing but the urgent conditions open in the controller's state.
StartGrace = time.Minute
// ResyncEvery: how often the open set is read from the controller again.
ResyncEvery = 10 * time.Minute
// RetryEvery: a channel that failed is tried again after this.
RetryEvery = time.Minute
// ClockSlack: the controller's clock and this one may differ by this much; an event this close
// to the reading is taken rather than assumed to be in it (taking one twice is harmless).
ClockSlack = 5 * time.Second
// DigestLines: a digest names at most this many, then says how many more.
DigestLines = 15
)
// Message is what a channel shows: a title line and a body, and the answers it offers.
type Message struct {
Title string
Body string
Urgent bool
Quiet bool // a clearing: shown without urgency
Actions []Choice
// Reply is the answer this message answers, by its id: the channel shows it where that was given.
Reply string
// To is the account it is for, when not the one linked: a notice to an account being unlinked, or an
// ask put to the account a relink replaces.
To string
// Secret: shown once, and kept by no channel (a link's code).
Secret bool
}
// Choice is one answer a message offers: the token a channel hands back when it is chosen, and its label.
type Choice struct {
Token string `json:"token"`
Label string `json:"label"`
}
func (m Message) Text() string {
if m.Body == "" {
return m.Title
}
return m.Title + "\n" + m.Body
}
// Channel is one way to the operator.
type Channel interface {
Name() string
Ready() error
Send(Message) (string, error)
Edit(id string, m Message) error
CanEdit() bool
}
// replacer is a channel that can say a message again in place of the one it showed under that id: the earlier
// one goes, the new one is shown as a new message is — it rings as its kind rings (issue 373). A reminder or
// an escalation is then one message per problem, never one more beside it.
type replacer interface {
Replace(id string, m Message) error
}
func replaces(ch Channel) bool {
_, ok := ch.(replacer)
return ok
}
// silentEditor is a channel whose edits notify nobody (Telegram's are): what is over is said there
// even when it ended while the holder was away, since saying it costs the operator nothing.
type silentEditor interface{ SilentEdit() bool }
func editsSilently(ch Channel) bool {
s, ok := ch.(silentEditor)
return ok && s.SilentEdit()
}
// Record is one open message, kept in the module's own state so a restart forgets nothing.
type Record struct {
Key string `json:"key"`
Kind string `json:"kind"`
Subject string `json:"subject"`
Severity string `json:"severity"`
Summary string `json:"summary"`
// Headline, Explanation and Resolved are the plain words the operator reads (novox/hq ADR 0253):
// the controller's, or a module's through notify; empty, the message is said from the summary.
Headline string `json:"headline,omitempty"`
Explanation string `json:"explanation,omitempty"`
Resolved string `json:"resolved,omitempty"`
// Needs is what the operator does, empty for nothing; Actions the answers offered, each by a token
// (Tokens: token -> its grant; Index -1 is Details, one per message shown) a channel hands back when chosen.
Needs string `json:"needs,omitempty"`
Actions []Action `json:"actions,omitempty"`
// Kept as "grants": the build before ADR 0258 kept "tokens" as token -> index, which this form cannot
// read; under its own name it is ignored and the record loads (the old tokens die, as they must).
Tokens map[string]Grant `json:"grants,omitempty"`
// QuietReminded: a warning kept quiet as needing nothing was said once after QuietRemindAfter, so
// nothing stays silent for ever.
QuietReminded bool `json:"quiet_reminded,omitempty"`
// Answered is what the operator chose from a notification, and what came of it.
Answered string `json:"answered,omitempty"`
// Ask is the open ask about this record, `<asker>.<id>`: its options are this record's answers
// (novox/hq ADR 0259).
Ask string `json:"ask,omitempty"`
Origin string `json:"origin"`
More string `json:"more"`
Raised time.Time `json:"raised"`
SilencedTill time.Time `json:"silenced_till,omitempty"`
Sent map[string]string `json:"sent,omitempty"` // channel -> its message's id; "" when said in a digest
FirstSent time.Time `json:"first_sent,omitempty"`
// LastSaid is when a message about it last went out: a reminder is due its bound after it, never sooner
// (an escalation followed by a reminder 31 s later, issue 373).
LastSaid time.Time `json:"last_said,omitempty"`
// ClearUnsaid: cleared, and the clearing not said yet — it is said once it stays cleared ClearSettle;
// raised again before, the message continues as if it never cleared. ClearSilent: said only where an edit
// notifies nobody (it ended while this holder was away).
ClearUnsaid bool `json:"clear_unsaid,omitempty"`
ClearSilent bool `json:"clear_silent,omitempty"`
// ClearHeld is what was waiting to be said of it when it cleared (a reminder, an escalation): said after
// all if it comes back before its clearing was said. Blips counts those comebacks, never said, so a
// condition that flaps is known as one and its message's "again, Nth time" counts only what was said.
ClearHeld string `json:"clear_held,omitempty"`
Blips int `json:"blips,omitempty"`
Reminded bool `json:"reminded,omitempty"`
Pending string `json:"pending,omitempty"` // what could not be said yet: raised, reminder, …
Folded bool `json:"folded,omitempty"` // held by the cap
Cleared time.Time `json:"cleared,omitempty"`
Count int `json:"count"`
Refused string `json:"refused,omitempty"`
// History: known from the state or from an old event, and never said — not news.
History bool `json:"history,omitempty"`
// Seen is when this holder last took word of it; the open set read after it is authoritative.
Seen time.Time `json:"seen,omitempty"`
}
// recordOf is a condition as this holder keeps it.
func recordOf(c Condition) Record {
return Record{Key: c.Key, Kind: c.Kind, Subject: c.SubjectWords(), Severity: c.Severity, Summary: c.Summary,
Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved, Needs: c.Needs, Actions: c.Actions,
Origin: "condition",
More: "conditions show " + c.Key, Raised: c.Raised, SilencedTill: c.SilencedTill}
}
// takeWords keeps the plain words current, as the summary is: a condition's words follow what is seen.
func (r *Record) takeWords(from Record) {
r.Headline, r.Explanation, r.Resolved, r.Needs = from.Headline, from.Explanation, from.Resolved, from.Needs
if !sameActions(r.Actions, from.Actions) {
// Details performs nothing and names no action, so its tickets outlive a change of the actions:
// every message still shown keeps its Details (issue 371). Any other ticket dies with its action.
r.Actions = from.Actions
for t, g := range r.Tokens {
if g.Index != -1 {
delete(r.Tokens, t)
}
}
}
}
func (r *Record) silenced(now time.Time) bool {
return !r.SilencedTill.IsZero() && now.Before(r.SilencedTill)
}
func (r *Record) told() bool { return len(r.Sent) > 0 }
// Sent is one message that went out, or was held, for the history.
type Sent struct {
At time.Time `json:"at"`
Channel string `json:"channel"`
Key string `json:"key"`
What string `json:"what"`
Outcome string `json:"outcome"` // sent, edited, held, in digest, failed: <why>
}
// RefusalNote is one refused message: never its text.
type RefusalNote struct {
At time.Time `json:"at"`
Key string `json:"key"`
Class string `json:"class"`
What string `json:"what"`
}
// Store is the module's own state (ADR 0201): the open messages, and the recent sends.
type Store interface {
Put(r Record) error
Delete(key string) error
All() ([]Record, error)
PutRecent([]Sent) error
Recent() ([]Sent, error)
}
// Holder is the seat's holder: the router (novox/hq ADR 0234 §5, ADR 0259). It keeps no channel's client:
// Away is the channel that reaches the operator away from the machines (a kind promising reaches-away),
// Desk the one at the machines; each is a kind of the `channel` bench, held by a module of its own.
type Holder struct {
Away Channel
Desk Channel
Store Store
// Asks keeps the asks, the operator's identities and their links (asks.go, link.go); Bench is what
// the controller says of every channel kind and what each says of itself; Publish says on a seat.
Asks AskStore
Bench *Bench
Publish func(subject string, body any, id string) error
// Node is the machine this router runs on; StateDir its own directory, where root leaves a break-glass
// request (selfact.go).
Node string
StateDir string
Now func() time.Time
Logf func(string, ...any)
// Emit states a fact as this module (refused); nil states nothing.
Emit func(event string, body any) error
// Ask calls a seat verb through the mesh, as this module: the answers the operator chooses.
Ask Asker
// RootFree asks the controller's verb root-free about machines; nil asks it through Ask (askRootFree).
RootFree func(machines []string) (map[string]RootVerdict, error)
// Zone is the operator's time zone, which every time in a message is said in; nil is this machine's.
Zone func() *time.Location
work sync.Mutex // one event, call or tick at a time
mu sync.Mutex // what the status reads
open map[string]*Record
recent []Sent
refusals []RefusalNote
outbox map[string][]item
lastSend map[string]time.Time
lastWarn map[string]time.Time
lastTry map[string]time.Time
capSaid map[string]time.Time
chanErr map[string]string
chanErrAt map[string]time.Time
chanOK map[string]time.Time
unreadable int
lastBad string
lastBadAt time.Time
saidOnce map[string]time.Time
storeErr string
heard map[string]int
lastHeard time.Time
started time.Time
syncedAt time.Time
syncErr string
wantSync bool
history int
stale int
// answered is what came of each answer event, by its id: a redelivered one is acted on once.
answered map[string]string
// asks are the asks this router holds, by `<asker>.<id>`, and tickets each ask's tickets to it.
asks map[string]*askEntry
tickets map[string]string
// lastUnlink is the last unlink of each kind.
lastUnlink map[string]Unlinked
// rootWord is the controller's last word on whether an agent can become root, by the machines asked.
rootWord map[string]rootWord
}
func (h *Holder) init() {
if h.open == nil {
h.open = map[string]*Record{}
h.outbox = map[string][]item{}
h.lastSend = map[string]time.Time{}
h.lastWarn = map[string]time.Time{}
h.lastTry = map[string]time.Time{}
h.capSaid = map[string]time.Time{}
h.chanErr = map[string]string{}
h.chanErrAt = map[string]time.Time{}
h.chanOK = map[string]time.Time{}
h.saidOnce = map[string]time.Time{}
h.heard = map[string]int{}
}
if h.Now == nil {
h.Now = time.Now
}
if h.Logf == nil {
h.Logf = func(string, ...any) {}
}
}
// Load reads back what was open and what was sent before a restart. What was waiting to be said is
// not said now: it is history, and the controller's state, read next, decides what is still news.
func (h *Holder) Load() error {
h.work.Lock()
defer h.work.Unlock()
h.mu.Lock()
defer h.mu.Unlock()
h.init()
if h.Store == nil {
return nil
}
recs, err := h.Store.All()
if err != nil {
h.storeErr = err.Error()
return err
}
for i := range recs {
r := recs[i]
if r.Pending != "" || r.Folded {
r.Pending, r.Folded = "", false
if !r.told() {
r.History = true
}
}
h.open[r.Key] = &r
}
if sent, err := h.Store.Recent(); err == nil {
h.recent = sent
}
h.mu.Unlock()
err = h.loadAsks()
h.mu.Lock()
return err
}
// Start marks the holder's start: the grace minute counts from here.
func (h *Holder) Start() {
h.work.Lock()
defer h.work.Unlock()
h.mu.Lock()
defer h.mu.Unlock()
h.init()
h.started = h.Now()
}
func (h *Holder) inGrace(now time.Time) bool {
return !h.started.IsZero() && now.Sub(h.started) < StartGrace
}
// Condition takes one of the controller's condition events. Whether it is news is decided by its own
// time, never by its arrival.
func (h *Holder) Condition(event string, c Condition) {
h.work.Lock()
defer h.work.Unlock()
h.mu.Lock()
h.init()
now := h.Now()
h.heard[event]++
h.lastHeard = now
synced := h.syncedAt
h.mu.Unlock()
rec := recordOf(c)
when := c.When(event)
quiet := false
switch {
case !when.IsZero() && !synced.IsZero() && when.Before(synced.Add(-ClockSlack)):
// Already in the state read from the controller after it: nothing to take.
h.mu.Lock()
h.history++
h.mu.Unlock()
return
case !when.IsZero() && now.Sub(when) > FreshFor:
// Old: a replay, a redelivery, or the holder was away. State, never a message — and the
// controller's state is read again, since what came between may be missing too.
quiet = true
h.mu.Lock()
h.stale++
first := !h.wantSync
h.wantSync = true
h.mu.Unlock()
if first {
h.Logf("[messenger] %s %s happened %s ago: taken as state, not news; reading the open conditions again",
event, c.Key, roughly(now.Sub(when)))
}
}
switch event {
case EventRaised:
h.raised(rec, quiet)
case EventChanged:
h.changed(rec, quiet)
case EventCleared:
h.cleared(rec.Key, quiet)
}
}
// Unreadable records an event that could not be read, and tells the operator — once an hour.
func (h *Holder) Unreadable(key string, err error) {
h.work.Lock()
defer h.work.Unlock()
h.mu.Lock()
h.init()
h.unreadable++
h.lastBad = err.Error()
h.lastBadAt = h.Now()
n := h.unreadable
h.mu.Unlock()
h.Logf("[messenger] refused %s: %v (unreadable events since start: %d)", key, err, n)
h.sayOnce("messenger.unreadable-event", time.Hour, Message{
Title: "A mesh message could not be read",
Body: fmt.Sprintf("The messenger could not read %d update(s) from the controller, so a problem may be open "+
"that was not said. An agent can find out why in the messenger's status.", n),
})
}
func (h *Holder) raised(rec Record, quiet bool) {
now := h.Now()
h.mu.Lock()
old := h.open[rec.Key]
h.mu.Unlock()
if old != nil && old.Cleared.IsZero() {
// Said again while open: the same message. Its words are kept current; nothing is sent — unless
// it was kept quiet as needing nothing and now needs the operator.
h.mu.Lock()
wasQuiet := needsNothing(old)
old.Summary, old.Subject, old.Kind, old.SilencedTill = rec.Summary, rec.Subject, rec.Kind, rec.SilencedTill
old.takeWords(rec)
if rec.Severity != "" {
old.Severity = rec.Severity
}
old.Seen = now
h.mu.Unlock()
h.nowNeeded(old, wasQuiet, quiet)
h.persist(old)
return
}
if old != nil && now.Sub(old.Cleared) < ReopenWindow {
// Cleared and raised again within ten minutes: the same message, back to open (to-be 45 §2).
h.mu.Lock()
blip := old.ClearUnsaid
held := old.ClearHeld
escalated := old.Severity == Warning && rec.Severity == Urgent
gone := now.Sub(old.Cleared)
old.Cleared = time.Time{}
old.ClearUnsaid, old.ClearSilent, old.ClearHeld = false, false, ""
if blip {
old.Blips++
} else {
old.Count++
}
old.Summary, old.Severity, old.SilencedTill = rec.Summary, rec.Severity, rec.SilencedTill
old.takeWords(rec)
old.Seen = now
h.mu.Unlock()
switch {
case quiet || old.silenced(now):
case blip && escalated:
// Back before its clearing was said, and urgent now: said as an escalation, which rings.
h.deliver(old, "escalated")
case blip && held != "":
// Back before its clearing was said: what was waiting to be said of it when it cleared is said now.
h.deliver(old, held)
case blip:
// Its clearing was never said (issue 373): to the operator it never ended, so nothing is said now.
h.Logf("[messenger] %s back %s after it cleared, which was not said yet: the same message continues, nothing said",
old.Key, roughly(gone))
h.record(Sent{At: now, Channel: "-", Key: old.Key, What: "reopened", Outcome: "quiet: back before its clearing was said"})
case old.told() && old.Severity == Urgent:
// Its end was said, and it is back: urgent, it rings again, in place of its message.
h.deliver(old, "reopened")
case old.told():
h.edit(old, "reopened", false)
case !needsNothing(old) && !h.queued(old.Key):
// Cleared before it was ever said, and back: a problem the operator has not heard of is said.
h.deliver(old, "raised")
}
h.persist(old)
return
}
r := rec
r.Count = 1
if r.Raised.IsZero() {
r.Raised = now
}
r.Sent = map[string]string{}
r.Seen = now
r.History = quiet
h.mu.Lock()
h.open[r.Key] = &r
h.mu.Unlock()
switch {
case quiet:
case r.silenced(now):
h.Logf("[messenger] %s raised while silenced until %s: nothing sent", r.Key, r.SilencedTill.Format(time.RFC3339))
default:
h.deliver(&r, "raised")
}
h.persist(&r)
}
// nowNeeded says a record kept quiet as needing nothing, once its words say it needs the operator.
func (h *Holder) nowNeeded(r *Record, wasQuiet, quiet bool) {
if wasQuiet && !needsNothing(r) && !r.told() && !quiet && !r.silenced(h.Now()) && !h.queued(r.Key) {
h.deliver(r, "raised")
}
}
func (h *Holder) changed(rec Record, quiet bool) {
h.mu.Lock()
old := h.open[rec.Key]
h.mu.Unlock()
if old == nil || !old.Cleared.IsZero() {
// A change to something this holder never heard raised: read as a raise, so it is said
// (when it is news).
h.Logf("[messenger] %s changed and was not open here; taken as raised", rec.Key)
h.raised(rec, quiet)
return
}
h.mu.Lock()
escalated := old.Severity == Warning && rec.Severity == Urgent
wasQuiet := needsNothing(old)
old.Summary, old.Subject, old.SilencedTill = rec.Summary, rec.Subject, rec.SilencedTill
old.takeWords(rec)
if rec.Severity != "" {
old.Severity = rec.Severity
}
old.Seen = h.Now()
h.mu.Unlock()
switch {
case escalated && !quiet && !old.silenced(h.Now()):
// Routing differs for urgent: said once more, to both channels.
h.deliver(old, "escalated")
default:
h.nowNeeded(old, wasQuiet, quiet)
}
h.persist(old)
}
// cleared ends a record. Something never said is ended without a word; what was waiting to be said
// about it is dropped. quiet: it ended while this holder was not listening, so it is said only where
// an edit notifies nobody.
func (h *Holder) cleared(key string, quiet bool) {
now := h.Now()
h.mu.Lock()
old := h.open[key]
h.mu.Unlock()
if old == nil || !old.Cleared.IsZero() {
h.Logf("[messenger] %s cleared and was not open here: nothing to say", key)
return
}
held := h.queuedWhat(key)
h.mu.Lock()
if held == "" {
held = old.Pending
}
old.Cleared = now
old.Pending = ""
old.Folded = false
old.Seen = now
told := old.told()
h.mu.Unlock()
dropped := h.unqueue(key)
switch {
case old.silenced(now):
// A silenced condition sends nothing, its clearing included.
case !told:
if dropped > 0 {
h.Logf("[messenger] %s cleared before it was said: nothing to unsay", key)
}
default:
// Said once it stays cleared (ClearSettle; a condition that came back, ReopenWindow): a blip of a few
// seconds is not said at all (issue 373).
h.mu.Lock()
old.ClearUnsaid, old.ClearSilent, old.ClearHeld = true, quiet, held
h.mu.Unlock()
}
h.persist(old)
}
// clearSettle is how long a record stays cleared before its clearing is said: one that came back before
// waits the whole reopen window, so a condition that flaps is said cleared once, when it stopped flapping.
func clearSettle(r *Record) time.Duration {
if r.Count > 1 || r.Blips > 0 {
return ReopenWindow - time.Minute
}
return ClearSettle
}
// lastSaid is when a message about a record last went out.
func lastSaid(r *Record) time.Time {
if !r.LastSaid.IsZero() {
return r.LastSaid
}
return r.FirstSent
}
// Tick does what time asks: reminders, retries, what is held, and forgetting what cleared long
// enough ago that a new raise is a new message.
func (h *Holder) Tick() {
h.warmRootFree(false) // outside the work: what is offered inside it reads a recent word
h.work.Lock()
defer h.work.Unlock()
h.mu.Lock()
h.init()
now := h.Now()
var recs []*Record
for _, r := range h.open {
recs = append(recs, r)
}
h.mu.Unlock()
sort.Slice(recs, func(i, j int) bool { return recs[i].Raised.Before(recs[j].Raised) })
h.sweepAsks(now)
h.breakGlass()
for _, r := range recs {
switch {
case !r.Cleared.IsZero():
if r.ClearUnsaid && now.Sub(r.Cleared) >= clearSettle(r) {
h.mu.Lock()
r.ClearUnsaid = false
silent := r.ClearSilent
h.mu.Unlock()
if !r.silenced(now) {
h.edit(r, "cleared", silent)
}
h.persist(r)
}
if !r.ClearUnsaid && now.Sub(r.Cleared) >= ReopenWindow && !h.queued(r.Key) {
h.mu.Lock()
delete(h.open, r.Key)
h.mu.Unlock()
if h.Store != nil {
if err := h.Store.Delete(r.Key); err != nil {
h.noteStore(err)
}
}
}
case r.silenced(now):
case r.Pending != "" && !h.queued(r.Key):
// No channel could take it when it was said: tried again.
h.deliver(r, r.Pending)
h.persist(r)
case needsNothing(r) && !r.told() && !r.QuietReminded && now.Sub(r.Raised) >= QuietRemindAfter:
// Quiet for a day: said once, so nothing stays silent for ever (ADR 0258).
h.mu.Lock()
r.QuietReminded = true
h.mu.Unlock()
h.deliver(r, "quiet-reminder")
h.persist(r)
case !r.Reminded && !r.FirstSent.IsZero() && now.Sub(r.Raised) >= remindAfter(r.Severity) &&
now.Sub(lastSaid(r)) >= remindAfter(r.Severity):
// Once more after its bound — counted from its raising, and never sooner after it was
// last said (an old condition said at a reading of the state is not reminded at once, nor one
// just escalated: issue 373).
h.mu.Lock()
r.Reminded = true
h.mu.Unlock()
h.deliver(r, "reminder")
h.persist(r)
}
}
h.flush(now, false)
}
// Flush sends what is due on each channel; the runtime calls it every few seconds, so a held
// message waits its window and not a minute more.
func (h *Holder) Flush() {
h.warmRootFree(false) // outside the work: what is offered inside it reads a recent word
h.work.Lock()
defer h.work.Unlock()
h.mu.Lock()
h.init()
now := h.Now()
h.mu.Unlock()
h.flush(now, false)
}
func remindAfter(severity string) time.Duration {
if severity == Urgent {
return RemindUrgent
}
return RemindWarning
}
// say composes a record's message under the content rule: refused, it is composed again with less of
// it, until what remains may leave. The refusal is recorded and stated once per record.
func (h *Holder) say(r *Record, what string) Message { return h.sayOn(r, what, "") }
// sayOn is a record's message on one channel kind: its words, what came of an answer, the answers that kind
// can carry, and where the others can be given. An empty kind is the words alone.
func (h *Holder) sayOn(r *Record, what, kind string) Message {
m := h.words(r, what)
if what == "cleared" || m.Title == "A mesh message was withheld" {
return m
}
if r.Answered != "" {
m = h.answeredWords(r, m)
}
if kind == "" {
return m
}
withheld := map[string]bool{} // the labels of answers whose effect could not be said
if rec := h.openAsk(r); rec != nil {
// Who asks, as the bus proves it (the asker is the subject's, never the body's), what each answer does,
// and what happens if nobody answers (the review of 2026-10-09).
for _, line := range h.askLines(rec) {
if _, ok := Check(line); ok {
m.Body = strings.TrimSpace(m.Body + "\n" + line)
}
}
// **An answer whose effect is not said is not offered** (the confirmation review of 2026-10-09): its line
// refused by the content rule, its button went out all the same, and a tap would authorise what the
// person was not shown.
for _, o := range rec.Ask.Options {
if _, ok := Check(o.Label + ": " + o.Does + "."); !ok || strings.TrimSpace(o.Does) == "" {
withheld[o.Label] = true
}
}
if line := h.elsewhere(rec, kind); line != "" {
if _, ok := Check(line); ok {
m.Body = strings.TrimSpace(m.Body + "\n" + line)
}
}
}
for _, c := range h.choices(r, kind) {
if !withheld[c.Label] {
m.Actions = append(m.Actions, c)
}
}
return m
}
// answeredWords is a message that says what came of its question. An ask's own message is the question
// itself, so its title becomes how it ended — "You chose Approve at 13:30 on Telegram · <headline>" — read at
// a glance on the phone, where the edit rings nothing (issue 372); the rest of what came of it, the
// explanation and when it was asked follow. A condition's message keeps its own title, and says it first.
func (h *Holder) answeredWords(r *Record, m Message) Message {
if r.Origin == "ask" {
first, rest, _ := strings.Cut(r.Answered, "\n")
headline, explanation := wordsOf(r)
out := Message{Title: strings.TrimSuffix(first, ".") + " · " + headline, Urgent: m.Urgent, Quiet: m.Quiet,
Body: strings.TrimSpace(rest + "\n" + explanation + "\nAsked " + when(r.Raised, h.Now(), h.zone()) + ".")}
if _, ok := Check(out.Text()); ok {
return out
}
// Its words refused (they are the asker's): the outcome is still said, before what may leave.
}
m.Body = strings.TrimSpace(r.Answered + "\n" + m.Body)
return m
}
// askLines are what an ask's message says of the ask itself: who asks, each option's effect, and its expiry.
func (h *Holder) askLines(rec *AskRecord) []string {
asker := rec.Asker
if asker == SelfAsker {
asker = "the router itself"
}
lines := []string{"Asked by " + asker + "."}
for _, o := range rec.Ask.Options {
if o.Does != "" {
lines = append(lines, o.Label+": "+o.Does+".")
}
}
if !rec.Ask.Expires.IsZero() && rec.Ask.OnExpiry != "" {
lines = append(lines, "If nobody answers by "+by(rec.Ask.Expires, h.Now(), h.zone())+": "+rec.Ask.OnExpiry+".")
}
return lines
}
// words is a record's message under the content rule: refused, it is composed again with less of it,
// until what remains may leave. The refusal is recorded and stated once per record.
func (h *Holder) words(r *Record, what string) Message {
now := h.Now()
if r.Refused != "" {
// Refused before: its words stay withheld in every later message too.
for w := 1; w <= 3; w++ {
m := compose(r, what, now, h.zone(), w)
if _, ok := Check(m.Text()); ok {
return m
}
}
}
m := compose(r, what, now, h.zone(), 0)
refusal, ok := Check(m.Text())
if ok {
return m
}
h.mu.Lock()
r.Refused = refusal.What
h.mu.Unlock()
key := r.Key
if _, keyOK := Check(key); !keyOK {
key = "(withheld)"
}
h.mu.Lock()
h.refusals = append(h.refusals, RefusalNote{At: now, Key: key, Class: refusal.Class, What: refusal.What})
if len(h.refusals) > KeptRefusals {
h.refusals = h.refusals[len(h.refusals)-KeptRefusals:]
}
h.mu.Unlock()
h.Logf("[messenger] refused the message for %s: it carried %s; sending channel-refused with its words withheld", key, refusal)
if h.Emit != nil {
if err := h.Emit("refused", map[string]any{"key": key, "class": refusal.Class, "what": refusal.What}); err != nil {
h.Logf("[messenger] could not state the refusal on the bus: %v", err)
}
}
for w := 1; w <= 3; w++ {
m := compose(r, what, now, h.zone(), w)
if _, ok := Check(m.Text()); ok {
return m
}
}
return withheld(refusal.What, r.Severity == Urgent)
}
// deliver sends a record's message where its severity routes it.
func (h *Holder) channels() []Channel {
var out []Channel
for _, c := range []Channel{h.Away, h.Desk} {
if c != nil {
out = append(out, c)
}
}
return out
}
func (h *Holder) record(s Sent) {
h.mu.Lock()
h.recent = append(h.recent, s)
if len(h.recent) > KeptSends {
h.recent = h.recent[len(h.recent)-KeptSends:]
}
recent := append([]Sent(nil), h.recent...)
h.mu.Unlock()
if h.Store != nil {
if err := h.Store.PutRecent(recent); err != nil {
h.noteStore(err)
}
}
}
func (h *Holder) persist(r *Record) {
if h.Store == nil {
return
}
h.mu.Lock()
c := *r
h.mu.Unlock()
if err := h.Store.Put(c); err != nil {
h.noteStore(err)
}
}
func (h *Holder) noteStore(err error) {
h.mu.Lock()
first := h.storeErr == ""
h.storeErr = err.Error()
h.mu.Unlock()
if first {
h.Logf("[messenger] cannot write its state (open messages are held in memory only until it can): %v", err)
}
}
func (h *Holder) noteChannel(name string, err error) {
h.mu.Lock()
defer h.mu.Unlock()
now := h.Now()
if err == nil {
if h.chanErr[name] != "" {
h.Logf("[messenger] %s sends again", name)
}
h.chanErr[name] = ""
h.chanOK[name] = now
return
}
// Said in the log when it changes, and at most every ten minutes while it holds.
if h.chanErr[name] != err.Error() || now.Sub(h.chanErrAt[name]) >= 10*time.Minute {
h.Logf("[messenger] %s cannot send: %v", name, err)
h.chanErrAt[name] = now
}
h.chanErr[name] = err.Error()
}
func (h *Holder) whyNot() string {
var parts []string
for _, ch := range h.channels() {
if err := ch.Ready(); err != nil {
parts = append(parts, ch.Name()+": "+err.Error())
continue
}
h.mu.Lock()
e := h.chanErr[ch.Name()]
h.mu.Unlock()
if e != "" {
parts = append(parts, ch.Name()+": "+e)
}
}
if len(parts) == 0 {
return "no channel"
}
return strings.Join(parts, "; ")
}
func roughly(d time.Duration) string {
switch {
case d < 0:
return "a moment"
case d < 2*time.Minute:
return fmt.Sprintf("%d s", int(d.Seconds()))
case d < 2*time.Hour:
return fmt.Sprintf("%d min", int(d.Minutes()))
case d < 48*time.Hour:
return fmt.Sprintf("%.1f h", d.Hours())
}
return fmt.Sprintf("%d days", int(d.Hours()/24))
}