mesh/merge-gate pass: builds dunst, messenger → g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group fix/notifications-after-review delivered: every member is delivered
The build live since 14:38 keeps tokens as token -> index; reading them as grants failed and skipped the record. A quiet warning known from history was never reminded, and an answer could silence another condition than its own.
744 lines
24 KiB
Go
744 lines
24 KiB
Go
package main
|
|
|
|
// The holder of the operator-channel seat (novox/hq to-be 45 §5, ADR 0227): what is sent, to whom,
|
|
// when, and how often. The controller decides what is wrong; this decides what is said.
|
|
//
|
|
// **History is never news** (novox/hq issue 271). The events stream keeps what it was given and hands
|
|
// all of it to a consumer that is new, recreated or was away; on 2026-10-06 a newly made consumer
|
|
// replayed three hours of raised-and-cleared conditions and each was said as if it had just happened.
|
|
// So what is said is decided by the state now and by when a thing happened, never by when it arrived:
|
|
//
|
|
// - On start, and whenever it may have missed something (every ten minutes, and at once when an
|
|
// old event arrives), the holder reads what is open now from the controller (`conditions`) and
|
|
// takes it as state: nothing it says is sent for what it already said, what cleared meanwhile is
|
|
// forgotten without a word (a Telegram message is edited, which notifies nobody), an urgent
|
|
// condition open and never said is said — all of them in one message — and a warning raised in
|
|
// the last ten minutes is said like any other; an older warning is state.
|
|
// - An event older than that reading is already in it, and is not acted on. An event older than
|
|
// FreshFor (ten minutes) by its own time is state only: it is recorded, never said.
|
|
// - A clearing of something never said says nothing; a held message whose condition cleared before
|
|
// it went out is dropped.
|
|
// - Nothing is sent in the first minute after start (StartGrace) but the one message of urgent
|
|
// conditions open in the controller's state; what arrives meanwhile waits for the minute's end.
|
|
//
|
|
// What is said, and how often:
|
|
//
|
|
// - On `condition-raised`: one message, deduplicated by the condition's key. Said again with the
|
|
// same key while open, it is the same message, not a second.
|
|
// - Once more if still open after 1 hour (urgent) or 12 hours (warning).
|
|
// - On `condition-cleared`: the first message is edited where the channel can (both can).
|
|
// Cleared and raised again within ten minutes, it is the same message, edited back to open.
|
|
// - A silenced condition sends nothing.
|
|
// - Urgent to both channels; warning to the desktop when the operator's session is there,
|
|
// otherwise to Telegram.
|
|
// - **Bursts are one message.** Messages for a channel are held for BurstWindow (30 s) from the
|
|
// first and go out together: one alone is sent as itself, several as one digest ("5 new
|
|
// warnings: …"). An urgent message is sent at once when nothing went out on its channel in the
|
|
// last 30 s.
|
|
// - **The desktop is gentle:** warnings reach it as at most one message every DesktopWarningEvery
|
|
// (15 min); urgent ones are not held to that.
|
|
// - At most twenty messages an hour per channel — a last line of defence, since the above keeps a
|
|
// channel far below it. Reached, it is said once in a message of its own and the rest is held,
|
|
// to go out as one digest when the hour allows.
|
|
// - A message carrying an address, a path or a secret is refused (content.go); what is sent in its
|
|
// place says `channel-refused`, with the offending part withheld.
|
|
// - A channel that cannot send says so in the status and the log, and is tried again every
|
|
// minute while what it holds is still open.
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
RemindUrgent = time.Hour
|
|
RemindWarning = 12 * time.Hour
|
|
// QuietRemindAfter: a warning kept quiet as needing nothing is said once after this.
|
|
QuietRemindAfter = 24 * time.Hour
|
|
ReopenWindow = 10 * time.Minute
|
|
CapPerHour = 20
|
|
KeptSends = 200
|
|
KeptRefusals = 50
|
|
|
|
// FreshFor: an event older than this by its own time is history — state, never a message.
|
|
FreshFor = 10 * time.Minute
|
|
// BurstWindow: what arrives for a channel within it goes out as one message.
|
|
BurstWindow = 30 * time.Second
|
|
// DesktopWarningEvery: warnings reach the desktop at most this often.
|
|
DesktopWarningEvery = 15 * time.Minute
|
|
// StartGrace: after start, nothing but the urgent conditions open in the controller's state.
|
|
StartGrace = time.Minute
|
|
// ResyncEvery: how often the open set is read from the controller again.
|
|
ResyncEvery = 10 * time.Minute
|
|
// RetryEvery: a channel that failed is tried again after this.
|
|
RetryEvery = time.Minute
|
|
// ClockSlack: the controller's clock and this one may differ by this much; an event this close
|
|
// to the reading is taken rather than assumed to be in it (taking one twice is harmless).
|
|
ClockSlack = 5 * time.Second
|
|
// DigestLines: a digest names at most this many, then says how many more.
|
|
DigestLines = 15
|
|
)
|
|
|
|
// Message is what a channel shows: a title line and a body, and the answers it offers.
|
|
type Message struct {
|
|
Title string
|
|
Body string
|
|
Urgent bool
|
|
Quiet bool // a clearing: shown without urgency
|
|
Actions []Choice
|
|
}
|
|
|
|
// Choice is one answer a message offers: the token a channel hands back when it is chosen, and its label.
|
|
type Choice struct {
|
|
Token string `json:"token"`
|
|
Label string `json:"label"`
|
|
}
|
|
|
|
func (m Message) Text() string {
|
|
if m.Body == "" {
|
|
return m.Title
|
|
}
|
|
return m.Title + "\n" + m.Body
|
|
}
|
|
|
|
// Channel is one way to the operator.
|
|
type Channel interface {
|
|
Name() string
|
|
Ready() error
|
|
Send(Message) (string, error)
|
|
Edit(id string, m Message) error
|
|
CanEdit() bool
|
|
}
|
|
|
|
// silentEditor is a channel whose edits notify nobody (Telegram's are): what is over is said there
|
|
// even when it ended while the holder was away, since saying it costs the operator nothing.
|
|
type silentEditor interface{ SilentEdit() bool }
|
|
|
|
func editsSilently(ch Channel) bool {
|
|
s, ok := ch.(silentEditor)
|
|
return ok && s.SilentEdit()
|
|
}
|
|
|
|
// Record is one open message, kept in the module's own state so a restart forgets nothing.
|
|
type Record struct {
|
|
Key string `json:"key"`
|
|
Kind string `json:"kind"`
|
|
Subject string `json:"subject"`
|
|
Severity string `json:"severity"`
|
|
Summary string `json:"summary"`
|
|
// Headline, Explanation and Resolved are the plain words the operator reads (novox/hq ADR 0253):
|
|
// the controller's, or a module's through notify; empty, the message is said from the summary.
|
|
Headline string `json:"headline,omitempty"`
|
|
Explanation string `json:"explanation,omitempty"`
|
|
Resolved string `json:"resolved,omitempty"`
|
|
// Needs is what the operator does, empty for nothing; Actions the answers offered, each by a token
|
|
// (Tokens: token -> the action's index, or -1 for the details) a channel hands back when chosen.
|
|
Needs string `json:"needs,omitempty"`
|
|
Actions []Action `json:"actions,omitempty"`
|
|
// Kept as "grants": the build before ADR 0258 kept "tokens" as token -> index, which this form cannot
|
|
// read; under its own name it is ignored and the record loads (the old tokens die, as they must).
|
|
Tokens map[string]Grant `json:"grants,omitempty"`
|
|
// QuietReminded: a warning kept quiet as needing nothing was said once after QuietRemindAfter, so
|
|
// nothing stays silent for ever.
|
|
QuietReminded bool `json:"quiet_reminded,omitempty"`
|
|
// Answered is what the operator chose from a notification, and what came of it.
|
|
Answered string `json:"answered,omitempty"`
|
|
Origin string `json:"origin"`
|
|
More string `json:"more"`
|
|
Raised time.Time `json:"raised"`
|
|
SilencedTill time.Time `json:"silenced_till,omitempty"`
|
|
Sent map[string]string `json:"sent,omitempty"` // channel -> the first message's id; "" when said in a digest
|
|
FirstSent time.Time `json:"first_sent,omitempty"`
|
|
Reminded bool `json:"reminded,omitempty"`
|
|
Pending string `json:"pending,omitempty"` // what could not be said yet: raised, reminder, …
|
|
Folded bool `json:"folded,omitempty"` // held by the cap
|
|
Cleared time.Time `json:"cleared,omitempty"`
|
|
Count int `json:"count"`
|
|
Refused string `json:"refused,omitempty"`
|
|
// History: known from the state or from an old event, and never said — not news.
|
|
History bool `json:"history,omitempty"`
|
|
// Seen is when this holder last took word of it; the open set read after it is authoritative.
|
|
Seen time.Time `json:"seen,omitempty"`
|
|
}
|
|
|
|
// recordOf is a condition as this holder keeps it.
|
|
func recordOf(c Condition) Record {
|
|
return Record{Key: c.Key, Kind: c.Kind, Subject: c.SubjectWords(), Severity: c.Severity, Summary: c.Summary,
|
|
Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved, Needs: c.Needs, Actions: c.Actions,
|
|
Origin: "condition",
|
|
More: "conditions show " + c.Key, Raised: c.Raised, SilencedTill: c.SilencedTill}
|
|
}
|
|
|
|
// takeWords keeps the plain words current, as the summary is: a condition's words follow what is seen.
|
|
func (r *Record) takeWords(from Record) {
|
|
r.Headline, r.Explanation, r.Resolved, r.Needs = from.Headline, from.Explanation, from.Resolved, from.Needs
|
|
if !sameActions(r.Actions, from.Actions) {
|
|
r.Actions, r.Tokens = from.Actions, nil
|
|
}
|
|
}
|
|
|
|
func (r *Record) silenced(now time.Time) bool {
|
|
return !r.SilencedTill.IsZero() && now.Before(r.SilencedTill)
|
|
}
|
|
|
|
func (r *Record) told() bool { return len(r.Sent) > 0 }
|
|
|
|
// Sent is one message that went out, or was held, for the history.
|
|
type Sent struct {
|
|
At time.Time `json:"at"`
|
|
Channel string `json:"channel"`
|
|
Key string `json:"key"`
|
|
What string `json:"what"`
|
|
Outcome string `json:"outcome"` // sent, edited, held, in digest, failed: <why>
|
|
}
|
|
|
|
// RefusalNote is one refused message: never its text.
|
|
type RefusalNote struct {
|
|
At time.Time `json:"at"`
|
|
Key string `json:"key"`
|
|
Class string `json:"class"`
|
|
What string `json:"what"`
|
|
}
|
|
|
|
// Store is the module's own state (ADR 0201): the open messages, and the recent sends.
|
|
type Store interface {
|
|
Put(r Record) error
|
|
Delete(key string) error
|
|
All() ([]Record, error)
|
|
PutRecent([]Sent) error
|
|
Recent() ([]Sent, error)
|
|
}
|
|
|
|
// Holder is the seat's holder.
|
|
type Holder struct {
|
|
Telegram Channel
|
|
Desktop Channel
|
|
Store Store
|
|
Now func() time.Time
|
|
Logf func(string, ...any)
|
|
// Emit states a fact as this module (refused); nil states nothing.
|
|
Emit func(event string, body any) error
|
|
// Ask calls a seat verb through the mesh, as this module: the answers the operator chooses.
|
|
Ask Asker
|
|
// Zone is the operator's time zone, which every time in a message is said in; nil is this machine's.
|
|
Zone func() *time.Location
|
|
|
|
work sync.Mutex // one event, call or tick at a time
|
|
mu sync.Mutex // what the status reads
|
|
|
|
open map[string]*Record
|
|
recent []Sent
|
|
refusals []RefusalNote
|
|
outbox map[string][]item
|
|
lastSend map[string]time.Time
|
|
lastWarn map[string]time.Time
|
|
lastTry map[string]time.Time
|
|
capSaid map[string]time.Time
|
|
chanErr map[string]string
|
|
chanErrAt map[string]time.Time
|
|
chanOK map[string]time.Time
|
|
unreadable int
|
|
lastBad string
|
|
lastBadAt time.Time
|
|
saidOnce map[string]time.Time
|
|
storeErr string
|
|
heard map[string]int
|
|
lastHeard time.Time
|
|
started time.Time
|
|
syncedAt time.Time
|
|
syncErr string
|
|
wantSync bool
|
|
history int
|
|
stale int
|
|
// answered is what came of each answer event, by its id: a redelivered one is acted on once.
|
|
answered map[string]string
|
|
}
|
|
|
|
func (h *Holder) init() {
|
|
if h.open == nil {
|
|
h.open = map[string]*Record{}
|
|
h.outbox = map[string][]item{}
|
|
h.lastSend = map[string]time.Time{}
|
|
h.lastWarn = map[string]time.Time{}
|
|
h.lastTry = map[string]time.Time{}
|
|
h.capSaid = map[string]time.Time{}
|
|
h.chanErr = map[string]string{}
|
|
h.chanErrAt = map[string]time.Time{}
|
|
h.chanOK = map[string]time.Time{}
|
|
h.saidOnce = map[string]time.Time{}
|
|
h.heard = map[string]int{}
|
|
}
|
|
if h.Now == nil {
|
|
h.Now = time.Now
|
|
}
|
|
if h.Logf == nil {
|
|
h.Logf = func(string, ...any) {}
|
|
}
|
|
}
|
|
|
|
// Load reads back what was open and what was sent before a restart. What was waiting to be said is
|
|
// not said now: it is history, and the controller's state, read next, decides what is still news.
|
|
func (h *Holder) Load() error {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.init()
|
|
if h.Store == nil {
|
|
return nil
|
|
}
|
|
recs, err := h.Store.All()
|
|
if err != nil {
|
|
h.storeErr = err.Error()
|
|
return err
|
|
}
|
|
for i := range recs {
|
|
r := recs[i]
|
|
if r.Pending != "" || r.Folded {
|
|
r.Pending, r.Folded = "", false
|
|
if !r.told() {
|
|
r.History = true
|
|
}
|
|
}
|
|
h.open[r.Key] = &r
|
|
}
|
|
if sent, err := h.Store.Recent(); err == nil {
|
|
h.recent = sent
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Start marks the holder's start: the grace minute counts from here.
|
|
func (h *Holder) Start() {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.init()
|
|
h.started = h.Now()
|
|
}
|
|
|
|
func (h *Holder) inGrace(now time.Time) bool {
|
|
return !h.started.IsZero() && now.Sub(h.started) < StartGrace
|
|
}
|
|
|
|
// Condition takes one of the controller's condition events. Whether it is news is decided by its own
|
|
// time, never by its arrival.
|
|
func (h *Holder) Condition(event string, c Condition) {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
now := h.Now()
|
|
h.heard[event]++
|
|
h.lastHeard = now
|
|
synced := h.syncedAt
|
|
h.mu.Unlock()
|
|
rec := recordOf(c)
|
|
when := c.When(event)
|
|
quiet := false
|
|
switch {
|
|
case !when.IsZero() && !synced.IsZero() && when.Before(synced.Add(-ClockSlack)):
|
|
// Already in the state read from the controller after it: nothing to take.
|
|
h.mu.Lock()
|
|
h.history++
|
|
h.mu.Unlock()
|
|
return
|
|
case !when.IsZero() && now.Sub(when) > FreshFor:
|
|
// Old: a replay, a redelivery, or the holder was away. State, never a message — and the
|
|
// controller's state is read again, since what came between may be missing too.
|
|
quiet = true
|
|
h.mu.Lock()
|
|
h.stale++
|
|
first := !h.wantSync
|
|
h.wantSync = true
|
|
h.mu.Unlock()
|
|
if first {
|
|
h.Logf("[messenger] %s %s happened %s ago: taken as state, not news; reading the open conditions again",
|
|
event, c.Key, roughly(now.Sub(when)))
|
|
}
|
|
}
|
|
switch event {
|
|
case EventRaised:
|
|
h.raised(rec, quiet)
|
|
case EventChanged:
|
|
h.changed(rec, quiet)
|
|
case EventCleared:
|
|
h.cleared(rec.Key, quiet)
|
|
}
|
|
}
|
|
|
|
// Unreadable records an event that could not be read, and tells the operator — once an hour.
|
|
func (h *Holder) Unreadable(key string, err error) {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
h.unreadable++
|
|
h.lastBad = err.Error()
|
|
h.lastBadAt = h.Now()
|
|
n := h.unreadable
|
|
h.mu.Unlock()
|
|
h.Logf("[messenger] refused %s: %v (unreadable events since start: %d)", key, err, n)
|
|
h.sayOnce("messenger.unreadable-event", time.Hour, Message{
|
|
Title: "A mesh message could not be read",
|
|
Body: fmt.Sprintf("The messenger could not read %d update(s) from the controller, so a problem may be open "+
|
|
"that was not said. An agent can find out why in the messenger's status.", n),
|
|
})
|
|
}
|
|
|
|
func (h *Holder) raised(rec Record, quiet bool) {
|
|
now := h.Now()
|
|
h.mu.Lock()
|
|
old := h.open[rec.Key]
|
|
h.mu.Unlock()
|
|
if old != nil && old.Cleared.IsZero() {
|
|
// Said again while open: the same message. Its words are kept current; nothing is sent — unless
|
|
// it was kept quiet as needing nothing and now needs the operator.
|
|
h.mu.Lock()
|
|
wasQuiet := needsNothing(old)
|
|
old.Summary, old.Subject, old.Kind, old.SilencedTill = rec.Summary, rec.Subject, rec.Kind, rec.SilencedTill
|
|
old.takeWords(rec)
|
|
if rec.Severity != "" {
|
|
old.Severity = rec.Severity
|
|
}
|
|
old.Seen = now
|
|
h.mu.Unlock()
|
|
h.nowNeeded(old, wasQuiet, quiet)
|
|
h.persist(old)
|
|
return
|
|
}
|
|
if old != nil && now.Sub(old.Cleared) < ReopenWindow {
|
|
// Cleared and raised again within ten minutes: the same message, back to open (to-be 45 §2).
|
|
h.mu.Lock()
|
|
old.Cleared = time.Time{}
|
|
old.Count++
|
|
old.Summary, old.Severity, old.SilencedTill = rec.Summary, rec.Severity, rec.SilencedTill
|
|
old.takeWords(rec)
|
|
old.Seen = now
|
|
h.mu.Unlock()
|
|
if !quiet && !old.silenced(now) && old.told() {
|
|
h.edit(old, "reopened", false)
|
|
}
|
|
h.persist(old)
|
|
return
|
|
}
|
|
r := rec
|
|
r.Count = 1
|
|
if r.Raised.IsZero() {
|
|
r.Raised = now
|
|
}
|
|
r.Sent = map[string]string{}
|
|
r.Seen = now
|
|
r.History = quiet
|
|
h.mu.Lock()
|
|
h.open[r.Key] = &r
|
|
h.mu.Unlock()
|
|
switch {
|
|
case quiet:
|
|
case r.silenced(now):
|
|
h.Logf("[messenger] %s raised while silenced until %s: nothing sent", r.Key, r.SilencedTill.Format(time.RFC3339))
|
|
default:
|
|
h.deliver(&r, "raised")
|
|
}
|
|
h.persist(&r)
|
|
}
|
|
|
|
// nowNeeded says a record kept quiet as needing nothing, once its words say it needs the operator.
|
|
func (h *Holder) nowNeeded(r *Record, wasQuiet, quiet bool) {
|
|
if wasQuiet && !needsNothing(r) && !r.told() && !quiet && !r.silenced(h.Now()) && !h.queued(r.Key) {
|
|
h.deliver(r, "raised")
|
|
}
|
|
}
|
|
|
|
func (h *Holder) changed(rec Record, quiet bool) {
|
|
h.mu.Lock()
|
|
old := h.open[rec.Key]
|
|
h.mu.Unlock()
|
|
if old == nil || !old.Cleared.IsZero() {
|
|
// A change to something this holder never heard raised: read as a raise, so it is said
|
|
// (when it is news).
|
|
h.Logf("[messenger] %s changed and was not open here; taken as raised", rec.Key)
|
|
h.raised(rec, quiet)
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
escalated := old.Severity == Warning && rec.Severity == Urgent
|
|
wasQuiet := needsNothing(old)
|
|
old.Summary, old.Subject, old.SilencedTill = rec.Summary, rec.Subject, rec.SilencedTill
|
|
old.takeWords(rec)
|
|
if rec.Severity != "" {
|
|
old.Severity = rec.Severity
|
|
}
|
|
old.Seen = h.Now()
|
|
h.mu.Unlock()
|
|
switch {
|
|
case escalated && !quiet && !old.silenced(h.Now()):
|
|
// Routing differs for urgent: said once more, to both channels.
|
|
h.deliver(old, "escalated")
|
|
default:
|
|
h.nowNeeded(old, wasQuiet, quiet)
|
|
}
|
|
h.persist(old)
|
|
}
|
|
|
|
// cleared ends a record. Something never said is ended without a word; what was waiting to be said
|
|
// about it is dropped. quiet: it ended while this holder was not listening, so it is said only where
|
|
// an edit notifies nobody.
|
|
func (h *Holder) cleared(key string, quiet bool) {
|
|
now := h.Now()
|
|
h.mu.Lock()
|
|
old := h.open[key]
|
|
h.mu.Unlock()
|
|
if old == nil || !old.Cleared.IsZero() {
|
|
h.Logf("[messenger] %s cleared and was not open here: nothing to say", key)
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
old.Cleared = now
|
|
old.Pending = ""
|
|
old.Folded = false
|
|
old.Seen = now
|
|
told := old.told()
|
|
h.mu.Unlock()
|
|
dropped := h.unqueue(key)
|
|
switch {
|
|
case old.silenced(now):
|
|
// A silenced condition sends nothing, its clearing included.
|
|
case !told:
|
|
if dropped > 0 {
|
|
h.Logf("[messenger] %s cleared before it was said: nothing to unsay", key)
|
|
}
|
|
default:
|
|
h.edit(old, "cleared", quiet)
|
|
}
|
|
h.persist(old)
|
|
}
|
|
|
|
// Tick does what time asks: reminders, retries, what is held, and forgetting what cleared long
|
|
// enough ago that a new raise is a new message.
|
|
func (h *Holder) Tick() {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
now := h.Now()
|
|
var recs []*Record
|
|
for _, r := range h.open {
|
|
recs = append(recs, r)
|
|
}
|
|
h.mu.Unlock()
|
|
sort.Slice(recs, func(i, j int) bool { return recs[i].Raised.Before(recs[j].Raised) })
|
|
for _, r := range recs {
|
|
switch {
|
|
case !r.Cleared.IsZero():
|
|
if now.Sub(r.Cleared) >= ReopenWindow {
|
|
h.mu.Lock()
|
|
delete(h.open, r.Key)
|
|
h.mu.Unlock()
|
|
if h.Store != nil {
|
|
if err := h.Store.Delete(r.Key); err != nil {
|
|
h.noteStore(err)
|
|
}
|
|
}
|
|
}
|
|
case r.silenced(now):
|
|
case r.Pending != "" && !h.queued(r.Key):
|
|
// No channel could take it when it was said: tried again.
|
|
h.deliver(r, r.Pending)
|
|
h.persist(r)
|
|
case needsNothing(r) && !r.told() && !r.QuietReminded && now.Sub(r.Raised) >= QuietRemindAfter:
|
|
// Quiet for a day: said once, so nothing stays silent for ever (ADR 0258).
|
|
h.mu.Lock()
|
|
r.QuietReminded = true
|
|
h.mu.Unlock()
|
|
h.deliver(r, "quiet-reminder")
|
|
h.persist(r)
|
|
case !r.Reminded && !r.FirstSent.IsZero() && now.Sub(r.Raised) >= remindAfter(r.Severity) &&
|
|
now.Sub(r.FirstSent) >= remindAfter(r.Severity):
|
|
// Once more after its bound — counted from its raising, and never sooner after it was
|
|
// first said (an old condition said at a reading of the state is not reminded at once).
|
|
h.mu.Lock()
|
|
r.Reminded = true
|
|
h.mu.Unlock()
|
|
h.deliver(r, "reminder")
|
|
h.persist(r)
|
|
}
|
|
}
|
|
h.flush(now, false)
|
|
}
|
|
|
|
// Flush sends what is due on each channel; the runtime calls it every few seconds, so a held
|
|
// message waits its window and not a minute more.
|
|
func (h *Holder) Flush() {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
now := h.Now()
|
|
h.mu.Unlock()
|
|
h.flush(now, false)
|
|
}
|
|
|
|
func remindAfter(severity string) time.Duration {
|
|
if severity == Urgent {
|
|
return RemindUrgent
|
|
}
|
|
return RemindWarning
|
|
}
|
|
|
|
// say composes a record's message under the content rule: refused, it is composed again with less of
|
|
// it, until what remains may leave. The refusal is recorded and stated once per record.
|
|
func (h *Holder) say(r *Record, what string) Message {
|
|
now := h.Now()
|
|
if r.Refused != "" {
|
|
// Refused before: its words stay withheld in every later message too.
|
|
for w := 1; w <= 3; w++ {
|
|
m := compose(r, what, now, h.zone(), w)
|
|
if _, ok := Check(m.Text()); ok {
|
|
return m
|
|
}
|
|
}
|
|
}
|
|
m := compose(r, what, now, h.zone(), 0)
|
|
refusal, ok := Check(m.Text())
|
|
if ok {
|
|
if what != "cleared" {
|
|
m.Actions = h.choices(r)
|
|
}
|
|
return m
|
|
}
|
|
h.mu.Lock()
|
|
r.Refused = refusal.What
|
|
h.mu.Unlock()
|
|
key := r.Key
|
|
if _, keyOK := Check(key); !keyOK {
|
|
key = "(withheld)"
|
|
}
|
|
h.mu.Lock()
|
|
h.refusals = append(h.refusals, RefusalNote{At: now, Key: key, Class: refusal.Class, What: refusal.What})
|
|
if len(h.refusals) > KeptRefusals {
|
|
h.refusals = h.refusals[len(h.refusals)-KeptRefusals:]
|
|
}
|
|
h.mu.Unlock()
|
|
h.Logf("[messenger] refused the message for %s: it carried %s; sending channel-refused with its words withheld", key, refusal)
|
|
if h.Emit != nil {
|
|
if err := h.Emit("refused", map[string]any{"key": key, "class": refusal.Class, "what": refusal.What}); err != nil {
|
|
h.Logf("[messenger] could not state the refusal on the bus: %v", err)
|
|
}
|
|
}
|
|
for w := 1; w <= 3; w++ {
|
|
m := compose(r, what, now, h.zone(), w)
|
|
if _, ok := Check(m.Text()); ok {
|
|
return m
|
|
}
|
|
}
|
|
return withheld(refusal.What, r.Severity == Urgent)
|
|
}
|
|
|
|
// deliver sends a record's message where its severity routes it.
|
|
|
|
func (h *Holder) channels() []Channel {
|
|
var out []Channel
|
|
for _, c := range []Channel{h.Telegram, h.Desktop} {
|
|
if c != nil {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (h *Holder) record(s Sent) {
|
|
h.mu.Lock()
|
|
h.recent = append(h.recent, s)
|
|
if len(h.recent) > KeptSends {
|
|
h.recent = h.recent[len(h.recent)-KeptSends:]
|
|
}
|
|
recent := append([]Sent(nil), h.recent...)
|
|
h.mu.Unlock()
|
|
if h.Store != nil {
|
|
if err := h.Store.PutRecent(recent); err != nil {
|
|
h.noteStore(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *Holder) persist(r *Record) {
|
|
if h.Store == nil {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
c := *r
|
|
h.mu.Unlock()
|
|
if err := h.Store.Put(c); err != nil {
|
|
h.noteStore(err)
|
|
}
|
|
}
|
|
|
|
func (h *Holder) noteStore(err error) {
|
|
h.mu.Lock()
|
|
first := h.storeErr == ""
|
|
h.storeErr = err.Error()
|
|
h.mu.Unlock()
|
|
if first {
|
|
h.Logf("[messenger] cannot write its state (open messages are held in memory only until it can): %v", err)
|
|
}
|
|
}
|
|
|
|
func (h *Holder) noteChannel(name string, err error) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
now := h.Now()
|
|
if err == nil {
|
|
if h.chanErr[name] != "" {
|
|
h.Logf("[messenger] %s sends again", name)
|
|
}
|
|
h.chanErr[name] = ""
|
|
h.chanOK[name] = now
|
|
return
|
|
}
|
|
// Said in the log when it changes, and at most every ten minutes while it holds.
|
|
if h.chanErr[name] != err.Error() || now.Sub(h.chanErrAt[name]) >= 10*time.Minute {
|
|
h.Logf("[messenger] %s cannot send: %v", name, err)
|
|
h.chanErrAt[name] = now
|
|
}
|
|
h.chanErr[name] = err.Error()
|
|
}
|
|
|
|
func (h *Holder) whyNot() string {
|
|
var parts []string
|
|
for _, ch := range h.channels() {
|
|
if err := ch.Ready(); err != nil {
|
|
parts = append(parts, ch.Name()+": "+err.Error())
|
|
continue
|
|
}
|
|
h.mu.Lock()
|
|
e := h.chanErr[ch.Name()]
|
|
h.mu.Unlock()
|
|
if e != "" {
|
|
parts = append(parts, ch.Name()+": "+e)
|
|
}
|
|
}
|
|
if len(parts) == 0 {
|
|
return "no channel"
|
|
}
|
|
return strings.Join(parts, "; ")
|
|
}
|
|
|
|
func roughly(d time.Duration) string {
|
|
switch {
|
|
case d < 0:
|
|
return "a moment"
|
|
case d < 2*time.Minute:
|
|
return fmt.Sprintf("%d s", int(d.Seconds()))
|
|
case d < 2*time.Hour:
|
|
return fmt.Sprintf("%d min", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%.1f h", d.Hours())
|
|
}
|
|
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
|
}
|