Review round 1 of mesh-catalog found sentLastHour counting tells against CapPerHour (ADR 0380 §8 says tells keep their own cap, never shared), and the drop log firing on every drop instead of once an hour per channel as the comment already said.
1203 lines
44 KiB
Go
1203 lines
44 KiB
Go
package main
|
|
|
|
// The holder of the operator-channel seat (novox/hq to-be 45 §5, ADR 0227): what is sent, to whom,
|
|
// when, and how often. The controller decides what is wrong; this decides what is said.
|
|
//
|
|
// **History is never news** (novox/hq issue 271). The events stream keeps what it was given and hands
|
|
// all of it to a consumer that is new, recreated or was away; on 2026-10-06 a newly made consumer
|
|
// replayed three hours of raised-and-cleared conditions and each was said as if it had just happened.
|
|
// So what is said is decided by the state now and by when a thing happened, never by when it arrived:
|
|
//
|
|
// - On start, and whenever it may have missed something (every ten minutes, and at once when an
|
|
// old event arrives), the holder reads what is open now from the controller (`conditions`) and
|
|
// takes it as state: nothing it says is sent for what it already said, what cleared meanwhile is
|
|
// forgotten without a word (a Telegram message is edited, which notifies nobody), an urgent
|
|
// condition open and never said is said — all of them in one message — and a warning raised in
|
|
// the last ten minutes is said like any other; an older warning is state.
|
|
// - An event older than that reading is already in it, and is not acted on. An event older than
|
|
// FreshFor (ten minutes) by its own time is state only: it is recorded, never said.
|
|
// - A clearing of something never said says nothing; a held message whose condition cleared before
|
|
// it went out is dropped.
|
|
// - Nothing is sent in the first minute after start (StartGrace) but the one message of urgent
|
|
// conditions open in the controller's state; what arrives meanwhile waits for the minute's end.
|
|
//
|
|
// What is said, and how often:
|
|
//
|
|
// - On `condition-raised`: one message, deduplicated by the condition's key. Said again with the
|
|
// same key while open, it is the same message, not a second.
|
|
// - Once more if still open after 1 hour (urgent) or 12 hours (warning), counted from when it was last
|
|
// said; a reminder or an escalation replaces its message, never stands beside it (issue 373).
|
|
// - On `condition-cleared`: once it stays cleared ClearSettle, its message is edited where the channel can
|
|
// (both can). Raised again before that, nothing is said: to the operator it never ended. Cleared, said
|
|
// so, and raised again within ten minutes, it is the same message, edited back to open.
|
|
// - A silenced condition sends nothing.
|
|
// - Urgent to both channels; warning to the desktop when the operator's session is there,
|
|
// otherwise to Telegram.
|
|
// - **Bursts are one message.** Messages for a channel are held for BurstWindow (30 s) from the
|
|
// first and go out together: one alone is sent as itself, several as one digest ("5 new
|
|
// warnings: …"). An urgent message is sent at once when nothing went out on its channel in the
|
|
// last 30 s.
|
|
// - **The desktop is gentle:** warnings reach it as at most one message every DesktopWarningEvery
|
|
// (15 min); urgent ones are not held to that.
|
|
// - At most twenty messages an hour per channel — a last line of defence, since the above keeps a
|
|
// channel far below it. Reached, it is said once in a message of its own and the rest is held,
|
|
// to go out as one digest when the hour allows.
|
|
// - A message carrying an address, a path or a secret is refused (content.go); what is sent in its
|
|
// place says `channel-refused`, with the offending part withheld.
|
|
// - A channel that cannot send says so in the status and the log, and is tried again every
|
|
// minute while what it holds is still open.
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
RemindUrgent = time.Hour
|
|
RemindWarning = 12 * time.Hour
|
|
// QuietRemindAfter: a warning kept quiet as needing nothing is said once after this.
|
|
QuietRemindAfter = 24 * time.Hour
|
|
ReopenWindow = 10 * time.Minute
|
|
// ClearSettle: a clearing is said once the condition stayed cleared this long. One raised again sooner
|
|
// (a unit failing again after each apply, issue 373) continues its message without a word, and its
|
|
// clearing is never said. Once it has come back, its next clearing waits the reopen window less a minute.
|
|
ClearSettle = 2 * time.Minute
|
|
CapPerHour = 20
|
|
KeptSends = 200
|
|
KeptRefusals = 50
|
|
|
|
// FreshFor: an event older than this by its own time is history — state, never a message.
|
|
FreshFor = 10 * time.Minute
|
|
// BurstWindow: what arrives for a channel within it goes out as one message.
|
|
BurstWindow = 30 * time.Second
|
|
// DesktopWarningEvery: warnings reach the desktop at most this often.
|
|
DesktopWarningEvery = 15 * time.Minute
|
|
// StartGrace: after start, nothing but the urgent conditions open in the controller's state.
|
|
StartGrace = time.Minute
|
|
// ResyncEvery: how often the open set is read from the controller again.
|
|
ResyncEvery = 10 * time.Minute
|
|
// RetryEvery: a channel that failed is tried again after this.
|
|
RetryEvery = time.Minute
|
|
// ClockSlack: the controller's clock and this one may differ by this much; an event this close
|
|
// to the reading is taken rather than assumed to be in it (taking one twice is harmless).
|
|
ClockSlack = 5 * time.Second
|
|
// DigestLines: a digest names at most this many, then says how many more.
|
|
DigestLines = 15
|
|
)
|
|
|
|
// Message is what a channel shows: a title line and a body, and the answers it offers.
|
|
type Message struct {
|
|
Title string
|
|
Body string
|
|
// Note is the secondary words, after the body and the answers: who asked, what each answer does, the
|
|
// expiry, the asker's Details (asks.Message.Note, issue 639).
|
|
Note string
|
|
Urgent bool
|
|
Quiet bool // a clearing: shown without urgency
|
|
Actions []Choice
|
|
// Reply is the answer this message answers, by its id: the channel shows it where that was given.
|
|
Reply string
|
|
// To is the account it is for, when not the one linked: a notice to an account being unlinked, or an
|
|
// ask put to the account a relink replaces.
|
|
To string
|
|
// Secret: shown once, and kept by no channel (a link's code).
|
|
Secret bool
|
|
}
|
|
|
|
// Choice is one answer a message offers: the token a channel hands back when it is chosen, and its label.
|
|
type Choice struct {
|
|
Token string `json:"token"`
|
|
Label string `json:"label"`
|
|
}
|
|
|
|
func (m Message) Text() string {
|
|
t := m.Title
|
|
for _, more := range []string{m.Body, m.Note} {
|
|
if more != "" {
|
|
t += "\n" + more
|
|
}
|
|
}
|
|
return t
|
|
}
|
|
|
|
// Channel is one way to the operator.
|
|
type Channel interface {
|
|
Name() string
|
|
Ready() error
|
|
Send(Message) (string, error)
|
|
Edit(id string, m Message) error
|
|
CanEdit() bool
|
|
}
|
|
|
|
// replacer is a channel that can say a message again in place of the one it showed under that id: the earlier
|
|
// one goes, the new one is shown as a new message is — it rings as its kind rings (issue 373). A reminder or
|
|
// an escalation is then one message per problem, never one more beside it.
|
|
type replacer interface {
|
|
Replace(id string, m Message) error
|
|
}
|
|
|
|
func replaces(ch Channel) bool {
|
|
_, ok := ch.(replacer)
|
|
return ok
|
|
}
|
|
|
|
// silentEditor is a channel whose edits notify nobody (Telegram's are): what is over is said there
|
|
// even when it ended while the holder was away, since saying it costs the operator nothing.
|
|
type silentEditor interface{ SilentEdit() bool }
|
|
|
|
func editsSilently(ch Channel) bool {
|
|
s, ok := ch.(silentEditor)
|
|
return ok && s.SilentEdit()
|
|
}
|
|
|
|
// Record is one open message, kept in the module's own state so a restart forgets nothing.
|
|
type Record struct {
|
|
Key string `json:"key"`
|
|
Kind string `json:"kind"`
|
|
Subject string `json:"subject"`
|
|
Severity string `json:"severity"`
|
|
Summary string `json:"summary"`
|
|
// Scope, SubjectID and Machine are the condition's subject as the controller gave it, which a walk's touches
|
|
// are matched against (ADR 0308 §2); empty for a record kept before, whose key names them.
|
|
Scope string `json:"scope,omitempty"`
|
|
SubjectID string `json:"subject_id,omitempty"`
|
|
Machine string `json:"machine,omitempty"`
|
|
// Headline, Explanation and Resolved are the plain words the operator reads (novox/hq ADR 0253):
|
|
// the controller's, or a module's through notify; empty, the message is said from the summary.
|
|
Headline string `json:"headline,omitempty"`
|
|
Explanation string `json:"explanation,omitempty"`
|
|
Resolved string `json:"resolved,omitempty"`
|
|
// Needs is what the operator does, empty for nothing; Actions the answers offered, each by a token
|
|
// (Tokens: token -> its grant; Index -1 is Details, one per message shown) a channel hands back when chosen.
|
|
Needs string `json:"needs,omitempty"`
|
|
Actions []Action `json:"actions,omitempty"`
|
|
// Kept as "grants": the build before ADR 0258 kept "tokens" as token -> index, which this form cannot
|
|
// read; under its own name it is ignored and the record loads (the old tokens die, as they must).
|
|
Tokens map[string]Grant `json:"grants,omitempty"`
|
|
// QuietReminded: a warning kept quiet as needing nothing was said once after QuietRemindAfter, so
|
|
// nothing stays silent for ever.
|
|
QuietReminded bool `json:"quiet_reminded,omitempty"`
|
|
// Answered is what the operator chose from a notification, and what came of it.
|
|
Answered string `json:"answered,omitempty"`
|
|
// Ask is the open ask about this record, `<asker>.<id>`: its options are this record's answers
|
|
// (novox/hq ADR 0259).
|
|
Ask string `json:"ask,omitempty"`
|
|
Origin string `json:"origin"`
|
|
More string `json:"more"`
|
|
Raised time.Time `json:"raised"`
|
|
SilencedTill time.Time `json:"silenced_till,omitempty"`
|
|
// SilencedWhy is the reason the silence was given with; SilenceSaid: the message shown was edited to say
|
|
// it is silenced (issue 373: a critical popup silenced for a week stood on the desk as if still open).
|
|
SilencedWhy string `json:"silenced_why,omitempty"`
|
|
SilenceSaid bool `json:"silence_said,omitempty"`
|
|
Sent map[string]string `json:"sent,omitempty"` // channel -> its message's id; "" when said in a digest
|
|
FirstSent time.Time `json:"first_sent,omitempty"`
|
|
// LastSaid is when a message about it last went out: a reminder is due its bound after it, never sooner
|
|
// (an escalation followed by a reminder 31 s later, issue 373).
|
|
LastSaid time.Time `json:"last_said,omitempty"`
|
|
// ClearUnsaid: cleared, and the clearing not said yet — it is said once it stays cleared ClearSettle;
|
|
// raised again before, the message continues as if it never cleared. ClearSilent: said only where an edit
|
|
// notifies nobody (it ended while this holder was away).
|
|
ClearUnsaid bool `json:"clear_unsaid,omitempty"`
|
|
ClearSilent bool `json:"clear_silent,omitempty"`
|
|
// ClearHeld is what was waiting to be said of it when it cleared (a reminder, an escalation): said after
|
|
// all if it comes back before its clearing was said. Blips counts those comebacks, never said, so a
|
|
// condition that flaps is known as one and its message's "again, Nth time" counts only what was said.
|
|
ClearHeld string `json:"clear_held,omitempty"`
|
|
Blips int `json:"blips,omitempty"`
|
|
Reminded bool `json:"reminded,omitempty"`
|
|
Pending string `json:"pending,omitempty"` // what could not be said yet: raised, reminder, …
|
|
// Owed is what a walk held of it and has not told yet (ADR 0308 §4, §5): kept in the state until a message
|
|
// carrying it was sent, so a router that restarts before says it on its own (issue 538), never history.
|
|
Owed string `json:"owed,omitempty"`
|
|
Folded bool `json:"folded,omitempty"` // held by the cap
|
|
Cleared time.Time `json:"cleared,omitempty"`
|
|
Count int `json:"count"`
|
|
Refused string `json:"refused,omitempty"`
|
|
// History: known from the state or from an old event, and never said — not news.
|
|
History bool `json:"history,omitempty"`
|
|
// Seen is when this holder last took word of it; the open set read after it is authoritative.
|
|
Seen time.Time `json:"seen,omitempty"`
|
|
}
|
|
|
|
// recordOf is a condition as this holder keeps it.
|
|
func recordOf(c Condition) Record {
|
|
return Record{Key: c.Key, Kind: c.Kind, Subject: c.SubjectWords(), Scope: c.Scope, SubjectID: c.ID, Machine: c.Machine,
|
|
Severity: c.Severity, Summary: c.Summary,
|
|
Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved, Needs: c.Needs, Actions: c.Actions,
|
|
Origin: "condition",
|
|
More: "conditions show " + c.Key, Raised: c.Raised, SilencedTill: c.SilencedTill, SilencedWhy: c.SilencedWhy}
|
|
}
|
|
|
|
// takeWords keeps the plain words current, as the summary is: a condition's words follow what is seen.
|
|
func (r *Record) takeWords(from Record) {
|
|
r.Headline, r.Explanation, r.Resolved, r.Needs = from.Headline, from.Explanation, from.Resolved, from.Needs
|
|
if !sameActions(r.Actions, from.Actions) {
|
|
// Details performs nothing and names no action, so its tickets outlive a change of the actions:
|
|
// every message still shown keeps its Details (issue 371). Any other ticket dies with its action.
|
|
r.Actions = from.Actions
|
|
for t, g := range r.Tokens {
|
|
if g.Index != -1 {
|
|
delete(r.Tokens, t)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// takeSubject keeps the subject as the controller says it now, when it says one.
|
|
func (r *Record) takeSubject(from Record) {
|
|
if from.Scope != "" || from.SubjectID != "" || from.Machine != "" {
|
|
r.Scope, r.SubjectID, r.Machine = from.Scope, from.SubjectID, from.Machine
|
|
}
|
|
}
|
|
|
|
func (r *Record) silenced(now time.Time) bool {
|
|
return !r.SilencedTill.IsZero() && now.Before(r.SilencedTill)
|
|
}
|
|
|
|
func (r *Record) told() bool { return len(r.Sent) > 0 }
|
|
|
|
// Sent is one message that went out, or was held, for the history.
|
|
type Sent struct {
|
|
At time.Time `json:"at"`
|
|
Channel string `json:"channel"`
|
|
Key string `json:"key"`
|
|
What string `json:"what"`
|
|
Outcome string `json:"outcome"` // sent, edited, held, in digest, failed: <why>
|
|
}
|
|
|
|
// RefusalNote is one refused message: never its text.
|
|
type RefusalNote struct {
|
|
At time.Time `json:"at"`
|
|
Key string `json:"key"`
|
|
Class string `json:"class"`
|
|
What string `json:"what"`
|
|
}
|
|
|
|
// Store is the module's own state (ADR 0201): the open messages, and the recent sends.
|
|
type Store interface {
|
|
Put(r Record) error
|
|
Delete(key string) error
|
|
All() ([]Record, error)
|
|
PutRecent([]Sent) error
|
|
Recent() ([]Sent, error)
|
|
}
|
|
|
|
// Holder is the seat's holder: the router (novox/hq ADR 0234 §5, ADR 0259). It keeps no channel's client:
|
|
// Away is the channel that reaches the operator away from the machines (a kind promising reaches-away),
|
|
// Desk the one at the machines; each is a kind of the `channel` bench, held by a module of its own.
|
|
type Holder struct {
|
|
Away Channel
|
|
Desk Channel
|
|
Store Store
|
|
// Asks keeps the asks, the operator's identities and their links (asks.go, link.go); Bench is what
|
|
// the controller says of every channel kind and what each says of itself; Publish says on a seat.
|
|
Asks AskStore
|
|
Bench *Bench
|
|
Publish func(subject string, body any, id string) error
|
|
// Node is the machine this router runs on; StateDir its own directory, where root leaves a break-glass
|
|
// request (selfact.go).
|
|
Node string
|
|
StateDir string
|
|
Now func() time.Time
|
|
Logf func(string, ...any)
|
|
// Emit states a fact as this module (refused); nil states nothing.
|
|
Emit func(event string, body any) error
|
|
// Ask calls a seat verb through the mesh, as this module: the answers the operator chooses.
|
|
Ask Asker
|
|
// RootFree asks the controller's verb root-free about machines; nil asks it through Ask (askRootFree).
|
|
RootFree func(machines []string) (map[string]RootVerdict, error)
|
|
// Zone is the operator's time zone, which every time in a message is said in; nil is this machine's.
|
|
Zone func() *time.Location
|
|
// Tracking asks the issue tracker which issues track which conditions; nil asks it through Ask (tracking.go).
|
|
Tracking func() (map[string][]TrackedIssue, error)
|
|
// NeedsOnly says whether a kind is set to say only what needs the operator (the setting
|
|
// `only-what-needs-me`, ADR 0308 §4); nil, none is.
|
|
NeedsOnly func(kind string) bool
|
|
|
|
work sync.Mutex // one event, call or tick at a time
|
|
mu sync.Mutex // what the status reads
|
|
|
|
open map[string]*Record
|
|
recent []Sent
|
|
refusals []RefusalNote
|
|
outbox map[string][]item
|
|
lastSend map[string]time.Time
|
|
lastWarn map[string]time.Time
|
|
lastTry map[string]time.Time
|
|
capSaid map[string]time.Time
|
|
tellCapSaid map[string]time.Time
|
|
chanErr map[string]string
|
|
chanErrAt map[string]time.Time
|
|
chanOK map[string]time.Time
|
|
unreadable int
|
|
lastBad string
|
|
lastBadAt time.Time
|
|
saidOnce map[string]time.Time
|
|
storeErr string
|
|
heard map[string]int
|
|
lastHeard time.Time
|
|
started time.Time
|
|
syncedAt time.Time
|
|
syncErr string
|
|
wantSync bool
|
|
history int
|
|
stale int
|
|
// tellSeq makes every tell's key its own, even when two calls land in the same instant.
|
|
tellSeq int
|
|
// answered is what came of each answer event, by its id: a redelivered one is acted on once.
|
|
answered map[string]string
|
|
// asks are the asks this router holds, by `<asker>.<id>`, and tickets each ask's tickets to it.
|
|
asks map[string]*askEntry
|
|
tickets map[string]string
|
|
// lastUnlink is the last unlink of each kind.
|
|
lastUnlink map[string]Unlinked
|
|
// rootWord is the controller's last word on whether an agent can become root, by the machines asked.
|
|
rootWord map[string]rootWord
|
|
// passes is, by kind, the restart pass this router last made in full, kept in its state (restart.go); lost
|
|
// the loss of answers it has yet to act on, by kind.
|
|
passes map[string]Pass
|
|
lost map[string]time.Time
|
|
// walks are the walks open now, by id, each with what it holds (walks.go, ADR 0308).
|
|
walks map[string]*Walk
|
|
// ended are the walks whose end was heard, by id: what a start heard after its end is matched against.
|
|
ended map[string]*Walk
|
|
// track is the issue tracker's last word on which issues track which conditions (tracking.go).
|
|
track trackWord
|
|
}
|
|
|
|
func (h *Holder) init() {
|
|
if h.open == nil {
|
|
h.open = map[string]*Record{}
|
|
h.outbox = map[string][]item{}
|
|
h.lastSend = map[string]time.Time{}
|
|
h.lastWarn = map[string]time.Time{}
|
|
h.lastTry = map[string]time.Time{}
|
|
h.capSaid = map[string]time.Time{}
|
|
h.tellCapSaid = map[string]time.Time{}
|
|
h.chanErr = map[string]string{}
|
|
h.chanErrAt = map[string]time.Time{}
|
|
h.chanOK = map[string]time.Time{}
|
|
h.saidOnce = map[string]time.Time{}
|
|
h.heard = map[string]int{}
|
|
h.passes = map[string]Pass{}
|
|
h.lost = map[string]time.Time{}
|
|
h.walks = map[string]*Walk{}
|
|
h.ended = map[string]*Walk{}
|
|
}
|
|
if h.Now == nil {
|
|
h.Now = time.Now
|
|
}
|
|
if h.Logf == nil {
|
|
h.Logf = func(string, ...any) {}
|
|
}
|
|
}
|
|
|
|
// Load reads back what was open and what was sent before a restart. What was waiting to be said is
|
|
// not said now: it is history, and the controller's state, read next, decides what is still news.
|
|
func (h *Holder) Load() error {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.init()
|
|
if h.Store == nil {
|
|
return nil
|
|
}
|
|
recs, err := h.Store.All()
|
|
if err != nil {
|
|
h.storeErr = err.Error()
|
|
return err
|
|
}
|
|
for i := range recs {
|
|
r := recs[i]
|
|
if r.Pending != "" || r.Folded {
|
|
r.Pending, r.Folded = "", false
|
|
if !r.told() {
|
|
r.History = true
|
|
}
|
|
}
|
|
if r.Owed != "" && r.Cleared.IsZero() {
|
|
// Held from the operator on purpose and not told yet: said on its own, as the next tick tries what is
|
|
// unsent (issue 538). What the operator could have heard is history; this they could not.
|
|
r.Pending, r.History = r.Owed, false
|
|
}
|
|
h.open[r.Key] = &r
|
|
}
|
|
if sent, err := h.Store.Recent(); err == nil {
|
|
h.recent = sent
|
|
}
|
|
if err := h.loadWalks(); err != nil {
|
|
h.storeErr = err.Error()
|
|
return err
|
|
}
|
|
h.mu.Unlock()
|
|
err = h.loadAsks()
|
|
h.mu.Lock()
|
|
return err
|
|
}
|
|
|
|
// Start marks the holder's start: the grace minute counts from here.
|
|
func (h *Holder) Start() {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.init()
|
|
h.started = h.Now()
|
|
}
|
|
|
|
func (h *Holder) inGrace(now time.Time) bool {
|
|
return !h.started.IsZero() && now.Sub(h.started) < StartGrace
|
|
}
|
|
|
|
// Condition takes one of the controller's condition events. Whether it is news is decided by its own
|
|
// time, never by its arrival.
|
|
func (h *Holder) Condition(event string, c Condition) {
|
|
h.warmTracking() // outside the work: what is said inside it reads a recent word
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
now := h.Now()
|
|
h.heard[event]++
|
|
h.lastHeard = now
|
|
synced := h.syncedAt
|
|
h.mu.Unlock()
|
|
rec := recordOf(c)
|
|
when := c.When(event)
|
|
quiet := false
|
|
switch {
|
|
case !when.IsZero() && !synced.IsZero() && when.Before(synced.Add(-ClockSlack)):
|
|
// Already in the state read from the controller after it: nothing to take.
|
|
h.mu.Lock()
|
|
h.history++
|
|
h.mu.Unlock()
|
|
return
|
|
case !when.IsZero() && now.Sub(when) > FreshFor:
|
|
// Old: a replay, a redelivery, or the holder was away. State, never a message — and the
|
|
// controller's state is read again, since what came between may be missing too.
|
|
quiet = true
|
|
h.mu.Lock()
|
|
h.stale++
|
|
first := !h.wantSync
|
|
h.wantSync = true
|
|
h.mu.Unlock()
|
|
if first {
|
|
h.Logf("[messenger] %s %s happened %s ago: taken as state, not news; reading the open conditions again",
|
|
event, c.Key, roughly(now.Sub(when)))
|
|
}
|
|
}
|
|
switch event {
|
|
case EventRaised:
|
|
h.raised(rec, quiet)
|
|
case EventChanged:
|
|
h.changed(rec, quiet)
|
|
case EventCleared:
|
|
h.cleared(rec.Key, quiet)
|
|
}
|
|
}
|
|
|
|
// Unreadable records an event that could not be read, and tells the operator — once an hour.
|
|
func (h *Holder) Unreadable(key string, err error) {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
h.unreadable++
|
|
h.lastBad = err.Error()
|
|
h.lastBadAt = h.Now()
|
|
n := h.unreadable
|
|
h.mu.Unlock()
|
|
h.Logf("[messenger] refused %s: %v (unreadable events since start: %d)", key, err, n)
|
|
h.sayOnce("messenger.unreadable-event", time.Hour, Message{
|
|
Title: "A mesh message could not be read",
|
|
Body: fmt.Sprintf("The messenger could not read %d update(s) from the controller, so a problem may be open "+
|
|
"that was not said. An agent can find out why in the messenger's status.", n),
|
|
})
|
|
}
|
|
|
|
func (h *Holder) raised(rec Record, quiet bool) {
|
|
now := h.Now()
|
|
h.mu.Lock()
|
|
old := h.open[rec.Key]
|
|
h.mu.Unlock()
|
|
if old != nil && old.Cleared.IsZero() {
|
|
// Said again while open: the same message. Its words are kept current; nothing is sent — unless
|
|
// it was kept quiet as needing nothing and now needs the operator.
|
|
h.mu.Lock()
|
|
wasQuiet := needsNothing(old)
|
|
wasSilenced := old.silenced(now)
|
|
old.Summary, old.Subject, old.Kind = rec.Summary, rec.Subject, rec.Kind
|
|
old.takeSubject(rec)
|
|
old.SilencedTill, old.SilencedWhy = rec.SilencedTill, rec.SilencedWhy
|
|
old.takeWords(rec)
|
|
if rec.Severity != "" {
|
|
old.Severity = rec.Severity
|
|
}
|
|
old.Seen = now
|
|
h.mu.Unlock()
|
|
h.silenceWords(old, wasSilenced)
|
|
h.nowNeeded(old, wasQuiet, quiet)
|
|
h.persist(old)
|
|
return
|
|
}
|
|
if old != nil && now.Sub(old.Cleared) < ReopenWindow {
|
|
// Cleared and raised again within ten minutes: the same message, back to open (to-be 45 §2).
|
|
h.mu.Lock()
|
|
blip := old.ClearUnsaid
|
|
held := old.ClearHeld
|
|
escalated := old.Severity == Warning && rec.Severity == Urgent
|
|
gone := now.Sub(old.Cleared)
|
|
old.Cleared = time.Time{}
|
|
old.ClearUnsaid, old.ClearSilent, old.ClearHeld = false, false, ""
|
|
if blip {
|
|
old.Blips++
|
|
} else {
|
|
old.Count++
|
|
}
|
|
old.Summary, old.Severity, old.SilencedTill, old.SilencedWhy = rec.Summary, rec.Severity, rec.SilencedTill, rec.SilencedWhy
|
|
old.takeWords(rec)
|
|
old.Seen = now
|
|
h.mu.Unlock()
|
|
switch {
|
|
case quiet || old.silenced(now):
|
|
case blip && escalated:
|
|
// Back before its clearing was said, and urgent now: said as an escalation, which rings.
|
|
h.deliver(old, "escalated")
|
|
case blip && held != "":
|
|
// Back before its clearing was said: what was waiting to be said of it when it cleared is said now.
|
|
h.deliver(old, held)
|
|
case blip:
|
|
// Its clearing was never said (issue 373): to the operator it never ended, so nothing is said now.
|
|
h.Logf("[messenger] %s back %s after it cleared, which was not said yet: the same message continues, nothing said",
|
|
old.Key, roughly(gone))
|
|
h.record(Sent{At: now, Channel: "-", Key: old.Key, What: "reopened", Outcome: "quiet: back before its clearing was said"})
|
|
case old.told() && old.Severity == Urgent:
|
|
// Its end was said, and it is back: urgent, it rings again, in place of its message.
|
|
h.deliver(old, "reopened")
|
|
case old.told():
|
|
h.edit(old, "reopened", false)
|
|
case !needsNothing(old) && !h.queued(old.Key):
|
|
// Cleared before it was ever said, and back: a problem the operator has not heard of is said.
|
|
h.deliver(old, "raised")
|
|
}
|
|
h.persist(old)
|
|
return
|
|
}
|
|
r := rec
|
|
r.Count = 1
|
|
if r.Raised.IsZero() {
|
|
r.Raised = now
|
|
}
|
|
r.Sent = map[string]string{}
|
|
r.Seen = now
|
|
r.History = quiet
|
|
h.mu.Lock()
|
|
h.open[r.Key] = &r
|
|
h.mu.Unlock()
|
|
switch {
|
|
case quiet:
|
|
case r.silenced(now):
|
|
h.Logf("[messenger] %s raised while silenced until %s: nothing sent", r.Key, r.SilencedTill.Format(time.RFC3339))
|
|
default:
|
|
h.deliver(&r, "raised")
|
|
}
|
|
h.persist(&r)
|
|
}
|
|
|
|
// nowNeeded says a record kept quiet as needing nothing, once its words say it needs the operator.
|
|
func (h *Holder) nowNeeded(r *Record, wasQuiet, quiet bool) {
|
|
if wasQuiet && !needsNothing(r) && !r.told() && !quiet && !r.silenced(h.Now()) && !h.queued(r.Key) {
|
|
h.deliver(r, "raised")
|
|
}
|
|
}
|
|
|
|
func (h *Holder) changed(rec Record, quiet bool) {
|
|
h.mu.Lock()
|
|
old := h.open[rec.Key]
|
|
h.mu.Unlock()
|
|
if old == nil || !old.Cleared.IsZero() {
|
|
// A change to something this holder never heard raised: read as a raise, so it is said
|
|
// (when it is news).
|
|
h.Logf("[messenger] %s changed and was not open here; taken as raised", rec.Key)
|
|
h.raised(rec, quiet)
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
escalated := old.Severity == Warning && rec.Severity == Urgent
|
|
wasQuiet := needsNothing(old)
|
|
wasSilenced := old.silenced(h.Now())
|
|
old.Summary, old.Subject = rec.Summary, rec.Subject
|
|
old.takeSubject(rec)
|
|
old.SilencedTill, old.SilencedWhy = rec.SilencedTill, rec.SilencedWhy
|
|
old.takeWords(rec)
|
|
if rec.Severity != "" {
|
|
old.Severity = rec.Severity
|
|
}
|
|
old.Seen = h.Now()
|
|
h.mu.Unlock()
|
|
h.silenceWords(old, wasSilenced)
|
|
switch {
|
|
case escalated && !quiet && !old.silenced(h.Now()):
|
|
// Routing differs for urgent: said once more, to both channels.
|
|
h.deliver(old, "escalated")
|
|
default:
|
|
h.nowNeeded(old, wasQuiet, quiet)
|
|
}
|
|
h.persist(old)
|
|
}
|
|
|
|
// silenceWords keeps a shown message true to a silence (issue 373). Silenced, the message is edited in place
|
|
// to say so — until when, and why — with no answers and no urgency, so the desk's popup times out instead of
|
|
// standing as if still open. When the silence ends and the condition is still open, the message is edited back
|
|
// to its open words, and the usual rules apply again.
|
|
func (h *Holder) silenceWords(r *Record, wasSilenced bool) {
|
|
now := h.Now()
|
|
h.mu.Lock()
|
|
silenced, told, said := r.silenced(now), r.told(), r.SilenceSaid
|
|
key, till, why := r.Key, r.SilencedTill, r.SilencedWhy
|
|
h.mu.Unlock()
|
|
switch {
|
|
case silenced && !wasSilenced && told && !said:
|
|
h.mu.Lock()
|
|
r.SilenceSaid = true
|
|
h.mu.Unlock()
|
|
h.Logf("[messenger] %s silenced until %s (%s): its message says so, and nothing more is said of it",
|
|
key, till.Format(time.RFC3339), why)
|
|
h.edit(r, "silenced", false)
|
|
case !silenced && said:
|
|
h.mu.Lock()
|
|
r.SilenceSaid = false
|
|
h.mu.Unlock()
|
|
h.Logf("[messenger] %s is no longer silenced and still open: its message says it is open again", key)
|
|
h.edit(r, "raised", false)
|
|
}
|
|
}
|
|
|
|
// cleared ends a record. Something never said is ended without a word; what was waiting to be said
|
|
// about it is dropped. quiet: it ended while this holder was not listening, so it is said only where
|
|
// an edit notifies nobody.
|
|
func (h *Holder) cleared(key string, quiet bool) {
|
|
now := h.Now()
|
|
h.mu.Lock()
|
|
old := h.open[key]
|
|
h.mu.Unlock()
|
|
if old == nil || !old.Cleared.IsZero() {
|
|
h.Logf("[messenger] %s cleared and was not open here: nothing to say", key)
|
|
return
|
|
}
|
|
held := h.queuedWhat(key)
|
|
h.mu.Lock()
|
|
if held == "" {
|
|
held = old.Pending
|
|
}
|
|
old.Cleared = now
|
|
old.Pending = ""
|
|
old.Owed = "" // what a walk held of it is owed no more: it ended (the review of PR 238)
|
|
old.Folded = false
|
|
old.Seen = now
|
|
told := old.told()
|
|
h.mu.Unlock()
|
|
dropped := h.unqueue(key)
|
|
switch {
|
|
case old.silenced(now) && !old.SilenceSaid:
|
|
// A silenced condition sends nothing, its clearing included.
|
|
case old.silenced(now):
|
|
// Its message says it is silenced: said resolved where an edit notifies nobody, so it does not stand.
|
|
h.mu.Lock()
|
|
old.ClearUnsaid, old.ClearSilent, old.ClearHeld = true, true, ""
|
|
h.mu.Unlock()
|
|
case !told:
|
|
if dropped > 0 {
|
|
h.Logf("[messenger] %s cleared before it was said: nothing to unsay", key)
|
|
}
|
|
default:
|
|
// Said once it stays cleared (ClearSettle; a condition that came back, ReopenWindow): a blip of a few
|
|
// seconds is not said at all (issue 373).
|
|
h.mu.Lock()
|
|
old.ClearUnsaid, old.ClearSilent, old.ClearHeld = true, quiet, held
|
|
h.mu.Unlock()
|
|
}
|
|
h.persist(old)
|
|
}
|
|
|
|
// clearSettle is how long a record stays cleared before its clearing is said: one that came back before
|
|
// waits the whole reopen window, so a condition that flaps is said cleared once, when it stopped flapping.
|
|
func clearSettle(r *Record) time.Duration {
|
|
if r.Count > 1 || r.Blips > 0 {
|
|
return ReopenWindow - time.Minute
|
|
}
|
|
return ClearSettle
|
|
}
|
|
|
|
// lastSaid is when a message about a record last went out.
|
|
func lastSaid(r *Record) time.Time {
|
|
if !r.LastSaid.IsZero() {
|
|
return r.LastSaid
|
|
}
|
|
return r.FirstSent
|
|
}
|
|
|
|
// Tick does what time asks: reminders, retries, what is held, and forgetting what cleared long
|
|
// enough ago that a new raise is a new message.
|
|
func (h *Holder) Tick() {
|
|
h.warmRootFree(false) // outside the work: what is offered inside it reads a recent word
|
|
h.warmTracking()
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
now := h.Now()
|
|
var recs []*Record
|
|
for _, r := range h.open {
|
|
recs = append(recs, r)
|
|
}
|
|
h.mu.Unlock()
|
|
sort.Slice(recs, func(i, j int) bool { return recs[i].Raised.Before(recs[j].Raised) })
|
|
h.sweepAsks(now)
|
|
h.breakGlass()
|
|
h.boundWalks(now)
|
|
h.duePasses(now)
|
|
for _, r := range recs {
|
|
switch {
|
|
case !r.Cleared.IsZero():
|
|
if r.ClearUnsaid && now.Sub(r.Cleared) >= clearSettle(r) {
|
|
h.mu.Lock()
|
|
r.ClearUnsaid = false
|
|
silent := r.ClearSilent
|
|
h.mu.Unlock()
|
|
if !r.silenced(now) || r.SilenceSaid {
|
|
h.edit(r, "cleared", silent || r.silenced(now))
|
|
}
|
|
h.persist(r)
|
|
}
|
|
if !r.ClearUnsaid && now.Sub(r.Cleared) >= ReopenWindow && !h.queued(r.Key) {
|
|
h.mu.Lock()
|
|
delete(h.open, r.Key)
|
|
h.mu.Unlock()
|
|
if h.Store != nil {
|
|
if err := h.Store.Delete(r.Key); err != nil {
|
|
h.noteStore(err)
|
|
}
|
|
}
|
|
}
|
|
case r.silenced(now):
|
|
case r.SilenceSaid:
|
|
// Its silence ran out while it is still open: the message is open again.
|
|
h.silenceWords(r, true)
|
|
h.persist(r)
|
|
case r.Pending != "" && !h.queued(r.Key):
|
|
// No channel could take it when it was said: tried again.
|
|
h.deliver(r, r.Pending)
|
|
h.persist(r)
|
|
case needsNothing(r) && !r.told() && !r.QuietReminded && now.Sub(r.Raised) >= QuietRemindAfter:
|
|
// Quiet for a day: said once, so nothing stays silent for ever (ADR 0258).
|
|
h.mu.Lock()
|
|
r.QuietReminded = true
|
|
h.mu.Unlock()
|
|
h.deliver(r, "quiet-reminder")
|
|
h.persist(r)
|
|
case !r.Reminded && !r.FirstSent.IsZero() && now.Sub(r.Raised) >= remindAfter(r.Severity) &&
|
|
now.Sub(lastSaid(r)) >= remindAfter(r.Severity):
|
|
// Once more after its bound — counted from its raising, and never sooner after it was
|
|
// last said (an old condition said at a reading of the state is not reminded at once, nor one
|
|
// just escalated: issue 373).
|
|
h.mu.Lock()
|
|
r.Reminded = true
|
|
h.mu.Unlock()
|
|
h.deliver(r, "reminder")
|
|
h.persist(r)
|
|
}
|
|
}
|
|
h.flush(now, false)
|
|
}
|
|
|
|
// Flush sends what is due on each channel; the runtime calls it every few seconds, so a held
|
|
// message waits its window and not a minute more.
|
|
func (h *Holder) Flush() {
|
|
h.warmRootFree(false) // outside the work: what is offered inside it reads a recent word
|
|
h.warmTracking()
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
now := h.Now()
|
|
h.mu.Unlock()
|
|
h.duePasses(now)
|
|
h.flush(now, false)
|
|
}
|
|
|
|
func remindAfter(severity string) time.Duration {
|
|
if severity == Urgent {
|
|
return RemindUrgent
|
|
}
|
|
return RemindWarning
|
|
}
|
|
|
|
// say composes a record's message under the content rule: refused, it is composed again with less of
|
|
// it, until what remains may leave. The refusal is recorded and stated once per record.
|
|
func (h *Holder) say(r *Record, what string) Message { return h.sayOn(r, what, "") }
|
|
|
|
// sayOn is a record's message on one channel kind: its words, what came of an answer, the answers that kind
|
|
// can carry, and where the others can be given. An empty kind is the words alone.
|
|
func (h *Holder) sayOn(r *Record, what, kind string) Message {
|
|
m := h.words(r, what)
|
|
if what == "cleared" || what == "silenced" || m.Title == "A mesh message was withheld" {
|
|
return m
|
|
}
|
|
// On a kind that proves who answers, an ask's explanation is shown with its values whole (issue 383): the
|
|
// path or the address the operator approves is readable there; a secret's shape still is not.
|
|
whole := h.wholeOf(r, kind)
|
|
if whole != "" {
|
|
if wm, ok := h.wholeWords(r, what, whole); ok {
|
|
m = wm
|
|
} else {
|
|
whole = ""
|
|
}
|
|
}
|
|
if r.Answered != "" {
|
|
m = h.answeredWords(r, m, kind)
|
|
}
|
|
if kind == "" {
|
|
return m
|
|
}
|
|
withheld := map[string]bool{} // the labels of answers whose effect could not be said
|
|
if rec := h.openAsk(r); rec != nil {
|
|
// The note (issue 639): who asks, as the bus proves it (the asker is the subject's, never the body's), what
|
|
// each answer does, what happens if nobody answers (the review of 2026-10-09), where else it is answered,
|
|
// and the asker's Details. The body stays the asker's summary and points, read at a glance.
|
|
var note []string
|
|
for _, line := range h.askLines(rec, r) {
|
|
if _, ok := Check(line); ok {
|
|
note = append(note, line)
|
|
}
|
|
}
|
|
// **An answer whose effect is not said is not offered** (the confirmation review of 2026-10-09): its line
|
|
// refused by the content rule, its button went out all the same, and a tap would authorise what the
|
|
// person was not shown.
|
|
for _, o := range rec.Ask.Options {
|
|
if _, ok := Check(o.Label + ": " + o.Does + "."); !ok || strings.TrimSpace(o.Does) == "" {
|
|
withheld[o.Label] = true
|
|
}
|
|
}
|
|
if line := h.elsewhere(rec, kind); line != "" {
|
|
if _, ok := Check(line); ok {
|
|
note = append(note, line)
|
|
}
|
|
}
|
|
// On a kind that proves who answers, the Details whole (issue 383): every value an approval sets, held to
|
|
// the secret rule alone, as the whole words are.
|
|
details, passes := rec.Ask.Details, Check
|
|
if wd := h.wholeDetailsOf(r, kind); wd != "" {
|
|
details, passes = wd, CheckSecret
|
|
}
|
|
for _, line := range strings.Split(strings.TrimSpace(details), "\n") {
|
|
if line = strings.TrimSpace(line); line == "" {
|
|
continue
|
|
}
|
|
if refusal, ok := passes(line); !ok {
|
|
line = withheldLine(line, refusal, passes)
|
|
}
|
|
note = append(note, line)
|
|
}
|
|
m.Note = strings.Join(note, "\n")
|
|
}
|
|
for _, c := range h.choices(r, kind) {
|
|
if !withheld[c.Label] {
|
|
m.Actions = append(m.Actions, c)
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// answeredWords is a message that says what came of its question. An ask's own message keeps the question
|
|
// whole — its headline as the title, the asker's summary, points and link as the body, its Details after them —
|
|
// so the operator can read back what they answered (issue 714, which undid issue 639's one line). How it ended
|
|
// ("You chose Approve at 13:30 on Telegram.") and what the router knows of the asker after it are the note,
|
|
// below the question, and its answers are gone (issue 372). A channel that must shorten it cuts the body from
|
|
// its end, the Details first; never the headline or the note. A condition's message keeps its own title, and
|
|
// says how it ended first.
|
|
func (h *Holder) answeredWords(r *Record, m Message, kind string) Message {
|
|
if r.Origin == "ask" {
|
|
headline, _ := wordsOf(r)
|
|
body := []string{}
|
|
if b := strings.TrimSpace(m.Body); b != "" {
|
|
body = append(body, b)
|
|
}
|
|
body = append(body, h.answeredDetails(r, kind)...)
|
|
out := Message{Title: headline, Body: strings.Join(body, "\n"), Note: strings.TrimSpace(r.Answered),
|
|
Urgent: m.Urgent, Quiet: m.Quiet}
|
|
if _, ok := Check(out.Title + "\n" + out.Note); ok {
|
|
return out
|
|
}
|
|
// Its headline refused (it is the asker's): the outcome is still said, before what may leave.
|
|
}
|
|
m.Body = strings.TrimSpace(r.Answered + "\n" + m.Body)
|
|
return m
|
|
}
|
|
|
|
// answeredDetails are an answered ask's Details, line by line, as its open message noted them (issue 714): whole
|
|
// on a kind that proves who answers, a line that may not leave withheld and said so.
|
|
func (h *Holder) answeredDetails(r *Record, kind string) []string {
|
|
if r.Ask == "" {
|
|
return nil
|
|
}
|
|
e := h.askOf(r.Ask)
|
|
if e == nil {
|
|
return nil
|
|
}
|
|
details, passes := e.rec.Ask.Details, Check
|
|
if wd := h.wholeDetailsOf(r, kind); wd != "" {
|
|
details, passes = wd, CheckSecret
|
|
}
|
|
var lines []string
|
|
for _, line := range strings.Split(strings.TrimSpace(details), "\n") {
|
|
if line = strings.TrimSpace(line); line == "" {
|
|
continue
|
|
}
|
|
if refusal, ok := passes(line); !ok {
|
|
line = withheldLine(line, refusal, passes)
|
|
}
|
|
lines = append(lines, line)
|
|
}
|
|
return lines
|
|
}
|
|
|
|
// wholeWords is a record's message with the ask's whole words in place of its explanation, when they carry no
|
|
// secret's shape once composed; else false, and the message under the whole content rule stands.
|
|
func (h *Holder) wholeWords(r *Record, what, whole string) (Message, bool) {
|
|
c := *r
|
|
c.Explanation = whole
|
|
m := compose(&c, what, h.Now(), h.zone(), 0)
|
|
if refusal, ok := CheckSecret(m.Text()); !ok {
|
|
h.Logf("[messenger] the whole words of %s carry %s; shown as on a channel that proves nothing", r.Key, refusal)
|
|
return Message{}, false
|
|
}
|
|
return m, true
|
|
}
|
|
|
|
// askLines are what an ask's message notes of the ask itself, in two lines: who asks and when, with what happens
|
|
// if nobody answers; then each option's effect.
|
|
func (h *Holder) askLines(rec *AskRecord, r *Record) []string {
|
|
asker := rec.Asker
|
|
if asker == SelfAsker {
|
|
asker = "the router itself"
|
|
}
|
|
first := "Asked by " + asker + " at " + when(r.Raised, h.Now(), h.zone()) + "."
|
|
if !rec.Ask.Expires.IsZero() && rec.Ask.OnExpiry != "" {
|
|
first += " Unanswered by " + by(rec.Ask.Expires, h.Now(), h.zone()) + ": " + strings.TrimSuffix(rec.Ask.OnExpiry, ".") + "."
|
|
}
|
|
lines := []string{first}
|
|
var does []string
|
|
for _, o := range rec.Ask.Options {
|
|
if o.Does != "" {
|
|
does = append(does, o.Label+": "+strings.TrimSuffix(o.Does, ".")+".")
|
|
}
|
|
}
|
|
if len(does) > 0 {
|
|
lines = append(lines, strings.Join(does, " "))
|
|
}
|
|
return lines
|
|
}
|
|
|
|
// words is a record's message under the content rule: refused, it is composed again with less of it,
|
|
// until what remains may leave. The refusal is recorded and stated once per record.
|
|
func (h *Holder) words(r *Record, what string) Message {
|
|
now := h.Now()
|
|
said, _, refusal, ok := h.allowedWords(r, what, now)
|
|
if ok || r.Refused != "" {
|
|
return said
|
|
}
|
|
h.mu.Lock()
|
|
r.Refused = refusal.What
|
|
h.mu.Unlock()
|
|
key := r.Key
|
|
if _, keyOK := Check(key); !keyOK {
|
|
key = "(withheld)"
|
|
}
|
|
h.mu.Lock()
|
|
h.refusals = append(h.refusals, RefusalNote{At: now, Key: key, Class: refusal.Class, What: refusal.What})
|
|
if len(h.refusals) > KeptRefusals {
|
|
h.refusals = h.refusals[len(h.refusals)-KeptRefusals:]
|
|
}
|
|
h.mu.Unlock()
|
|
h.Logf("[messenger] refused the message for %s: it carried %s; sending channel-refused with its words withheld", key, refusal)
|
|
if h.Emit != nil {
|
|
if err := h.Emit("refused", map[string]any{"key": key, "class": refusal.Class, "what": refusal.What}); err != nil {
|
|
h.Logf("[messenger] could not state the refusal on the bus: %v", err)
|
|
}
|
|
}
|
|
return said
|
|
}
|
|
|
|
// composeTracked is compose with the line on the issues that track the condition (tracking.go), so the content
|
|
// rule judges the message as it is sent.
|
|
func (h *Holder) composeTracked(r *Record, what string, now time.Time, withhold int) (Message, string) {
|
|
m := compose(r, what, now, h.zone(), withhold)
|
|
if !h.tracksAsked() || withhold >= 2 {
|
|
return m, ""
|
|
}
|
|
issues, known := h.trackedBy(r.Key)
|
|
return withTracking(m, r, what, issues, known, now, h.zone())
|
|
}
|
|
|
|
// allowedWords is the message the content rule leaves of a record: its own words when they may leave the
|
|
// mesh, else the same message composed again with less of it until what remains may, else the withheld
|
|
// one. It answers the tracking line it carries, what its own words carried, and whether those words are
|
|
// what it says. **It changes nothing**: `words` is this walk with the refusal recorded and stated once,
|
|
// and `Preview` (preview.go) is this walk alone, so a read of a wording cannot alter what a send says.
|
|
func (h *Holder) allowedWords(r *Record, what string, now time.Time) (Message, string, Refusal, bool) {
|
|
if r.Refused != "" {
|
|
// Refused before: its words stay withheld in every later message too.
|
|
for w := 1; w <= 3; w++ {
|
|
if m, line := h.composeTracked(r, what, now, w); allowed(m) {
|
|
return m, line, Refusal{What: r.Refused}, false
|
|
}
|
|
}
|
|
}
|
|
m, line := h.composeTracked(r, what, now, 0)
|
|
refusal, ok := Check(m.Text())
|
|
if ok {
|
|
return m, line, Refusal{}, true
|
|
}
|
|
// Composed from a copy carrying the reason, because a withheld message says what it carried.
|
|
reduced := *r
|
|
reduced.Refused = refusal.What
|
|
for w := 1; w <= 3; w++ {
|
|
if m, line := h.composeTracked(&reduced, what, now, w); allowed(m) {
|
|
return m, line, refusal, false
|
|
}
|
|
}
|
|
return withheld(refusal.What, r.Severity == Urgent), "", refusal, false
|
|
}
|
|
|
|
// allowed says whether a message's words may leave the mesh.
|
|
func allowed(m Message) bool {
|
|
_, ok := Check(m.Text())
|
|
return ok
|
|
}
|
|
|
|
// deliver sends a record's message where its severity routes it.
|
|
|
|
func (h *Holder) channels() []Channel {
|
|
var out []Channel
|
|
for _, c := range []Channel{h.Away, h.Desk} {
|
|
if c != nil {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (h *Holder) record(s Sent) {
|
|
h.mu.Lock()
|
|
h.recent = append(h.recent, s)
|
|
if len(h.recent) > KeptSends {
|
|
h.recent = h.recent[len(h.recent)-KeptSends:]
|
|
}
|
|
recent := append([]Sent(nil), h.recent...)
|
|
h.mu.Unlock()
|
|
if h.Store != nil {
|
|
if err := h.Store.PutRecent(recent); err != nil {
|
|
h.noteStore(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *Holder) persist(r *Record) {
|
|
if h.Store == nil {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
c := *r
|
|
h.mu.Unlock()
|
|
if err := h.Store.Put(c); err != nil {
|
|
h.noteStore(err)
|
|
}
|
|
}
|
|
|
|
func (h *Holder) noteStore(err error) {
|
|
h.mu.Lock()
|
|
first := h.storeErr == ""
|
|
h.storeErr = err.Error()
|
|
h.mu.Unlock()
|
|
if first {
|
|
h.Logf("[messenger] cannot write its state (open messages are held in memory only until it can): %v", err)
|
|
}
|
|
}
|
|
|
|
func (h *Holder) noteChannel(name string, err error) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
now := h.Now()
|
|
if err == nil {
|
|
if h.chanErr[name] != "" {
|
|
h.Logf("[messenger] %s sends again", name)
|
|
}
|
|
h.chanErr[name] = ""
|
|
h.chanOK[name] = now
|
|
return
|
|
}
|
|
// Said in the log when it changes, and at most every ten minutes while it holds.
|
|
if h.chanErr[name] != err.Error() || now.Sub(h.chanErrAt[name]) >= 10*time.Minute {
|
|
h.Logf("[messenger] %s cannot send: %v", name, err)
|
|
h.chanErrAt[name] = now
|
|
}
|
|
h.chanErr[name] = err.Error()
|
|
}
|
|
|
|
func (h *Holder) whyNot() string {
|
|
var parts []string
|
|
for _, ch := range h.channels() {
|
|
if err := ch.Ready(); err != nil {
|
|
parts = append(parts, ch.Name()+": "+err.Error())
|
|
continue
|
|
}
|
|
h.mu.Lock()
|
|
e := h.chanErr[ch.Name()]
|
|
h.mu.Unlock()
|
|
if e != "" {
|
|
parts = append(parts, ch.Name()+": "+e)
|
|
}
|
|
}
|
|
if len(parts) == 0 {
|
|
return "no channel"
|
|
}
|
|
return strings.Join(parts, "; ")
|
|
}
|
|
|
|
func roughly(d time.Duration) string {
|
|
switch {
|
|
case d < 0:
|
|
return "a moment"
|
|
case d < 2*time.Minute:
|
|
return fmt.Sprintf("%d s", int(d.Seconds()))
|
|
case d < 2*time.Hour:
|
|
return fmt.Sprintf("%d min", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%.1f h", d.Hours())
|
|
}
|
|
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
|
}
|
|
|
|
// withheldLine is a note line the rule refused, said withheld: a change's line ("+ key: value") keeps its key's name,
|
|
// so the operator knows which setting holds a value they are not shown (the review of mesh-controller #304); any
|
|
// other line, or one whose key the rule refuses too, is said withheld whole.
|
|
func withheldLine(line string, refusal Refusal, passes func(string) (Refusal, bool)) string {
|
|
if len(line) > 2 && strings.ContainsRune("+~-=", rune(line[0])) && line[1] == ' ' {
|
|
key := line[2:]
|
|
if i := strings.IndexAny(key, ": "); i > 0 {
|
|
key = key[:i]
|
|
}
|
|
named := line[:2] + key + " (a value carrying " + refusal.What + " is withheld here)"
|
|
if _, ok := passes(named); ok {
|
|
return named
|
|
}
|
|
}
|
|
return "(a line carrying " + refusal.What + " is withheld here)"
|
|
}
|