jochen
0ae7933d54
The mesh noticed 48 core failures in six days and told nobody (ADR 0227). messenger holds the operator-channel seat: it consumes the controller's condition events and sends them to Telegram and the desktop notifier, deduplicated by key, reminded once, edited on clear, capped at 20 an hour with the rest folded, and refusing anything carrying an address, a path or a secret. mesh-watcher, on a machine other than the control node, sends to Telegram directly when the self-check heartbeat or the bus goes silent.
776 lines
21 KiB
Go
776 lines
21 KiB
Go
package main
|
|
|
|
// The holder of the operator-channel seat (novox/hq to-be 45 §5, ADR 0227): what is sent, to whom,
|
|
// when, and how often. The controller decides what is wrong; this decides what is said.
|
|
//
|
|
// - On `condition-raised`: one message, deduplicated by the condition's key. Said again with the
|
|
// same key while open, it is the same message, not a second.
|
|
// - Once more if still open after 1 hour (urgent) or 12 hours (warning).
|
|
// - On `condition-cleared`: the first message is edited where the channel can (both can);
|
|
// otherwise a new one says it. Cleared and raised again within ten minutes, it is the same
|
|
// message, edited back to open — not a new one.
|
|
// - A silenced condition sends nothing.
|
|
// - Urgent to both channels; warning to the desktop when the operator's session is there,
|
|
// otherwise to Telegram.
|
|
// - At most twenty messages an hour per channel; the excess is held and folded into one message
|
|
// naming them all, sent at most every ten minutes — the cap is said, never silent.
|
|
// - A message carrying an address, a path or a secret is refused (content.go); what is sent in its
|
|
// place says `channel-refused`, with the offending part withheld.
|
|
// - A channel that cannot send says so in the status and the log, and the message is tried again
|
|
// every minute while the condition is open.
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
RemindUrgent = time.Hour
|
|
RemindWarning = 12 * time.Hour
|
|
ReopenWindow = 10 * time.Minute
|
|
CapPerHour = 20
|
|
FoldEvery = 10 * time.Minute
|
|
KeptSends = 200
|
|
KeptRefusals = 50
|
|
)
|
|
|
|
// Message is what a channel shows: a title line and a body.
|
|
type Message struct {
|
|
Title string
|
|
Body string
|
|
Urgent bool
|
|
Quiet bool // a clearing: shown without urgency
|
|
}
|
|
|
|
func (m Message) Text() string {
|
|
if m.Body == "" {
|
|
return m.Title
|
|
}
|
|
return m.Title + "\n" + m.Body
|
|
}
|
|
|
|
// Channel is one way to the operator.
|
|
type Channel interface {
|
|
Name() string
|
|
Ready() error
|
|
Send(Message) (string, error)
|
|
Edit(id string, m Message) error
|
|
CanEdit() bool
|
|
}
|
|
|
|
// Record is one open message, kept in the module's own state so a restart forgets nothing.
|
|
type Record struct {
|
|
Key string `json:"key"`
|
|
Kind string `json:"kind"`
|
|
Subject string `json:"subject"`
|
|
Severity string `json:"severity"`
|
|
Summary string `json:"summary"`
|
|
Origin string `json:"origin"`
|
|
More string `json:"more"`
|
|
Raised time.Time `json:"raised"`
|
|
SilencedTill time.Time `json:"silenced_till,omitempty"`
|
|
Sent map[string]string `json:"sent,omitempty"` // channel -> the first message's id
|
|
FirstSent time.Time `json:"first_sent,omitempty"`
|
|
Reminded bool `json:"reminded,omitempty"`
|
|
Pending string `json:"pending,omitempty"` // what is still to be said: raised, reminder, …
|
|
Folded bool `json:"folded,omitempty"`
|
|
Cleared time.Time `json:"cleared,omitempty"`
|
|
Count int `json:"count"`
|
|
Refused string `json:"refused,omitempty"`
|
|
}
|
|
|
|
func (r *Record) silenced(now time.Time) bool {
|
|
return !r.SilencedTill.IsZero() && now.Before(r.SilencedTill)
|
|
}
|
|
|
|
// Sent is one message that went out, or was held, for the history.
|
|
type Sent struct {
|
|
At time.Time `json:"at"`
|
|
Channel string `json:"channel"`
|
|
Key string `json:"key"`
|
|
What string `json:"what"`
|
|
Outcome string `json:"outcome"` // sent, edited, folded, failed: <why>
|
|
}
|
|
|
|
// RefusalNote is one refused message: never its text.
|
|
type RefusalNote struct {
|
|
At time.Time `json:"at"`
|
|
Key string `json:"key"`
|
|
Class string `json:"class"`
|
|
What string `json:"what"`
|
|
}
|
|
|
|
// Store is the module's own state (ADR 0201): the open messages, and the recent sends.
|
|
type Store interface {
|
|
Put(r Record) error
|
|
Delete(key string) error
|
|
All() ([]Record, error)
|
|
PutRecent([]Sent) error
|
|
Recent() ([]Sent, error)
|
|
}
|
|
|
|
type foldEntry struct {
|
|
Key, Severity, What string
|
|
}
|
|
|
|
// Holder is the seat's holder.
|
|
type Holder struct {
|
|
Telegram Channel
|
|
Desktop Channel
|
|
Store Store
|
|
Now func() time.Time
|
|
Logf func(string, ...any)
|
|
// Emit states a fact as this module (refused); nil states nothing.
|
|
Emit func(event string, body any) error
|
|
|
|
work sync.Mutex // one event, call or tick at a time
|
|
mu sync.Mutex // what the status reads
|
|
|
|
open map[string]*Record
|
|
recent []Sent
|
|
refusals []RefusalNote
|
|
folds map[string][]foldEntry
|
|
lastFold map[string]time.Time
|
|
chanErr map[string]string
|
|
chanErrAt map[string]time.Time
|
|
chanOK map[string]time.Time
|
|
unreadable int
|
|
lastBad string
|
|
lastBadAt time.Time
|
|
saidOnce map[string]time.Time
|
|
storeErr string
|
|
heard map[string]int
|
|
lastHeard time.Time
|
|
}
|
|
|
|
func (h *Holder) init() {
|
|
if h.open == nil {
|
|
h.open = map[string]*Record{}
|
|
h.folds = map[string][]foldEntry{}
|
|
h.lastFold = map[string]time.Time{}
|
|
h.chanErr = map[string]string{}
|
|
h.chanErrAt = map[string]time.Time{}
|
|
h.chanOK = map[string]time.Time{}
|
|
h.saidOnce = map[string]time.Time{}
|
|
h.heard = map[string]int{}
|
|
}
|
|
if h.Now == nil {
|
|
h.Now = time.Now
|
|
}
|
|
if h.Logf == nil {
|
|
h.Logf = func(string, ...any) {}
|
|
}
|
|
}
|
|
|
|
// Load reads back what was open and what was sent before a restart.
|
|
func (h *Holder) Load() error {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.init()
|
|
if h.Store == nil {
|
|
return nil
|
|
}
|
|
recs, err := h.Store.All()
|
|
if err != nil {
|
|
h.storeErr = err.Error()
|
|
return err
|
|
}
|
|
for i := range recs {
|
|
r := recs[i]
|
|
h.open[r.Key] = &r
|
|
}
|
|
if sent, err := h.Store.Recent(); err == nil {
|
|
h.recent = sent
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Condition takes one of the controller's condition events.
|
|
func (h *Holder) Condition(event string, c Condition) {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
h.heard[event]++
|
|
h.lastHeard = h.Now()
|
|
h.mu.Unlock()
|
|
rec := Record{
|
|
Key: c.Key, Kind: c.Kind, Subject: c.SubjectWords(), Severity: c.Severity, Summary: c.Summary,
|
|
Origin: "condition", More: "conditions show " + c.Key, Raised: c.Raised, SilencedTill: c.SilencedTill,
|
|
}
|
|
switch event {
|
|
case EventRaised:
|
|
h.raised(rec)
|
|
case EventChanged:
|
|
h.changed(rec)
|
|
case EventCleared:
|
|
h.cleared(rec.Key)
|
|
}
|
|
}
|
|
|
|
// Unreadable records an event that could not be read, and tells the operator — once an hour.
|
|
func (h *Holder) Unreadable(key string, err error) {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
h.unreadable++
|
|
h.lastBad = err.Error()
|
|
h.lastBadAt = h.Now()
|
|
n := h.unreadable
|
|
h.mu.Unlock()
|
|
h.Logf("[messenger] refused %s: %v (unreadable events since start: %d)", key, err, n)
|
|
h.sayOnce("messenger.unreadable-event", time.Hour, Message{
|
|
Title: "WARNING: a condition event could not be read",
|
|
Body: fmt.Sprintf("the operator-channel's holder could not read %d condition event(s) from the controller; "+
|
|
"what was wrong is in messenger_status. A condition may be open that was not said.", n),
|
|
})
|
|
}
|
|
|
|
func (h *Holder) raised(rec Record) {
|
|
now := h.Now()
|
|
h.mu.Lock()
|
|
old := h.open[rec.Key]
|
|
h.mu.Unlock()
|
|
if old != nil && old.Cleared.IsZero() {
|
|
// Said again while open: the same message. Its words are kept current; nothing is sent.
|
|
h.mu.Lock()
|
|
old.Summary, old.Subject, old.Kind, old.SilencedTill = rec.Summary, rec.Subject, rec.Kind, rec.SilencedTill
|
|
if rec.Severity != "" {
|
|
old.Severity = rec.Severity
|
|
}
|
|
h.mu.Unlock()
|
|
h.persist(old)
|
|
return
|
|
}
|
|
if old != nil && now.Sub(old.Cleared) < ReopenWindow {
|
|
// Cleared and raised again within ten minutes: the same message, back to open (to-be 45 §2).
|
|
h.mu.Lock()
|
|
old.Cleared = time.Time{}
|
|
old.Count++
|
|
old.Summary, old.Severity, old.SilencedTill = rec.Summary, rec.Severity, rec.SilencedTill
|
|
h.mu.Unlock()
|
|
if !old.silenced(now) && len(old.Sent) > 0 {
|
|
h.edit(old, "reopened")
|
|
}
|
|
h.persist(old)
|
|
return
|
|
}
|
|
r := rec
|
|
r.Count = 1
|
|
if r.Raised.IsZero() {
|
|
r.Raised = now
|
|
}
|
|
r.Sent = map[string]string{}
|
|
h.mu.Lock()
|
|
h.open[r.Key] = &r
|
|
h.mu.Unlock()
|
|
if r.silenced(now) {
|
|
h.Logf("[messenger] %s raised while silenced until %s: nothing sent", r.Key, r.SilencedTill.Format(time.RFC3339))
|
|
h.persist(&r)
|
|
return
|
|
}
|
|
h.deliver(&r, "raised")
|
|
h.persist(&r)
|
|
}
|
|
|
|
func (h *Holder) changed(rec Record) {
|
|
h.mu.Lock()
|
|
old := h.open[rec.Key]
|
|
h.mu.Unlock()
|
|
if old == nil || !old.Cleared.IsZero() {
|
|
// A change to something this holder never heard raised: read as a raise, so it is said.
|
|
h.Logf("[messenger] %s changed and was not open here; taken as raised", rec.Key)
|
|
h.raised(rec)
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
escalated := old.Severity == Warning && rec.Severity == Urgent
|
|
old.Summary, old.Subject, old.SilencedTill = rec.Summary, rec.Subject, rec.SilencedTill
|
|
if rec.Severity != "" {
|
|
old.Severity = rec.Severity
|
|
}
|
|
h.mu.Unlock()
|
|
if escalated && !old.silenced(h.Now()) {
|
|
// Routing differs for urgent: said once more, to both channels.
|
|
h.deliver(old, "escalated")
|
|
}
|
|
h.persist(old)
|
|
}
|
|
|
|
func (h *Holder) cleared(key string) {
|
|
now := h.Now()
|
|
h.mu.Lock()
|
|
old := h.open[key]
|
|
h.mu.Unlock()
|
|
if old == nil || !old.Cleared.IsZero() {
|
|
h.Logf("[messenger] %s cleared and was not open here: nothing to say", key)
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
old.Cleared = now
|
|
pending := old.Pending
|
|
old.Pending = ""
|
|
sent := len(old.Sent) > 0
|
|
folded := old.Folded
|
|
h.mu.Unlock()
|
|
switch {
|
|
case old.silenced(now):
|
|
// A silenced condition sends nothing, its clearing included.
|
|
case sent:
|
|
h.edit(old, "cleared")
|
|
case folded:
|
|
// Held by the cap and never sent on its own: its clearing is said like any message.
|
|
h.deliver(old, "cleared")
|
|
case pending != "":
|
|
h.Logf("[messenger] %s cleared before it could be sent: nothing to unsay", key)
|
|
}
|
|
h.persist(old)
|
|
}
|
|
|
|
// Tick does what time asks: reminders, retries, the folded message, and forgetting what cleared
|
|
// long enough ago that a new raise is a new message.
|
|
func (h *Holder) Tick() {
|
|
h.work.Lock()
|
|
defer h.work.Unlock()
|
|
h.mu.Lock()
|
|
h.init()
|
|
now := h.Now()
|
|
var recs []*Record
|
|
for _, r := range h.open {
|
|
recs = append(recs, r)
|
|
}
|
|
h.mu.Unlock()
|
|
sort.Slice(recs, func(i, j int) bool { return recs[i].Raised.Before(recs[j].Raised) })
|
|
for _, r := range recs {
|
|
switch {
|
|
case !r.Cleared.IsZero():
|
|
if now.Sub(r.Cleared) >= ReopenWindow {
|
|
h.mu.Lock()
|
|
delete(h.open, r.Key)
|
|
h.mu.Unlock()
|
|
if h.Store != nil {
|
|
if err := h.Store.Delete(r.Key); err != nil {
|
|
h.noteStore(err)
|
|
}
|
|
}
|
|
}
|
|
case r.silenced(now):
|
|
case r.Pending != "":
|
|
h.deliver(r, r.Pending)
|
|
h.persist(r)
|
|
case !r.Reminded && !r.FirstSent.IsZero() && now.Sub(r.Raised) >= remindAfter(r.Severity):
|
|
h.mu.Lock()
|
|
r.Reminded = true
|
|
h.mu.Unlock()
|
|
h.deliver(r, "reminder")
|
|
h.persist(r)
|
|
}
|
|
}
|
|
h.flushFolds(now)
|
|
}
|
|
|
|
func remindAfter(severity string) time.Duration {
|
|
if severity == Urgent {
|
|
return RemindUrgent
|
|
}
|
|
return RemindWarning
|
|
}
|
|
|
|
// compose is the message for a record. withhold names what the content rule refused, so the words
|
|
// that carried it are not sent.
|
|
func compose(r *Record, what string, now time.Time, withhold int) Message {
|
|
sev := strings.ToUpper(r.Severity)
|
|
if sev == "" {
|
|
sev = "WARNING"
|
|
}
|
|
summary := r.Summary
|
|
if withhold > 0 {
|
|
summary = "channel-refused: this message carried " + r.Refused + ", so its words are withheld"
|
|
}
|
|
var title string
|
|
switch what {
|
|
case "raised":
|
|
title = sev + ": " + summary
|
|
case "reminder":
|
|
title = "STILL OPEN after " + roughly(now.Sub(r.Raised)) + ": " + summary
|
|
case "escalated":
|
|
title = "NOW URGENT: " + summary
|
|
case "reopened":
|
|
title = sev + " (open again, " + fmt.Sprint(r.Count) + " times): " + summary
|
|
case "cleared":
|
|
title = "CLEARED after " + roughly(r.Cleared.Sub(r.Raised)) + ": " + summary
|
|
default:
|
|
title = sev + ": " + summary
|
|
}
|
|
lines := []string{}
|
|
if withhold < 3 && r.Subject != "" {
|
|
about := "about: " + r.Subject
|
|
if r.Kind != "" {
|
|
about += " (" + r.Kind + ")"
|
|
}
|
|
lines = append(lines, about)
|
|
}
|
|
lines = append(lines, "since: "+r.Raised.UTC().Format("2006-01-02 15:04")+" UTC")
|
|
if withhold < 2 {
|
|
lines = append(lines, "key: "+r.Key)
|
|
if r.More != "" {
|
|
lines = append(lines, "more: "+r.More)
|
|
}
|
|
} else {
|
|
lines = append(lines, "more: conditions (the open ones, through the mesh)")
|
|
}
|
|
return Message{Title: title, Body: strings.Join(lines, "\n"), Urgent: r.Severity == Urgent, Quiet: what == "cleared"}
|
|
}
|
|
|
|
// say composes a record's message under the content rule: refused, it is composed again with less of
|
|
// it, until what remains may leave. The refusal is recorded and stated once per record.
|
|
func (h *Holder) say(r *Record, what string) Message {
|
|
now := h.Now()
|
|
if r.Refused != "" {
|
|
// Refused before: its words stay withheld in every later message too.
|
|
for w := 1; w <= 3; w++ {
|
|
m := compose(r, what, now, w)
|
|
if _, ok := Check(m.Text()); ok {
|
|
return m
|
|
}
|
|
}
|
|
}
|
|
m := compose(r, what, now, 0)
|
|
refusal, ok := Check(m.Text())
|
|
if ok {
|
|
return m
|
|
}
|
|
h.mu.Lock()
|
|
r.Refused = refusal.What
|
|
h.mu.Unlock()
|
|
key := r.Key
|
|
if _, keyOK := Check(key); !keyOK {
|
|
key = "(withheld)"
|
|
}
|
|
h.mu.Lock()
|
|
h.refusals = append(h.refusals, RefusalNote{At: now, Key: key, Class: refusal.Class, What: refusal.What})
|
|
if len(h.refusals) > KeptRefusals {
|
|
h.refusals = h.refusals[len(h.refusals)-KeptRefusals:]
|
|
}
|
|
h.mu.Unlock()
|
|
h.Logf("[messenger] refused the message for %s: it carried %s; sending channel-refused with its words withheld", key, refusal)
|
|
if h.Emit != nil {
|
|
if err := h.Emit("refused", map[string]any{"key": key, "class": refusal.Class, "what": refusal.What}); err != nil {
|
|
h.Logf("[messenger] could not state the refusal on the bus: %v", err)
|
|
}
|
|
}
|
|
for w := 1; w <= 3; w++ {
|
|
m := compose(r, what, now, w)
|
|
if _, ok := Check(m.Text()); ok {
|
|
return m
|
|
}
|
|
}
|
|
return Message{Title: "WARNING: channel-refused: a message carried " + refusal.What + " and was withheld",
|
|
Body: "more: conditions (the open ones, through the mesh)", Urgent: r.Severity == Urgent}
|
|
}
|
|
|
|
// deliver sends a record's message where its severity routes it.
|
|
func (h *Holder) deliver(r *Record, what string) {
|
|
m := h.say(r, what)
|
|
now := h.Now()
|
|
delivered := false
|
|
if r.Severity == Urgent {
|
|
for _, ch := range h.channels() {
|
|
if h.sendOn(ch, r, what, m) {
|
|
delivered = true
|
|
}
|
|
}
|
|
} else {
|
|
if h.Desktop != nil && h.Desktop.Ready() == nil {
|
|
delivered = h.sendOn(h.Desktop, r, what, m)
|
|
}
|
|
if !delivered && h.Telegram != nil {
|
|
delivered = h.sendOn(h.Telegram, r, what, m)
|
|
}
|
|
}
|
|
h.mu.Lock()
|
|
if delivered {
|
|
r.Pending = ""
|
|
if r.FirstSent.IsZero() && (what == "raised" || what == "escalated") {
|
|
r.FirstSent = now
|
|
}
|
|
} else {
|
|
// Nothing took it: said in the status and the log, tried again next minute.
|
|
r.Pending = what
|
|
}
|
|
h.mu.Unlock()
|
|
if !delivered {
|
|
h.Logf("[messenger] could not send %s %s on any channel; trying again every minute: %s", what, r.Key, h.whyNot())
|
|
}
|
|
}
|
|
|
|
func (h *Holder) channels() []Channel {
|
|
var out []Channel
|
|
for _, c := range []Channel{h.Telegram, h.Desktop} {
|
|
if c != nil {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// sendOn sends one message on one channel under its cap; held by the cap, it is folded, which counts
|
|
// as delivered — the fold will say it.
|
|
func (h *Holder) sendOn(ch Channel, r *Record, what string, m Message) bool {
|
|
if err := ch.Ready(); err != nil {
|
|
h.noteChannel(ch.Name(), err)
|
|
return false
|
|
}
|
|
if !h.allow(ch.Name()) {
|
|
h.foldOn(ch.Name(), r, what)
|
|
return true
|
|
}
|
|
id, err := ch.Send(m)
|
|
h.noteChannel(ch.Name(), err)
|
|
if err != nil {
|
|
h.record(Sent{At: h.Now(), Channel: ch.Name(), Key: r.Key, What: what, Outcome: "failed: " + err.Error()})
|
|
return false
|
|
}
|
|
h.mu.Lock()
|
|
if r.Sent == nil {
|
|
r.Sent = map[string]string{}
|
|
}
|
|
if _, has := r.Sent[ch.Name()]; !has && what != "cleared" {
|
|
r.Sent[ch.Name()] = id
|
|
}
|
|
h.mu.Unlock()
|
|
h.record(Sent{At: h.Now(), Channel: ch.Name(), Key: r.Key, What: what, Outcome: "sent"})
|
|
return true
|
|
}
|
|
|
|
// edit changes the first message on each channel that showed it; a channel that cannot, or whose
|
|
// edit fails, is sent a new message instead.
|
|
func (h *Holder) edit(r *Record, what string) {
|
|
m := h.say(r, what)
|
|
h.mu.Lock()
|
|
sent := map[string]string{}
|
|
for k, v := range r.Sent {
|
|
sent[k] = v
|
|
}
|
|
h.mu.Unlock()
|
|
for _, ch := range h.channels() {
|
|
id, shown := sent[ch.Name()]
|
|
if !shown {
|
|
continue
|
|
}
|
|
if ch.CanEdit() {
|
|
err := ch.Edit(id, m)
|
|
h.noteChannel(ch.Name(), err)
|
|
if err == nil {
|
|
h.record(Sent{At: h.Now(), Channel: ch.Name(), Key: r.Key, What: what, Outcome: "edited"})
|
|
continue
|
|
}
|
|
h.Logf("[messenger] could not edit the message for %s on %s (%v); sending a new one", r.Key, ch.Name(), err)
|
|
}
|
|
h.sendOn(ch, r, what, m)
|
|
}
|
|
}
|
|
|
|
func (h *Holder) foldOn(channel string, r *Record, what string) {
|
|
h.mu.Lock()
|
|
r.Folded = true
|
|
list := h.folds[channel]
|
|
replaced := false
|
|
for i := range list {
|
|
if list[i].Key == r.Key {
|
|
list[i].What, list[i].Severity, replaced = what, r.Severity, true
|
|
}
|
|
}
|
|
if !replaced {
|
|
list = append(list, foldEntry{Key: r.Key, Severity: r.Severity, What: what})
|
|
}
|
|
h.folds[channel] = list
|
|
first := len(list) == 1 && !replaced
|
|
h.mu.Unlock()
|
|
if first {
|
|
h.Logf("[messenger] %s is at its cap of %d messages an hour: holding the rest, to be folded into one", channel, CapPerHour)
|
|
}
|
|
h.record(Sent{At: h.Now(), Channel: channel, Key: r.Key, What: what, Outcome: "folded"})
|
|
}
|
|
|
|
func (h *Holder) flushFolds(now time.Time) {
|
|
for _, ch := range h.channels() {
|
|
h.mu.Lock()
|
|
list := append([]foldEntry(nil), h.folds[ch.Name()]...)
|
|
last := h.lastFold[ch.Name()]
|
|
h.mu.Unlock()
|
|
if len(list) == 0 || now.Sub(last) < FoldEvery {
|
|
continue
|
|
}
|
|
urgent := false
|
|
lines := []string{}
|
|
for i, e := range list {
|
|
if i == 40 {
|
|
lines = append(lines, fmt.Sprintf("and %d more", len(list)-40))
|
|
break
|
|
}
|
|
key := e.Key
|
|
if _, ok := Check(key); !ok {
|
|
key = "(a key withheld)"
|
|
}
|
|
lines = append(lines, e.Severity+" "+e.What+": "+key)
|
|
urgent = urgent || e.Severity == Urgent
|
|
}
|
|
m := Message{
|
|
Title: fmt.Sprintf("HELD BACK: %d message(s) over the cap of %d an hour", len(list), CapPerHour),
|
|
Body: strings.Join(lines, "\n") + "\nmore: conditions (through the mesh)",
|
|
Urgent: urgent,
|
|
}
|
|
if ch.Ready() != nil {
|
|
continue
|
|
}
|
|
_, err := ch.Send(m)
|
|
h.noteChannel(ch.Name(), err)
|
|
if err != nil {
|
|
h.record(Sent{At: now, Channel: ch.Name(), Key: "(folded)", What: "fold", Outcome: "failed: " + err.Error()})
|
|
continue
|
|
}
|
|
h.mu.Lock()
|
|
h.folds[ch.Name()] = h.folds[ch.Name()][len(list):]
|
|
h.lastFold[ch.Name()] = now
|
|
h.mu.Unlock()
|
|
h.record(Sent{At: now, Channel: ch.Name(), Key: "(folded)", What: fmt.Sprintf("fold of %d", len(list)), Outcome: "sent"})
|
|
}
|
|
}
|
|
|
|
// allow says whether a channel is under its cap: sends in the last hour, as recorded.
|
|
func (h *Holder) allow(channel string) bool {
|
|
return h.sentLastHour(channel) < CapPerHour
|
|
}
|
|
|
|
func (h *Holder) sentLastHour(channel string) int {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
since := h.Now().Add(-time.Hour)
|
|
n := 0
|
|
for _, s := range h.recent {
|
|
if s.Channel == channel && s.Outcome == "sent" && s.At.After(since) && s.Key != "(folded)" {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// sayOnce sends a message of the holder's own at most once per interval, on every channel ready.
|
|
func (h *Holder) sayOnce(key string, every time.Duration, m Message) {
|
|
now := h.Now()
|
|
h.mu.Lock()
|
|
if last, said := h.saidOnce[key]; said && now.Sub(last) < every {
|
|
h.mu.Unlock()
|
|
return
|
|
}
|
|
h.saidOnce[key] = now
|
|
h.mu.Unlock()
|
|
r := &Record{Key: key, Severity: Warning, Raised: now, Sent: map[string]string{}}
|
|
for _, ch := range h.channels() {
|
|
if ch.Ready() != nil {
|
|
continue
|
|
}
|
|
if h.sendOn(ch, r, "notice", m) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *Holder) record(s Sent) {
|
|
h.mu.Lock()
|
|
h.recent = append(h.recent, s)
|
|
if len(h.recent) > KeptSends {
|
|
h.recent = h.recent[len(h.recent)-KeptSends:]
|
|
}
|
|
recent := append([]Sent(nil), h.recent...)
|
|
h.mu.Unlock()
|
|
if h.Store != nil {
|
|
if err := h.Store.PutRecent(recent); err != nil {
|
|
h.noteStore(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *Holder) persist(r *Record) {
|
|
if h.Store == nil {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
c := *r
|
|
h.mu.Unlock()
|
|
if err := h.Store.Put(c); err != nil {
|
|
h.noteStore(err)
|
|
}
|
|
}
|
|
|
|
func (h *Holder) noteStore(err error) {
|
|
h.mu.Lock()
|
|
first := h.storeErr == ""
|
|
h.storeErr = err.Error()
|
|
h.mu.Unlock()
|
|
if first {
|
|
h.Logf("[messenger] cannot write its state (open messages are held in memory only until it can): %v", err)
|
|
}
|
|
}
|
|
|
|
func (h *Holder) noteChannel(name string, err error) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
now := h.Now()
|
|
if err == nil {
|
|
if h.chanErr[name] != "" {
|
|
h.Logf("[messenger] %s sends again", name)
|
|
}
|
|
h.chanErr[name] = ""
|
|
h.chanOK[name] = now
|
|
return
|
|
}
|
|
// Said in the log when it changes, and at most every ten minutes while it holds.
|
|
if h.chanErr[name] != err.Error() || now.Sub(h.chanErrAt[name]) >= 10*time.Minute {
|
|
h.Logf("[messenger] %s cannot send: %v", name, err)
|
|
h.chanErrAt[name] = now
|
|
}
|
|
h.chanErr[name] = err.Error()
|
|
}
|
|
|
|
func (h *Holder) whyNot() string {
|
|
var parts []string
|
|
for _, ch := range h.channels() {
|
|
if err := ch.Ready(); err != nil {
|
|
parts = append(parts, ch.Name()+": "+err.Error())
|
|
continue
|
|
}
|
|
h.mu.Lock()
|
|
e := h.chanErr[ch.Name()]
|
|
h.mu.Unlock()
|
|
if e != "" {
|
|
parts = append(parts, ch.Name()+": "+e)
|
|
}
|
|
}
|
|
if len(parts) == 0 {
|
|
return "no channel"
|
|
}
|
|
return strings.Join(parts, "; ")
|
|
}
|
|
|
|
func roughly(d time.Duration) string {
|
|
switch {
|
|
case d < 0:
|
|
return "a moment"
|
|
case d < 2*time.Minute:
|
|
return fmt.Sprintf("%d s", int(d.Seconds()))
|
|
case d < 2*time.Hour:
|
|
return fmt.Sprintf("%d min", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%.1f h", d.Hours())
|
|
}
|
|
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
|
}
|