Tell the operator what the mesh finds wrong, and watch the watcher (hq to-be 45 phase 1)
The mesh noticed 48 core failures in six days and told nobody (ADR 0227). messenger holds the operator-channel seat: it consumes the controller's condition events and sends them to Telegram and the desktop notifier, deduplicated by key, reminded once, edited on clear, capped at 20 an hour with the rest folded, and refusing anything carrying an address, a path or a secret. mesh-watcher, on a machine other than the control node, sends to Telegram directly when the self-check heartbeat or the bus goes silent.
This commit is contained in:
jochen
@@ -0,0 +1,262 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ChannelStatus is one channel as the status says it.
|
||||
type ChannelStatus struct {
|
||||
Name string `json:"name"`
|
||||
CanSend bool `json:"can_send"`
|
||||
NotReady string `json:"not_ready,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
LastSent string `json:"last_sent,omitempty"`
|
||||
SentLastHour int `json:"sent_last_hour"`
|
||||
Cap int `json:"cap_per_hour"`
|
||||
Held int `json:"held_by_cap"`
|
||||
}
|
||||
|
||||
// Status is the holder's whole account of itself.
|
||||
type Status struct {
|
||||
Verdict string `json:"verdict"`
|
||||
Channels []ChannelStatus `json:"channels"`
|
||||
Open int `json:"open"`
|
||||
Silenced int `json:"silenced"`
|
||||
Unsent []string `json:"unsent,omitempty"`
|
||||
Refused int `json:"refused_since_start"`
|
||||
Unreadable int `json:"unreadable_events_since_start"`
|
||||
LastBad string `json:"last_unreadable,omitempty"`
|
||||
Heard map[string]int `json:"events_heard_since_start"`
|
||||
LastHeard string `json:"last_event_heard,omitempty"`
|
||||
Listening string `json:"listening"`
|
||||
StateProblem string `json:"state_problem,omitempty"`
|
||||
Rules []string `json:"rules"`
|
||||
}
|
||||
|
||||
// Status answers at once from what the holder keeps.
|
||||
func (h *Holder) Status(listening string) Status {
|
||||
now := h.Now()
|
||||
var st Status
|
||||
for _, ch := range h.channels() {
|
||||
cs := ChannelStatus{Name: ch.Name(), Cap: CapPerHour, SentLastHour: h.sentLastHour(ch.Name())}
|
||||
if err := ch.Ready(); err != nil {
|
||||
cs.NotReady = err.Error()
|
||||
} else {
|
||||
cs.CanSend = true
|
||||
}
|
||||
h.mu.Lock()
|
||||
cs.LastError = h.chanErr[ch.Name()]
|
||||
if t, ok := h.chanOK[ch.Name()]; ok {
|
||||
cs.LastSent = t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
cs.Held = len(h.folds[ch.Name()])
|
||||
h.mu.Unlock()
|
||||
if cs.LastError != "" {
|
||||
cs.CanSend = false
|
||||
}
|
||||
st.Channels = append(st.Channels, cs)
|
||||
}
|
||||
h.mu.Lock()
|
||||
for _, r := range h.open {
|
||||
if !r.Cleared.IsZero() {
|
||||
continue
|
||||
}
|
||||
st.Open++
|
||||
if r.silenced(now) {
|
||||
st.Silenced++
|
||||
}
|
||||
if r.Pending != "" {
|
||||
st.Unsent = append(st.Unsent, r.Key+" ("+r.Pending+")")
|
||||
}
|
||||
}
|
||||
st.Refused = len(h.refusals)
|
||||
st.Unreadable = h.unreadable
|
||||
st.LastBad = h.lastBad
|
||||
st.Heard = map[string]int{}
|
||||
for k, v := range h.heard {
|
||||
st.Heard[k] = v
|
||||
}
|
||||
if !h.lastHeard.IsZero() {
|
||||
st.LastHeard = h.lastHeard.UTC().Format(time.RFC3339)
|
||||
}
|
||||
st.StateProblem = h.storeErr
|
||||
h.mu.Unlock()
|
||||
sort.Strings(st.Unsent)
|
||||
st.Listening = listening
|
||||
st.Rules = []string{
|
||||
"urgent: Telegram and the desktop; warning: the desktop where a session answers, otherwise Telegram",
|
||||
"deduplicated by the condition's key; reminded once after 1 h (urgent) or 12 h (warning); cleared by editing the first message",
|
||||
"at most 20 messages an hour per channel; the rest folded into one message, at most every 10 min",
|
||||
"refused: anything carrying an address, a path or a secret; channel-refused is sent with the words withheld",
|
||||
"silenced conditions send nothing; silence is `conditions silence` through the mesh",
|
||||
}
|
||||
anyCan := false
|
||||
for _, c := range st.Channels {
|
||||
anyCan = anyCan || c.CanSend
|
||||
}
|
||||
switch {
|
||||
case !anyCan:
|
||||
st.Verdict = "CANNOT SEND: no channel can reach the operator — see channels"
|
||||
case len(st.Unsent) > 0:
|
||||
st.Verdict = "BEHIND: some messages could not be sent yet and are tried every minute"
|
||||
case listening != "listening":
|
||||
st.Verdict = "NOT LISTENING: condition events are not reaching this holder — " + listening
|
||||
case st.StateProblem != "":
|
||||
st.Verdict = "STATE: open messages are held in memory only — " + st.StateProblem
|
||||
default:
|
||||
st.Verdict = "ok"
|
||||
for _, c := range st.Channels {
|
||||
if !c.CanSend {
|
||||
st.Verdict = "ok on one channel: " + c.Name + " cannot send"
|
||||
}
|
||||
}
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// OpenMessage is one open message as `open` answers it.
|
||||
type OpenMessage struct {
|
||||
Key string `json:"key"`
|
||||
Severity string `json:"severity"`
|
||||
Summary string `json:"summary"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Since string `json:"since"`
|
||||
Origin string `json:"origin"`
|
||||
SentOn []string `json:"sent_on,omitempty"`
|
||||
Silenced string `json:"silenced_until,omitempty"`
|
||||
Reminded bool `json:"reminded"`
|
||||
Unsent string `json:"unsent,omitempty"`
|
||||
Held bool `json:"held_by_cap,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Count int `json:"times_opened"`
|
||||
}
|
||||
|
||||
// Open is what is unresolved now, urgent first, oldest first.
|
||||
func (h *Holder) Open() []OpenMessage {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
var out []OpenMessage
|
||||
for _, r := range h.open {
|
||||
if !r.Cleared.IsZero() {
|
||||
continue
|
||||
}
|
||||
o := OpenMessage{Key: r.Key, Severity: r.Severity, Summary: r.Summary, Subject: r.Subject,
|
||||
Since: r.Raised.UTC().Format(time.RFC3339), Origin: r.Origin, Reminded: r.Reminded,
|
||||
Unsent: r.Pending, Held: r.Folded, Refused: r.Refused, Count: r.Count}
|
||||
for ch := range r.Sent {
|
||||
o.SentOn = append(o.SentOn, ch)
|
||||
}
|
||||
sort.Strings(o.SentOn)
|
||||
if r.silenced(now) {
|
||||
o.Silenced = r.SilencedTill.UTC().Format(time.RFC3339)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Severity != out[j].Severity {
|
||||
return out[i].Severity == Urgent
|
||||
}
|
||||
return out[i].Since < out[j].Since
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// History is what went out, newest first, and the refusals.
|
||||
func (h *Holder) History(limit int) map[string]any {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
sends := []Sent{}
|
||||
for i := len(h.recent) - 1; i >= 0 && len(sends) < limit; i-- {
|
||||
sends = append(sends, h.recent[i])
|
||||
}
|
||||
refusals := []RefusalNote{}
|
||||
for i := len(h.refusals) - 1; i >= 0 && len(refusals) < limit; i-- {
|
||||
refusals = append(refusals, h.refusals[i])
|
||||
}
|
||||
return map[string]any{"sends": sends, "refusals": refusals}
|
||||
}
|
||||
|
||||
// Test sends a test message on one channel or both, through the content rule and the cap.
|
||||
func (h *Holder) Test(channel, text string) map[string]string {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
if text == "" {
|
||||
text = "a test from the operator-channel's holder: this channel reaches you"
|
||||
}
|
||||
out := map[string]string{}
|
||||
if refusal, ok := Check(text); !ok {
|
||||
out["refused"] = "it carried " + refusal.String() + "; nothing was sent"
|
||||
return out
|
||||
}
|
||||
r := &Record{Key: "messenger.test", Severity: Warning, Raised: h.Now(), Sent: map[string]string{}}
|
||||
m := Message{Title: "TEST: " + text, Body: "nothing is wrong; this was asked for"}
|
||||
for _, ch := range h.channels() {
|
||||
if channel != "" && channel != "both" && channel != ch.Name() {
|
||||
continue
|
||||
}
|
||||
if err := ch.Ready(); err != nil {
|
||||
out[ch.Name()] = "not sent: " + err.Error()
|
||||
continue
|
||||
}
|
||||
if !h.allow(ch.Name()) {
|
||||
out[ch.Name()] = "not sent: the channel is at its cap of 20 an hour"
|
||||
continue
|
||||
}
|
||||
if h.sendOn(ch, r, "test", m) {
|
||||
out[ch.Name()] = "sent"
|
||||
} else {
|
||||
h.mu.Lock()
|
||||
out[ch.Name()] = "failed: " + h.chanErr[ch.Name()]
|
||||
h.mu.Unlock()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Notify takes a message from a module that uses the seat (research 028 Q5, modules as sources):
|
||||
// the same shape, rule, cap and deduplication as a condition. Its key is put under `notify.` so it
|
||||
// can never be taken for one of the controller's conditions.
|
||||
func (h *Holder) Notify(key, severity, summary, subject string, clear bool) (map[string]any, error) {
|
||||
if key == "" {
|
||||
return nil, errorf("key is required: the same key is the same message")
|
||||
}
|
||||
if len(key) < 7 || key[:7] != "notify." {
|
||||
key = "notify." + key
|
||||
}
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
h.mu.Unlock()
|
||||
if clear {
|
||||
h.cleared(key)
|
||||
return map[string]any{"key": key, "cleared": true}, nil
|
||||
}
|
||||
switch severity {
|
||||
case Urgent, Warning:
|
||||
case "":
|
||||
return nil, errorf("severity is required: urgent or warning")
|
||||
default:
|
||||
return nil, errorf("severity %q is neither urgent nor warning", severity)
|
||||
}
|
||||
if summary == "" {
|
||||
return nil, errorf("summary is required: one line in the mesh's words")
|
||||
}
|
||||
h.raised(Record{Key: key, Kind: "notice", Subject: subject, Severity: severity, Summary: summary,
|
||||
Origin: "notify", More: "operator-channel.open"})
|
||||
h.mu.Lock()
|
||||
r := h.open[key]
|
||||
var sent []string
|
||||
refused, pending := "", ""
|
||||
if r != nil {
|
||||
for ch := range r.Sent {
|
||||
sent = append(sent, ch)
|
||||
}
|
||||
refused, pending = r.Refused, r.Pending
|
||||
}
|
||||
h.mu.Unlock()
|
||||
sort.Strings(sent)
|
||||
return map[string]any{"key": key, "sent_on": sent, "refused": refused, "unsent": pending}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user