Tell the operator what the mesh finds wrong, and watch the watcher (hq to-be 45 phase 1)
The mesh noticed 48 core failures in six days and told nobody (ADR 0227). messenger holds the operator-channel seat: it consumes the controller's condition events and sends them to Telegram and the desktop notifier, deduplicated by key, reminded once, edited on clear, capped at 20 an hour with the rest folded, and refusing anything carrying an address, a path or a secret. mesh-watcher, on a machine other than the control node, sends to Telegram directly when the self-check heartbeat or the bus goes silent.
This commit is contained in:
jochen
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const goodToken = "123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsawQ"
|
||||
|
||||
func tokenFile(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "telegram-token")
|
||||
if content != "" {
|
||||
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestTelegramSaysWhatItLacks(t *testing.T) {
|
||||
for _, c := range []struct{ token, chat, says string }{
|
||||
{"", "42", "not on this machine yet"},
|
||||
{"r4nd0mlyMadeByTheMeshBecauseNobodyAcceptedOne", "42", "not a bot token"},
|
||||
{goodToken, "", "telegram-chat-id is not given"},
|
||||
{goodToken, "not a chat", "is not a chat id"},
|
||||
} {
|
||||
tg := NewTelegram(TelegramConfig{TokenFile: tokenFile(t, c.token), ChatID: func() string { return c.chat }})
|
||||
err := tg.Ready()
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%+v: %v", c, err)
|
||||
}
|
||||
if err != nil && strings.Contains(err.Error(), goodToken) {
|
||||
t.Errorf("the token is in the words")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramSendsEditsAndNeverSaysItsToken(t *testing.T) {
|
||||
var asked []string
|
||||
var bodies []map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
asked = append(asked, r.URL.Path)
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
var b map[string]any
|
||||
_ = json.Unmarshal(raw, &b)
|
||||
bodies = append(bodies, b)
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/sendMessage"):
|
||||
_, _ = w.Write([]byte(`{"ok":true,"result":{"message_id":77}}`))
|
||||
case strings.HasSuffix(r.URL.Path, "/editMessageText"):
|
||||
_, _ = w.Write([]byte(`{"ok":false,"error_code":400,"description":"Bad Request: message to edit not found"}`))
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"ok":true,"result":{"username":"mesh_bot"}}`))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
old := TelegramAPI
|
||||
TelegramAPI = srv.URL
|
||||
defer func() { TelegramAPI = old }()
|
||||
tg := NewTelegram(TelegramConfig{TokenFile: tokenFile(t, goodToken+"\n"), ChatID: func() string { return "-1001" }})
|
||||
id, err := tg.Send(Message{Title: "URGENT: x", Body: "key: a.b.c"})
|
||||
if err != nil || id != "77" {
|
||||
t.Fatalf("%q %v", id, err)
|
||||
}
|
||||
if asked[0] != "/bot"+goodToken+"/sendMessage" || bodies[0]["chat_id"] != "-1001" || bodies[0]["text"] != "URGENT: x\nkey: a.b.c" {
|
||||
t.Fatalf("%v %v", asked, bodies[0])
|
||||
}
|
||||
err = tg.Edit("77", Message{Title: "CLEARED"})
|
||||
if err == nil || !strings.Contains(err.Error(), "message to edit not found") || strings.Contains(err.Error(), goodToken) {
|
||||
t.Fatalf("edit: %v", err)
|
||||
}
|
||||
if bodies[1]["message_id"] != float64(77) {
|
||||
t.Fatalf("message id: %v", bodies[1]["message_id"])
|
||||
}
|
||||
if who, err := tg.Who(); err != nil || who != "mesh_bot" {
|
||||
t.Fatalf("%q %v", who, err)
|
||||
}
|
||||
// Nothing answers: the error is in words, without the URL that carries the token.
|
||||
srv.Close()
|
||||
_, err = tg.Send(Message{Title: "x"})
|
||||
if err == nil || strings.Contains(err.Error(), goodToken) || strings.Contains(err.Error(), "/bot") {
|
||||
t.Fatalf("unreachable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheDesktopAsksTheNotifierSeatOnEachMachine(t *testing.T) {
|
||||
var asked []string
|
||||
d := &Desktop{
|
||||
Machines: func() []string { return []string{"one", "two"} },
|
||||
Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
args := body.(map[string]any)
|
||||
if args["body"] != "a <b>" || args["urgency"] != "critical" {
|
||||
t.Errorf("args: %v", args)
|
||||
}
|
||||
if strings.HasSuffix(key, "@two") {
|
||||
return nil, errors.New("the account is not logged in")
|
||||
}
|
||||
return json.RawMessage(`{"id":12}`), nil
|
||||
},
|
||||
}
|
||||
id, err := d.Send(Message{Title: "t", Body: "a <b>", Urgent: true})
|
||||
if err != nil || id != "one=12" {
|
||||
t.Fatalf("%q %v", id, err)
|
||||
}
|
||||
if asked[0] != "seat:node-notifier.send@one" || asked[1] != "seat:node-notifier.send@two" {
|
||||
t.Fatalf("%v", asked)
|
||||
}
|
||||
if d.LastAnswers()["two"] == "" || d.LastAnswers()["one"] != "" {
|
||||
t.Fatalf("%v", d.LastAnswers())
|
||||
}
|
||||
asked = nil
|
||||
if err := d.Edit("one=12", Message{Title: "t", Body: "a <b>", Urgent: true}); err != nil || len(asked) != 1 {
|
||||
t.Fatalf("edit asked %v: %v", asked, err)
|
||||
}
|
||||
none := &Desktop{Machines: func() []string { return nil }}
|
||||
if err := none.Ready(); err == nil || !strings.Contains(err.Error(), "desktop-machines") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNotifiersAnswerIsReadInEitherShape(t *testing.T) {
|
||||
for raw, want := range map[string]int{
|
||||
`{"id":5}`: 5,
|
||||
`{"content":[{"type":"text","text":"{\"id\":6}"}]}`: 6,
|
||||
`"{\"id\":7}"`: 7,
|
||||
} {
|
||||
if got, err := notificationID(json.RawMessage(raw)); err != nil || got != want {
|
||||
t.Errorf("%s: %d %v", raw, got, err)
|
||||
}
|
||||
}
|
||||
if _, err := notificationID(json.RawMessage(`{"content":[{"text":"no session"}],"isError":true}`)); err == nil {
|
||||
t.Errorf("an error answer read as an id")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSettingsAreReadAsTheMeshMergesThem(t *testing.T) {
|
||||
p := filepath.Join(t.TempDir(), "settings.json")
|
||||
_ = os.WriteFile(p, []byte(`{"telegram-chat-id": 123456, "desktop-machines": ["a", " ", "b"]}`), 0o600)
|
||||
s, err := readSettings(p)
|
||||
if err != nil || s.TelegramChatID != "123456" || len(s.DesktopMachines) != 2 {
|
||||
t.Fatalf("%+v %v", s, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
package main
|
||||
|
||||
// The controller's condition events, read in this one place (novox/hq to-be 45 §2).
|
||||
//
|
||||
// **The contract this reads, and every assumption it makes about it**, because the controller's
|
||||
// side was built at the same time from the same design and the design names the events and the
|
||||
// fields but not their spelling on the wire:
|
||||
//
|
||||
// - The events are the mesh-controller seat's own: `condition-raised`, `condition-changed` and
|
||||
// `condition-cleared`, consumed as `mesh-controller.<event>` (subject
|
||||
// `mesh.seat.mesh-controller.event.<event>`), exactly as `applied` and `built-before` are.
|
||||
// - The body is the condition as the store holds it, one JSON object, with the field names of the
|
||||
// design's table in kebab-case: key, kind, subject, severity, summary, evidence, source, raised,
|
||||
// last-observed, observations, tried, resolver, silenced, epoch. snake_case and camelCase
|
||||
// spellings of the two-word names are read too.
|
||||
// - `subject` is an object {scope, id, machine}; a plain string is read as well.
|
||||
// - `severity` is `urgent` or `warning`. Anything else is not guessed: the event is unreadable.
|
||||
// - `silenced` is absent, null, or an object {until, by, why}; `until` in the past is not silenced.
|
||||
// - Times are RFC 3339.
|
||||
// - `key` is `<scope>.<id>.<kind>`. When it is absent it is made from subject and kind; when
|
||||
// neither gives one the event is unreadable.
|
||||
// - A cleared event carries the condition as last held, and may add `cleared` (its time).
|
||||
//
|
||||
// An event this cannot read is refused by name — which event, which field, why — counted, said in
|
||||
// the status and in the log, and told to the operator; never read as an empty condition.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The events, by their local names under the controller's seat.
|
||||
const (
|
||||
EventRaised = "condition-raised"
|
||||
EventChanged = "condition-changed"
|
||||
EventCleared = "condition-cleared"
|
||||
// ControllerSeat is the role the events are stated under.
|
||||
ControllerSeat = "mesh-controller"
|
||||
)
|
||||
|
||||
// Severities: two levels, no more (to-be 45 §2).
|
||||
const (
|
||||
Urgent = "urgent"
|
||||
Warning = "warning"
|
||||
)
|
||||
|
||||
// Condition is what this holder needs of one.
|
||||
type Condition struct {
|
||||
Key string
|
||||
Kind string
|
||||
Scope string
|
||||
ID string
|
||||
Machine string
|
||||
Severity string
|
||||
Summary string
|
||||
Source string
|
||||
Resolver string
|
||||
Raised time.Time
|
||||
LastObserved time.Time
|
||||
Observations int
|
||||
SilencedTill time.Time
|
||||
SilencedWhy string
|
||||
Cleared time.Time
|
||||
}
|
||||
|
||||
// SubjectWords is the subject as a person reads it: "machine ace", "plan 41", "provider keycloak".
|
||||
func (c Condition) SubjectWords() string {
|
||||
parts := []string{}
|
||||
if c.Scope != "" {
|
||||
parts = append(parts, c.Scope)
|
||||
}
|
||||
if c.ID != "" {
|
||||
parts = append(parts, c.ID)
|
||||
}
|
||||
if c.Machine != "" && c.Machine != c.ID {
|
||||
parts = append(parts, "on "+c.Machine)
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// SilencedAt says whether the condition's messages are stopped at that moment.
|
||||
func (c Condition) SilencedAt(now time.Time) bool {
|
||||
return !c.SilencedTill.IsZero() && now.Before(c.SilencedTill)
|
||||
}
|
||||
|
||||
// eventOf is the local event name of an envelope's key: `mesh-controller.condition-raised` is
|
||||
// `condition-raised`. Anything not of the controller's seat is not a condition event.
|
||||
func eventOf(key string) (string, bool) {
|
||||
emitter, event, ok := strings.Cut(key, ".")
|
||||
if !ok || emitter != ControllerSeat {
|
||||
return "", false
|
||||
}
|
||||
switch event {
|
||||
case EventRaised, EventChanged, EventCleared:
|
||||
return event, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// DecodeCondition reads one condition event's body.
|
||||
func DecodeCondition(event string, body []byte) (Condition, error) {
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return Condition{}, fmt.Errorf("%s: the body is not a JSON object: %v", event, err)
|
||||
}
|
||||
if raw == nil {
|
||||
return Condition{}, fmt.Errorf("%s: the body is null", event)
|
||||
}
|
||||
var c Condition
|
||||
var err error
|
||||
str := func(names ...string) string {
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, n := range names {
|
||||
v, ok := raw[n]
|
||||
if !ok || string(v) == "null" {
|
||||
continue
|
||||
}
|
||||
var s string
|
||||
if e := json.Unmarshal(v, &s); e != nil {
|
||||
err = fmt.Errorf("%s: %s is not a string", event, n)
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
when := func(names ...string) time.Time {
|
||||
s := str(names...)
|
||||
if s == "" || err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
t, e := time.Parse(time.RFC3339Nano, s)
|
||||
if e != nil {
|
||||
err = fmt.Errorf("%s: %s is not an RFC 3339 time: %q", event, names[0], s)
|
||||
}
|
||||
return t
|
||||
}
|
||||
c.Key = str("key")
|
||||
c.Kind = str("kind")
|
||||
c.Severity = str("severity")
|
||||
c.Summary = str("summary")
|
||||
c.Resolver = str("resolver")
|
||||
c.Raised = when("raised")
|
||||
c.LastObserved = when("last-observed", "last_observed", "lastObserved")
|
||||
c.Cleared = when("cleared")
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
// source: a string, or an object naming the row, probe or event.
|
||||
if v, ok := raw["source"]; ok && string(v) != "null" {
|
||||
var s string
|
||||
if json.Unmarshal(v, &s) == nil {
|
||||
c.Source = s
|
||||
} else {
|
||||
var o map[string]any
|
||||
if json.Unmarshal(v, &o) == nil {
|
||||
for _, k := range []string{"row", "probe", "event", "name", "id"} {
|
||||
if s, ok := o[k].(string); ok && s != "" {
|
||||
c.Source = s
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"observations", "count"} {
|
||||
if v, ok := raw[n]; ok && string(v) != "null" {
|
||||
var f float64
|
||||
if json.Unmarshal(v, &f) != nil {
|
||||
return Condition{}, fmt.Errorf("%s: %s is not a number", event, n)
|
||||
}
|
||||
c.Observations = int(f)
|
||||
break
|
||||
}
|
||||
}
|
||||
if v, ok := raw["subject"]; ok && string(v) != "null" {
|
||||
var s string
|
||||
if json.Unmarshal(v, &s) == nil {
|
||||
c.ID = strings.TrimSpace(s)
|
||||
} else {
|
||||
var o struct {
|
||||
Scope string `json:"scope"`
|
||||
ID string `json:"id"`
|
||||
Machine string `json:"machine"`
|
||||
Node string `json:"node"`
|
||||
}
|
||||
if e := json.Unmarshal(v, &o); e != nil {
|
||||
return Condition{}, fmt.Errorf("%s: subject is neither a string nor {scope, id, machine}", event)
|
||||
}
|
||||
c.Scope, c.ID, c.Machine = o.Scope, o.ID, o.Machine
|
||||
if c.Machine == "" {
|
||||
c.Machine = o.Node
|
||||
}
|
||||
}
|
||||
}
|
||||
if v, ok := raw["silenced"]; ok && string(v) != "null" && string(v) != "{}" && string(v) != `""` && string(v) != "false" {
|
||||
var o struct {
|
||||
Until string `json:"until"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
if e := json.Unmarshal(v, &o); e != nil {
|
||||
return Condition{}, fmt.Errorf("%s: silenced is not {until, by, why}", event)
|
||||
}
|
||||
if o.Until != "" {
|
||||
t, e := time.Parse(time.RFC3339Nano, o.Until)
|
||||
if e != nil {
|
||||
return Condition{}, fmt.Errorf("%s: silenced.until is not an RFC 3339 time: %q", event, o.Until)
|
||||
}
|
||||
c.SilencedTill = t
|
||||
}
|
||||
c.SilencedWhy = o.Why
|
||||
}
|
||||
if c.Key == "" && c.Scope != "" && c.ID != "" && c.Kind != "" {
|
||||
c.Key = c.Scope + "." + c.ID + "." + c.Kind
|
||||
}
|
||||
if c.Key == "" {
|
||||
return Condition{}, fmt.Errorf("%s: no key, and no subject and kind to make one from", event)
|
||||
}
|
||||
if c.Scope == "" || c.ID == "" || c.Kind == "" {
|
||||
// The key names them (`<scope>.<id>.<kind>`, the id itself possibly dotted).
|
||||
parts := strings.Split(c.Key, ".")
|
||||
if len(parts) >= 3 {
|
||||
if c.Scope == "" {
|
||||
c.Scope = parts[0]
|
||||
}
|
||||
if c.Kind == "" {
|
||||
c.Kind = parts[len(parts)-1]
|
||||
}
|
||||
if c.ID == "" {
|
||||
c.ID = strings.Join(parts[1:len(parts)-1], ".")
|
||||
}
|
||||
}
|
||||
}
|
||||
switch c.Severity {
|
||||
case Urgent, Warning:
|
||||
case "":
|
||||
if event != EventCleared {
|
||||
return Condition{}, fmt.Errorf("%s %s: no severity", event, c.Key)
|
||||
}
|
||||
default:
|
||||
return Condition{}, fmt.Errorf("%s %s: severity %q is neither urgent nor warning", event, c.Key, c.Severity)
|
||||
}
|
||||
if c.Summary == "" && event != EventCleared {
|
||||
return Condition{}, fmt.Errorf("%s %s: no summary", event, c.Key)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The contract with the controller's condition events (to-be 45 §2), as condition.go states it.
|
||||
|
||||
func TestTheEventsAreTheControllersSeat(t *testing.T) {
|
||||
for key, want := range map[string]string{
|
||||
"mesh-controller.condition-raised": EventRaised,
|
||||
"mesh-controller.condition-changed": EventChanged,
|
||||
"mesh-controller.condition-cleared": EventCleared,
|
||||
} {
|
||||
if got, ok := eventOf(key); !ok || got != want {
|
||||
t.Errorf("%s: %q %v", key, got, ok)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"mesh-controller.applied", "gitea.condition-raised", "condition-raised"} {
|
||||
if _, ok := eventOf(key); ok {
|
||||
t.Errorf("%s read as a condition event", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConditionAsTheStoreHoldsIt(t *testing.T) {
|
||||
body := `{
|
||||
"key": "machine.ace.silent", "kind": "silent",
|
||||
"subject": {"scope": "machine", "id": "ace", "machine": "ace"},
|
||||
"severity": "urgent", "summary": "the home server has not been heard for 15 min",
|
||||
"evidence": [{"at": "2026-10-06T12:00:00Z", "what": "last heartbeat"}],
|
||||
"source": "S1", "raised": "2026-10-06T12:15:00Z", "last-observed": "2026-10-06T12:16:00Z",
|
||||
"observations": 3, "tried": [], "resolver": "self",
|
||||
"silenced": {"until": "2026-10-06T14:00:00Z", "by": "operator", "why": "moving it"},
|
||||
"epoch": 57
|
||||
}`
|
||||
c, err := DecodeCondition(EventRaised, []byte(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Key != "machine.ace.silent" || c.Kind != "silent" || c.Scope != "machine" || c.ID != "ace" ||
|
||||
c.Severity != Urgent || c.Source != "S1" || c.Observations != 3 || c.Resolver != "self" {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
if !c.Raised.Equal(time.Date(2026, 10, 6, 12, 15, 0, 0, time.UTC)) || c.LastObserved.IsZero() {
|
||||
t.Fatalf("times: %+v", c)
|
||||
}
|
||||
if !c.SilencedAt(time.Date(2026, 10, 6, 13, 0, 0, 0, time.UTC)) || c.SilencedAt(time.Date(2026, 10, 6, 15, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("silenced: %v", c.SilencedTill)
|
||||
}
|
||||
if c.SubjectWords() != "machine ace" {
|
||||
t.Fatalf("subject words: %q", c.SubjectWords())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOtherSpellingsAndAKeyMadeFromItsParts(t *testing.T) {
|
||||
c, err := DecodeCondition(EventChanged, []byte(`{"kind":"stalled","subject":{"scope":"plan","id":"41"},
|
||||
"severity":"warning","summary":"plan 41 waits","last_observed":"2026-10-06T12:00:00Z","silenced":null,
|
||||
"source":{"row":"S3"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Key != "plan.41.stalled" || c.Source != "S3" || c.LastObserved.IsZero() || !c.SilencedTill.IsZero() {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
c, err = DecodeCondition(EventRaised, []byte(`{"key":"provider.keycloak.ace.gitea.failing","subject":"keycloak",
|
||||
"severity":"warning","summary":"the identity provider fails gitea"}`))
|
||||
if err != nil || c.Scope != "provider" || c.Kind != "failing" || c.ID != "keycloak" {
|
||||
t.Fatalf("%+v %v", c, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAClearedEventNeedsOnlyItsKey(t *testing.T) {
|
||||
c, err := DecodeCondition(EventCleared, []byte(`{"key":"machine.ace.silent","cleared":"2026-10-06T12:30:00Z"}`))
|
||||
if err != nil || c.Key != "machine.ace.silent" || c.Cleared.IsZero() {
|
||||
t.Fatalf("%+v %v", c, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatCannotBeReadIsRefusedByName(t *testing.T) {
|
||||
for body, says := range map[string]string{
|
||||
`not json`: "not a JSON object",
|
||||
`null`: "null",
|
||||
`{"severity":"urgent","summary":"x"}`: "no key",
|
||||
`{"key":"a.b.c","summary":"x"}`: "no severity",
|
||||
`{"key":"a.b.c","severity":"critical","summary":"x"}`: "neither urgent nor warning",
|
||||
`{"key":"a.b.c","severity":"urgent"}`: "no summary",
|
||||
`{"key":"a.b.c","severity":"urgent","summary":"x","raised":"yesterday"}`: "not an RFC 3339 time",
|
||||
`{"key":7}`: "key is not a string",
|
||||
`{"key":"a.b.c","severity":"urgent","summary":"x","silenced":{"until":"soon"}}`: "silenced.until",
|
||||
} {
|
||||
_, err := DecodeCondition(EventRaised, []byte(body))
|
||||
if err == nil || !strings.Contains(err.Error(), says) {
|
||||
t.Errorf("%s: %v, want %q", body, err, says)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
// What may leave the mesh (novox/hq to-be 45 §5, research 028 Q8, ADR 0227): roles and words. A
|
||||
// message carrying an address, a path or anything shaped like a secret is refused here, by the holder,
|
||||
// because Telegram is not end-to-end encrypted for bots and this rule is the only thing between a
|
||||
// condition's words and someone else's server. It is not trusted to each source.
|
||||
//
|
||||
// Deliberately wider than it must be: a commit id or a long random name is refused too. A source
|
||||
// that wants its message through says it in words; a refusal is said, never silent.
|
||||
|
||||
import (
|
||||
"math"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Refusal says why a text may not leave: the class of what it carried, never the text itself.
|
||||
type Refusal struct {
|
||||
Class string // address, path or secret
|
||||
What string // a few words: "an IPv4 address", "a URL", …
|
||||
}
|
||||
|
||||
func (r Refusal) String() string { return r.Class + " (" + r.What + ")" }
|
||||
|
||||
var (
|
||||
reURL = regexp.MustCompile(`(?i)\b[a-z][a-z0-9+.-]*://`)
|
||||
reEmail = regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}`)
|
||||
reIPv4 = regexp.MustCompile(`\b\d{1,3}(\.\d{1,3}){3}\b`)
|
||||
reIPv6 = regexp.MustCompile(`(?i)(^|[^0-9a-z:])(([0-9a-f]{1,4}:){4,7}[0-9a-f]{1,4}|([0-9a-f]{1,4}:)*[0-9a-f]{0,4}::([0-9a-f]{1,4}:)*[0-9a-f]{0,4})([^0-9a-z:]|$)`)
|
||||
reMAC = regexp.MustCompile(`(?i)\b([0-9a-f]{2}[:-]){5}[0-9a-f]{2}\b`)
|
||||
rePEM = regexp.MustCompile(`-----BEGIN [A-Z ]+-----`)
|
||||
reJWT = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`)
|
||||
reBotToken = regexp.MustCompile(`\b\d{6,}:[A-Za-z0-9_-]{30,}`)
|
||||
reKnown = regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{16,}|sk-[A-Za-z0-9_-]{16,}|xox[abprs]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16})`)
|
||||
reAssigned = regexp.MustCompile(`(?i)\b(password|passwd|passphrase|secret|token|api[_-]?key|apikey|credential|private[_-]?key)\s*[=:]\s*\S`)
|
||||
reHex = regexp.MustCompile(`(?i)\b[0-9a-f]{32,}\b`)
|
||||
reRun = regexp.MustCompile(`[A-Za-z0-9+/=_]{20,}`)
|
||||
reWinPath = regexp.MustCompile(`(?i)\b[a-z]:\\`)
|
||||
)
|
||||
|
||||
// topLevel are names that end a host name — the generic and country ones a mesh's names use, and the
|
||||
// private ones (.internal, .lan, .home, .local) a mesh is likelier to.
|
||||
var topLevel = map[string]bool{}
|
||||
|
||||
func init() {
|
||||
for _, t := range strings.Fields(`com net org edu gov mil int io dev app cloud ai co me info biz xyz
|
||||
site online tech page link
|
||||
be nl de fr uk lu eu ch at it es pt se no dk fi pl cz us ca au nz jp cn ru in br ie
|
||||
internal lan home local localdomain corp intranet private arpa test example invalid localhost`) {
|
||||
topLevel[t] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Check says whether a text may leave the mesh, and when not, why.
|
||||
func Check(text string) (Refusal, bool) {
|
||||
switch {
|
||||
case reURL.MatchString(text):
|
||||
return Refusal{"address", "a URL"}, false
|
||||
case reEmail.MatchString(text):
|
||||
return Refusal{"address", "a mail address"}, false
|
||||
case reIPv4.MatchString(text):
|
||||
return Refusal{"address", "an IPv4 address"}, false
|
||||
case reMAC.MatchString(text):
|
||||
return Refusal{"address", "a hardware address"}, false
|
||||
case reIPv6.MatchString(text):
|
||||
return Refusal{"address", "an IPv6 address"}, false
|
||||
case rePEM.MatchString(text):
|
||||
return Refusal{"secret", "a key block"}, false
|
||||
case reJWT.MatchString(text):
|
||||
return Refusal{"secret", "a signed token"}, false
|
||||
case reBotToken.MatchString(text):
|
||||
return Refusal{"secret", "a bot token"}, false
|
||||
case reKnown.MatchString(text):
|
||||
return Refusal{"secret", "a known token shape"}, false
|
||||
case reAssigned.MatchString(text):
|
||||
return Refusal{"secret", "a value given to a secret's name"}, false
|
||||
case reHex.MatchString(text):
|
||||
return Refusal{"secret", "a long hexadecimal string"}, false
|
||||
case reWinPath.MatchString(text):
|
||||
return Refusal{"path", "a drive path"}, false
|
||||
}
|
||||
for _, run := range reRun.FindAllString(text, -1) {
|
||||
if looksRandom(run) {
|
||||
return Refusal{"secret", "a long random-looking string"}, false
|
||||
}
|
||||
}
|
||||
for _, word := range strings.FieldsFunc(text, func(r rune) bool {
|
||||
return unicode.IsSpace(r) || strings.ContainsRune("\"'`()[]{}<>,;|", r)
|
||||
}) {
|
||||
w := strings.TrimRight(word, ".:!?")
|
||||
if isPath(w) {
|
||||
return Refusal{"path", "a file path"}, false
|
||||
}
|
||||
if isHostName(w) {
|
||||
return Refusal{"address", "a host name"}, false
|
||||
}
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
// isPath: absolute, home-relative or dot-relative, or two separators deep. A mesh address names one
|
||||
// machine and one tool (`ace/postgres.query`) and has one; a ratio ("3/4") has digits only.
|
||||
func isPath(w string) bool {
|
||||
if w == "" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(w, "/") && len(w) > 1 {
|
||||
return true
|
||||
}
|
||||
for _, p := range []string{"~/", "./", "../", "$HOME", "${"} {
|
||||
if strings.HasPrefix(w, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return strings.Count(w, "/") >= 2 || strings.Contains(w, "\\")
|
||||
}
|
||||
|
||||
// isHostName: two names or more, the last a top-level one. A condition key's last name is its kind
|
||||
// (`machine.ace.silent`), which none of these is.
|
||||
func isHostName(w string) bool {
|
||||
w = strings.ToLower(w)
|
||||
if w == "localhost" {
|
||||
return true
|
||||
}
|
||||
parts := strings.Split(w, ".")
|
||||
if len(parts) < 2 {
|
||||
return false
|
||||
}
|
||||
for _, p := range parts {
|
||||
if p == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return topLevel[parts[len(parts)-1]]
|
||||
}
|
||||
|
||||
// looksRandom: letters and digits mixed, and the characters spread as a random string's are. A
|
||||
// sentence's words are separated, so only an unbroken run reaches here.
|
||||
func looksRandom(s string) bool {
|
||||
var letters, digits int
|
||||
counts := map[rune]int{}
|
||||
for _, r := range s {
|
||||
counts[r]++
|
||||
switch {
|
||||
case unicode.IsLetter(r):
|
||||
letters++
|
||||
case unicode.IsDigit(r):
|
||||
digits++
|
||||
}
|
||||
}
|
||||
if letters == 0 || digits == 0 {
|
||||
// One class only: a word, or a number. A long number of digits alone is a count or a time.
|
||||
return letters > 0 && hasUpperAndLower(s) && entropy(counts, len(s)) >= 4.0
|
||||
}
|
||||
return entropy(counts, len(s)) >= 3.3
|
||||
}
|
||||
|
||||
func hasUpperAndLower(s string) bool {
|
||||
return strings.IndexFunc(s, unicode.IsUpper) >= 0 && strings.IndexFunc(s, unicode.IsLower) >= 0
|
||||
}
|
||||
|
||||
func entropy(counts map[rune]int, n int) float64 {
|
||||
e := 0.0
|
||||
for _, c := range counts {
|
||||
p := float64(c) / float64(n)
|
||||
e -= p * math.Log2(p)
|
||||
}
|
||||
return e
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRolesAndWordsMayLeave(t *testing.T) {
|
||||
for _, s := range []string{
|
||||
"URGENT: the home server has not been heard for 15 min",
|
||||
"key: provider.keycloak.ace.gitea.failing",
|
||||
"more: conditions show machine.ace.silent",
|
||||
"since: 2026-10-06 12:30 UTC",
|
||||
"call call-1759752000123456789-12 has run past its bound of 10 min",
|
||||
"ace/postgres.query answers no more",
|
||||
"3/4 machines answered; plan 41 waits on build 7f3a9c1e",
|
||||
"the bus advisory slow-consumer for ace_records",
|
||||
"core.controller.novox.rolled-back",
|
||||
"seat node-notifier has no live holder",
|
||||
"HELD BACK: 5 message(s) over the cap of 20 an hour",
|
||||
"STILL OPEN after 1.5 h: the controller's event loop takes no message",
|
||||
"node-engine and node tools builds differ from the plan's",
|
||||
} {
|
||||
if r, ok := Check(s); !ok {
|
||||
t.Errorf("refused %q: %s", s, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAddressAPathOrASecretMayNot(t *testing.T) {
|
||||
for s, class := range map[string]string{
|
||||
"cannot reach 192.168.1.20": "address",
|
||||
"listening on 10.0.0.7:5432": "address",
|
||||
"route fd00:1234:5678::1 is gone": "address",
|
||||
"fe80::1 answered": "address",
|
||||
"2001:db8:0:0:0:0:2:1 answered": "address",
|
||||
"see https://git.example.org/x": "address",
|
||||
"git.novox.internal does not answer": "address",
|
||||
"the mail for admin@example.org bounced": "address",
|
||||
"zurag.be is down": "address",
|
||||
"localhost refused": "address",
|
||||
"the card aa:bb:cc:dd:ee:ff went away": "address",
|
||||
"/var/lib/mesh-controller is full": "path",
|
||||
"~/.config/hal/env changed": "path",
|
||||
"read ./grants.json": "path",
|
||||
"services/postgres/data/pg_hba.conf": "path",
|
||||
"C:\\Users\\x": "path",
|
||||
"token 123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw": "secret",
|
||||
"ghp_abcdefghijklmnopqrstuvwxyz0123456789": "secret",
|
||||
"password=hunter2": "secret",
|
||||
"api_key: x": "secret",
|
||||
"-----BEGIN OPENSSH PRIVATE KEY-----": "secret",
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0": "secret",
|
||||
"commit 3f2a9c1e5b7d4f6a8c0e2b4d6f8a0c2e4b6d8f0a": "secret",
|
||||
"value Zx8Qp2Lm9Rt4Vb7Nc1Kd5Hs3Jf6": "secret",
|
||||
} {
|
||||
r, ok := Check(s)
|
||||
if ok {
|
||||
t.Errorf("let %q leave", s)
|
||||
continue
|
||||
}
|
||||
if r.Class != class {
|
||||
t.Errorf("%q refused as %s, want %s", s, r, class)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package main
|
||||
|
||||
// The desktop notifier (novox/hq to-be 45 §5, ADR 0208): the `node-notifier` seat's `send` verb on
|
||||
// the machine the operator is at, asked through the mesh. Nothing new runs on that machine — the
|
||||
// seat's holder (dunst) already answers `send` in the operator's session, and answers with an error
|
||||
// when no session is there. That error is what "the operator's session is there" is read from until
|
||||
// presence is decided (research 028 Q4): a machine whose notifier answers has a session.
|
||||
//
|
||||
// Which machines to try is the setting desktop-machines, in order. None given: the channel is not
|
||||
// configured, the status says so, and a warning goes to Telegram.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Asker calls a tool through the mesh, as stdio.Ask does.
|
||||
type Asker func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
// Desktop sends to the notifier of each configured machine.
|
||||
type Desktop struct {
|
||||
Machines func() []string
|
||||
Ask Asker
|
||||
|
||||
mu sync.Mutex
|
||||
last map[string]string // machine -> the last error, or "" after a success
|
||||
}
|
||||
|
||||
func (d *Desktop) Name() string { return "desktop" }
|
||||
func (d *Desktop) CanEdit() bool { return true }
|
||||
|
||||
func (d *Desktop) Ready() error {
|
||||
if len(d.Machines()) == 0 {
|
||||
return errors.New("no machine to show it on: the setting desktop-machines is not given " +
|
||||
"(`settings` for messenger with {\"desktop-machines\": [\"<the machine the operator sits at>\"]})")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Send shows the message on every configured machine that has a session, and answers
|
||||
// `<machine>=<id>` for each one that took it. It fails only when none did.
|
||||
func (d *Desktop) Send(m Message) (string, error) {
|
||||
return d.show(m, nil)
|
||||
}
|
||||
|
||||
// Edit replaces the notification shown before on each machine that showed it.
|
||||
func (d *Desktop) Edit(id string, m Message) error {
|
||||
shown := map[string]int{}
|
||||
for _, part := range strings.Split(id, ",") {
|
||||
machine, n, ok := strings.Cut(part, "=")
|
||||
var i int
|
||||
if ok {
|
||||
if _, err := fmt.Sscan(n, &i); err == nil {
|
||||
shown[machine] = i
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(shown) == 0 {
|
||||
return errors.New("no notification on record to replace")
|
||||
}
|
||||
_, err := d.show(m, shown)
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *Desktop) show(m Message, replace map[string]int) (string, error) {
|
||||
if err := d.Ready(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
urgency := "normal"
|
||||
switch {
|
||||
case m.Quiet:
|
||||
urgency = "low"
|
||||
case m.Urgent:
|
||||
urgency = "critical"
|
||||
}
|
||||
var took []string
|
||||
var failed []string
|
||||
for _, machine := range d.Machines() {
|
||||
if replace != nil {
|
||||
if _, shown := replace[machine]; !shown {
|
||||
continue
|
||||
}
|
||||
}
|
||||
args := map[string]any{
|
||||
"summary": m.Title,
|
||||
"body": html.EscapeString(m.Body),
|
||||
"urgency": urgency,
|
||||
"app_name": "mesh",
|
||||
}
|
||||
if replace != nil {
|
||||
args["replace_id"] = replace[machine]
|
||||
}
|
||||
raw, err := d.Ask("seat:node-notifier.send@"+machine, args)
|
||||
if err == nil {
|
||||
var id int
|
||||
id, err = notificationID(raw)
|
||||
if err == nil {
|
||||
took = append(took, fmt.Sprintf("%s=%d", machine, id))
|
||||
}
|
||||
}
|
||||
d.note(machine, err)
|
||||
if err != nil {
|
||||
failed = append(failed, machine+": "+err.Error())
|
||||
}
|
||||
}
|
||||
if len(took) == 0 {
|
||||
return "", errors.New("no machine showed it — " + strings.Join(failed, "; "))
|
||||
}
|
||||
return strings.Join(took, ","), nil
|
||||
}
|
||||
|
||||
func (d *Desktop) note(machine string, err error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.last == nil {
|
||||
d.last = map[string]string{}
|
||||
}
|
||||
if err != nil {
|
||||
d.last[machine] = err.Error()
|
||||
} else {
|
||||
d.last[machine] = ""
|
||||
}
|
||||
}
|
||||
|
||||
// Machines' last answers, for the status: "" is a machine that took the last message shown to it.
|
||||
func (d *Desktop) LastAnswers() map[string]string {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
out := map[string]string{}
|
||||
for k, v := range d.last {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// notificationID reads the notifier's answer, {"id": N}, whether the runtime hands it over bare or in
|
||||
// the tool reply's text content.
|
||||
func notificationID(raw json.RawMessage) (int, error) {
|
||||
var direct struct {
|
||||
ID *int `json:"id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &direct) == nil && direct.ID != nil {
|
||||
return *direct.ID, nil
|
||||
}
|
||||
var wrapped struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 {
|
||||
if wrapped.IsError {
|
||||
return 0, errors.New(wrapped.Content[0].Text)
|
||||
}
|
||||
if json.Unmarshal([]byte(wrapped.Content[0].Text), &direct) == nil && direct.ID != nil {
|
||||
return *direct.ID, nil
|
||||
}
|
||||
}
|
||||
var s string
|
||||
if json.Unmarshal(raw, &s) == nil && json.Unmarshal([]byte(s), &direct) == nil && direct.ID != nil {
|
||||
return *direct.ID, nil
|
||||
}
|
||||
return 0, fmt.Errorf("the notifier answered no id: %.80s", string(raw))
|
||||
}
|
||||
@@ -0,0 +1,775 @@
|
||||
package main
|
||||
|
||||
// The holder of the operator-channel seat (novox/hq to-be 45 §5, ADR 0227): what is sent, to whom,
|
||||
// when, and how often. The controller decides what is wrong; this decides what is said.
|
||||
//
|
||||
// - On `condition-raised`: one message, deduplicated by the condition's key. Said again with the
|
||||
// same key while open, it is the same message, not a second.
|
||||
// - Once more if still open after 1 hour (urgent) or 12 hours (warning).
|
||||
// - On `condition-cleared`: the first message is edited where the channel can (both can);
|
||||
// otherwise a new one says it. Cleared and raised again within ten minutes, it is the same
|
||||
// message, edited back to open — not a new one.
|
||||
// - A silenced condition sends nothing.
|
||||
// - Urgent to both channels; warning to the desktop when the operator's session is there,
|
||||
// otherwise to Telegram.
|
||||
// - At most twenty messages an hour per channel; the excess is held and folded into one message
|
||||
// naming them all, sent at most every ten minutes — the cap is said, never silent.
|
||||
// - A message carrying an address, a path or a secret is refused (content.go); what is sent in its
|
||||
// place says `channel-refused`, with the offending part withheld.
|
||||
// - A channel that cannot send says so in the status and the log, and the message is tried again
|
||||
// every minute while the condition is open.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
RemindUrgent = time.Hour
|
||||
RemindWarning = 12 * time.Hour
|
||||
ReopenWindow = 10 * time.Minute
|
||||
CapPerHour = 20
|
||||
FoldEvery = 10 * time.Minute
|
||||
KeptSends = 200
|
||||
KeptRefusals = 50
|
||||
)
|
||||
|
||||
// Message is what a channel shows: a title line and a body.
|
||||
type Message struct {
|
||||
Title string
|
||||
Body string
|
||||
Urgent bool
|
||||
Quiet bool // a clearing: shown without urgency
|
||||
}
|
||||
|
||||
func (m Message) Text() string {
|
||||
if m.Body == "" {
|
||||
return m.Title
|
||||
}
|
||||
return m.Title + "\n" + m.Body
|
||||
}
|
||||
|
||||
// Channel is one way to the operator.
|
||||
type Channel interface {
|
||||
Name() string
|
||||
Ready() error
|
||||
Send(Message) (string, error)
|
||||
Edit(id string, m Message) error
|
||||
CanEdit() bool
|
||||
}
|
||||
|
||||
// Record is one open message, kept in the module's own state so a restart forgets nothing.
|
||||
type Record struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Severity string `json:"severity"`
|
||||
Summary string `json:"summary"`
|
||||
Origin string `json:"origin"`
|
||||
More string `json:"more"`
|
||||
Raised time.Time `json:"raised"`
|
||||
SilencedTill time.Time `json:"silenced_till,omitempty"`
|
||||
Sent map[string]string `json:"sent,omitempty"` // channel -> the first message's id
|
||||
FirstSent time.Time `json:"first_sent,omitempty"`
|
||||
Reminded bool `json:"reminded,omitempty"`
|
||||
Pending string `json:"pending,omitempty"` // what is still to be said: raised, reminder, …
|
||||
Folded bool `json:"folded,omitempty"`
|
||||
Cleared time.Time `json:"cleared,omitempty"`
|
||||
Count int `json:"count"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
func (r *Record) silenced(now time.Time) bool {
|
||||
return !r.SilencedTill.IsZero() && now.Before(r.SilencedTill)
|
||||
}
|
||||
|
||||
// Sent is one message that went out, or was held, for the history.
|
||||
type Sent struct {
|
||||
At time.Time `json:"at"`
|
||||
Channel string `json:"channel"`
|
||||
Key string `json:"key"`
|
||||
What string `json:"what"`
|
||||
Outcome string `json:"outcome"` // sent, edited, folded, failed: <why>
|
||||
}
|
||||
|
||||
// RefusalNote is one refused message: never its text.
|
||||
type RefusalNote struct {
|
||||
At time.Time `json:"at"`
|
||||
Key string `json:"key"`
|
||||
Class string `json:"class"`
|
||||
What string `json:"what"`
|
||||
}
|
||||
|
||||
// Store is the module's own state (ADR 0201): the open messages, and the recent sends.
|
||||
type Store interface {
|
||||
Put(r Record) error
|
||||
Delete(key string) error
|
||||
All() ([]Record, error)
|
||||
PutRecent([]Sent) error
|
||||
Recent() ([]Sent, error)
|
||||
}
|
||||
|
||||
type foldEntry struct {
|
||||
Key, Severity, What string
|
||||
}
|
||||
|
||||
// Holder is the seat's holder.
|
||||
type Holder struct {
|
||||
Telegram Channel
|
||||
Desktop Channel
|
||||
Store Store
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
// Emit states a fact as this module (refused); nil states nothing.
|
||||
Emit func(event string, body any) error
|
||||
|
||||
work sync.Mutex // one event, call or tick at a time
|
||||
mu sync.Mutex // what the status reads
|
||||
|
||||
open map[string]*Record
|
||||
recent []Sent
|
||||
refusals []RefusalNote
|
||||
folds map[string][]foldEntry
|
||||
lastFold map[string]time.Time
|
||||
chanErr map[string]string
|
||||
chanErrAt map[string]time.Time
|
||||
chanOK map[string]time.Time
|
||||
unreadable int
|
||||
lastBad string
|
||||
lastBadAt time.Time
|
||||
saidOnce map[string]time.Time
|
||||
storeErr string
|
||||
heard map[string]int
|
||||
lastHeard time.Time
|
||||
}
|
||||
|
||||
func (h *Holder) init() {
|
||||
if h.open == nil {
|
||||
h.open = map[string]*Record{}
|
||||
h.folds = map[string][]foldEntry{}
|
||||
h.lastFold = map[string]time.Time{}
|
||||
h.chanErr = map[string]string{}
|
||||
h.chanErrAt = map[string]time.Time{}
|
||||
h.chanOK = map[string]time.Time{}
|
||||
h.saidOnce = map[string]time.Time{}
|
||||
h.heard = map[string]int{}
|
||||
}
|
||||
if h.Now == nil {
|
||||
h.Now = time.Now
|
||||
}
|
||||
if h.Logf == nil {
|
||||
h.Logf = func(string, ...any) {}
|
||||
}
|
||||
}
|
||||
|
||||
// Load reads back what was open and what was sent before a restart.
|
||||
func (h *Holder) Load() error {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if h.Store == nil {
|
||||
return nil
|
||||
}
|
||||
recs, err := h.Store.All()
|
||||
if err != nil {
|
||||
h.storeErr = err.Error()
|
||||
return err
|
||||
}
|
||||
for i := range recs {
|
||||
r := recs[i]
|
||||
h.open[r.Key] = &r
|
||||
}
|
||||
if sent, err := h.Store.Recent(); err == nil {
|
||||
h.recent = sent
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Condition takes one of the controller's condition events.
|
||||
func (h *Holder) Condition(event string, c Condition) {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
h.heard[event]++
|
||||
h.lastHeard = h.Now()
|
||||
h.mu.Unlock()
|
||||
rec := Record{
|
||||
Key: c.Key, Kind: c.Kind, Subject: c.SubjectWords(), Severity: c.Severity, Summary: c.Summary,
|
||||
Origin: "condition", More: "conditions show " + c.Key, Raised: c.Raised, SilencedTill: c.SilencedTill,
|
||||
}
|
||||
switch event {
|
||||
case EventRaised:
|
||||
h.raised(rec)
|
||||
case EventChanged:
|
||||
h.changed(rec)
|
||||
case EventCleared:
|
||||
h.cleared(rec.Key)
|
||||
}
|
||||
}
|
||||
|
||||
// Unreadable records an event that could not be read, and tells the operator — once an hour.
|
||||
func (h *Holder) Unreadable(key string, err error) {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
h.unreadable++
|
||||
h.lastBad = err.Error()
|
||||
h.lastBadAt = h.Now()
|
||||
n := h.unreadable
|
||||
h.mu.Unlock()
|
||||
h.Logf("[messenger] refused %s: %v (unreadable events since start: %d)", key, err, n)
|
||||
h.sayOnce("messenger.unreadable-event", time.Hour, Message{
|
||||
Title: "WARNING: a condition event could not be read",
|
||||
Body: fmt.Sprintf("the operator-channel's holder could not read %d condition event(s) from the controller; "+
|
||||
"what was wrong is in messenger_status. A condition may be open that was not said.", n),
|
||||
})
|
||||
}
|
||||
|
||||
func (h *Holder) raised(rec Record) {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
old := h.open[rec.Key]
|
||||
h.mu.Unlock()
|
||||
if old != nil && old.Cleared.IsZero() {
|
||||
// Said again while open: the same message. Its words are kept current; nothing is sent.
|
||||
h.mu.Lock()
|
||||
old.Summary, old.Subject, old.Kind, old.SilencedTill = rec.Summary, rec.Subject, rec.Kind, rec.SilencedTill
|
||||
if rec.Severity != "" {
|
||||
old.Severity = rec.Severity
|
||||
}
|
||||
h.mu.Unlock()
|
||||
h.persist(old)
|
||||
return
|
||||
}
|
||||
if old != nil && now.Sub(old.Cleared) < ReopenWindow {
|
||||
// Cleared and raised again within ten minutes: the same message, back to open (to-be 45 §2).
|
||||
h.mu.Lock()
|
||||
old.Cleared = time.Time{}
|
||||
old.Count++
|
||||
old.Summary, old.Severity, old.SilencedTill = rec.Summary, rec.Severity, rec.SilencedTill
|
||||
h.mu.Unlock()
|
||||
if !old.silenced(now) && len(old.Sent) > 0 {
|
||||
h.edit(old, "reopened")
|
||||
}
|
||||
h.persist(old)
|
||||
return
|
||||
}
|
||||
r := rec
|
||||
r.Count = 1
|
||||
if r.Raised.IsZero() {
|
||||
r.Raised = now
|
||||
}
|
||||
r.Sent = map[string]string{}
|
||||
h.mu.Lock()
|
||||
h.open[r.Key] = &r
|
||||
h.mu.Unlock()
|
||||
if r.silenced(now) {
|
||||
h.Logf("[messenger] %s raised while silenced until %s: nothing sent", r.Key, r.SilencedTill.Format(time.RFC3339))
|
||||
h.persist(&r)
|
||||
return
|
||||
}
|
||||
h.deliver(&r, "raised")
|
||||
h.persist(&r)
|
||||
}
|
||||
|
||||
func (h *Holder) changed(rec Record) {
|
||||
h.mu.Lock()
|
||||
old := h.open[rec.Key]
|
||||
h.mu.Unlock()
|
||||
if old == nil || !old.Cleared.IsZero() {
|
||||
// A change to something this holder never heard raised: read as a raise, so it is said.
|
||||
h.Logf("[messenger] %s changed and was not open here; taken as raised", rec.Key)
|
||||
h.raised(rec)
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
escalated := old.Severity == Warning && rec.Severity == Urgent
|
||||
old.Summary, old.Subject, old.SilencedTill = rec.Summary, rec.Subject, rec.SilencedTill
|
||||
if rec.Severity != "" {
|
||||
old.Severity = rec.Severity
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if escalated && !old.silenced(h.Now()) {
|
||||
// Routing differs for urgent: said once more, to both channels.
|
||||
h.deliver(old, "escalated")
|
||||
}
|
||||
h.persist(old)
|
||||
}
|
||||
|
||||
func (h *Holder) cleared(key string) {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
old := h.open[key]
|
||||
h.mu.Unlock()
|
||||
if old == nil || !old.Cleared.IsZero() {
|
||||
h.Logf("[messenger] %s cleared and was not open here: nothing to say", key)
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
old.Cleared = now
|
||||
pending := old.Pending
|
||||
old.Pending = ""
|
||||
sent := len(old.Sent) > 0
|
||||
folded := old.Folded
|
||||
h.mu.Unlock()
|
||||
switch {
|
||||
case old.silenced(now):
|
||||
// A silenced condition sends nothing, its clearing included.
|
||||
case sent:
|
||||
h.edit(old, "cleared")
|
||||
case folded:
|
||||
// Held by the cap and never sent on its own: its clearing is said like any message.
|
||||
h.deliver(old, "cleared")
|
||||
case pending != "":
|
||||
h.Logf("[messenger] %s cleared before it could be sent: nothing to unsay", key)
|
||||
}
|
||||
h.persist(old)
|
||||
}
|
||||
|
||||
// Tick does what time asks: reminders, retries, the folded message, and forgetting what cleared
|
||||
// long enough ago that a new raise is a new message.
|
||||
func (h *Holder) Tick() {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
now := h.Now()
|
||||
var recs []*Record
|
||||
for _, r := range h.open {
|
||||
recs = append(recs, r)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
sort.Slice(recs, func(i, j int) bool { return recs[i].Raised.Before(recs[j].Raised) })
|
||||
for _, r := range recs {
|
||||
switch {
|
||||
case !r.Cleared.IsZero():
|
||||
if now.Sub(r.Cleared) >= ReopenWindow {
|
||||
h.mu.Lock()
|
||||
delete(h.open, r.Key)
|
||||
h.mu.Unlock()
|
||||
if h.Store != nil {
|
||||
if err := h.Store.Delete(r.Key); err != nil {
|
||||
h.noteStore(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
case r.silenced(now):
|
||||
case r.Pending != "":
|
||||
h.deliver(r, r.Pending)
|
||||
h.persist(r)
|
||||
case !r.Reminded && !r.FirstSent.IsZero() && now.Sub(r.Raised) >= remindAfter(r.Severity):
|
||||
h.mu.Lock()
|
||||
r.Reminded = true
|
||||
h.mu.Unlock()
|
||||
h.deliver(r, "reminder")
|
||||
h.persist(r)
|
||||
}
|
||||
}
|
||||
h.flushFolds(now)
|
||||
}
|
||||
|
||||
func remindAfter(severity string) time.Duration {
|
||||
if severity == Urgent {
|
||||
return RemindUrgent
|
||||
}
|
||||
return RemindWarning
|
||||
}
|
||||
|
||||
// compose is the message for a record. withhold names what the content rule refused, so the words
|
||||
// that carried it are not sent.
|
||||
func compose(r *Record, what string, now time.Time, withhold int) Message {
|
||||
sev := strings.ToUpper(r.Severity)
|
||||
if sev == "" {
|
||||
sev = "WARNING"
|
||||
}
|
||||
summary := r.Summary
|
||||
if withhold > 0 {
|
||||
summary = "channel-refused: this message carried " + r.Refused + ", so its words are withheld"
|
||||
}
|
||||
var title string
|
||||
switch what {
|
||||
case "raised":
|
||||
title = sev + ": " + summary
|
||||
case "reminder":
|
||||
title = "STILL OPEN after " + roughly(now.Sub(r.Raised)) + ": " + summary
|
||||
case "escalated":
|
||||
title = "NOW URGENT: " + summary
|
||||
case "reopened":
|
||||
title = sev + " (open again, " + fmt.Sprint(r.Count) + " times): " + summary
|
||||
case "cleared":
|
||||
title = "CLEARED after " + roughly(r.Cleared.Sub(r.Raised)) + ": " + summary
|
||||
default:
|
||||
title = sev + ": " + summary
|
||||
}
|
||||
lines := []string{}
|
||||
if withhold < 3 && r.Subject != "" {
|
||||
about := "about: " + r.Subject
|
||||
if r.Kind != "" {
|
||||
about += " (" + r.Kind + ")"
|
||||
}
|
||||
lines = append(lines, about)
|
||||
}
|
||||
lines = append(lines, "since: "+r.Raised.UTC().Format("2006-01-02 15:04")+" UTC")
|
||||
if withhold < 2 {
|
||||
lines = append(lines, "key: "+r.Key)
|
||||
if r.More != "" {
|
||||
lines = append(lines, "more: "+r.More)
|
||||
}
|
||||
} else {
|
||||
lines = append(lines, "more: conditions (the open ones, through the mesh)")
|
||||
}
|
||||
return Message{Title: title, Body: strings.Join(lines, "\n"), Urgent: r.Severity == Urgent, Quiet: what == "cleared"}
|
||||
}
|
||||
|
||||
// say composes a record's message under the content rule: refused, it is composed again with less of
|
||||
// it, until what remains may leave. The refusal is recorded and stated once per record.
|
||||
func (h *Holder) say(r *Record, what string) Message {
|
||||
now := h.Now()
|
||||
if r.Refused != "" {
|
||||
// Refused before: its words stay withheld in every later message too.
|
||||
for w := 1; w <= 3; w++ {
|
||||
m := compose(r, what, now, w)
|
||||
if _, ok := Check(m.Text()); ok {
|
||||
return m
|
||||
}
|
||||
}
|
||||
}
|
||||
m := compose(r, what, now, 0)
|
||||
refusal, ok := Check(m.Text())
|
||||
if ok {
|
||||
return m
|
||||
}
|
||||
h.mu.Lock()
|
||||
r.Refused = refusal.What
|
||||
h.mu.Unlock()
|
||||
key := r.Key
|
||||
if _, keyOK := Check(key); !keyOK {
|
||||
key = "(withheld)"
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.refusals = append(h.refusals, RefusalNote{At: now, Key: key, Class: refusal.Class, What: refusal.What})
|
||||
if len(h.refusals) > KeptRefusals {
|
||||
h.refusals = h.refusals[len(h.refusals)-KeptRefusals:]
|
||||
}
|
||||
h.mu.Unlock()
|
||||
h.Logf("[messenger] refused the message for %s: it carried %s; sending channel-refused with its words withheld", key, refusal)
|
||||
if h.Emit != nil {
|
||||
if err := h.Emit("refused", map[string]any{"key": key, "class": refusal.Class, "what": refusal.What}); err != nil {
|
||||
h.Logf("[messenger] could not state the refusal on the bus: %v", err)
|
||||
}
|
||||
}
|
||||
for w := 1; w <= 3; w++ {
|
||||
m := compose(r, what, now, w)
|
||||
if _, ok := Check(m.Text()); ok {
|
||||
return m
|
||||
}
|
||||
}
|
||||
return Message{Title: "WARNING: channel-refused: a message carried " + refusal.What + " and was withheld",
|
||||
Body: "more: conditions (the open ones, through the mesh)", Urgent: r.Severity == Urgent}
|
||||
}
|
||||
|
||||
// deliver sends a record's message where its severity routes it.
|
||||
func (h *Holder) deliver(r *Record, what string) {
|
||||
m := h.say(r, what)
|
||||
now := h.Now()
|
||||
delivered := false
|
||||
if r.Severity == Urgent {
|
||||
for _, ch := range h.channels() {
|
||||
if h.sendOn(ch, r, what, m) {
|
||||
delivered = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if h.Desktop != nil && h.Desktop.Ready() == nil {
|
||||
delivered = h.sendOn(h.Desktop, r, what, m)
|
||||
}
|
||||
if !delivered && h.Telegram != nil {
|
||||
delivered = h.sendOn(h.Telegram, r, what, m)
|
||||
}
|
||||
}
|
||||
h.mu.Lock()
|
||||
if delivered {
|
||||
r.Pending = ""
|
||||
if r.FirstSent.IsZero() && (what == "raised" || what == "escalated") {
|
||||
r.FirstSent = now
|
||||
}
|
||||
} else {
|
||||
// Nothing took it: said in the status and the log, tried again next minute.
|
||||
r.Pending = what
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if !delivered {
|
||||
h.Logf("[messenger] could not send %s %s on any channel; trying again every minute: %s", what, r.Key, h.whyNot())
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) channels() []Channel {
|
||||
var out []Channel
|
||||
for _, c := range []Channel{h.Telegram, h.Desktop} {
|
||||
if c != nil {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sendOn sends one message on one channel under its cap; held by the cap, it is folded, which counts
|
||||
// as delivered — the fold will say it.
|
||||
func (h *Holder) sendOn(ch Channel, r *Record, what string, m Message) bool {
|
||||
if err := ch.Ready(); err != nil {
|
||||
h.noteChannel(ch.Name(), err)
|
||||
return false
|
||||
}
|
||||
if !h.allow(ch.Name()) {
|
||||
h.foldOn(ch.Name(), r, what)
|
||||
return true
|
||||
}
|
||||
id, err := ch.Send(m)
|
||||
h.noteChannel(ch.Name(), err)
|
||||
if err != nil {
|
||||
h.record(Sent{At: h.Now(), Channel: ch.Name(), Key: r.Key, What: what, Outcome: "failed: " + err.Error()})
|
||||
return false
|
||||
}
|
||||
h.mu.Lock()
|
||||
if r.Sent == nil {
|
||||
r.Sent = map[string]string{}
|
||||
}
|
||||
if _, has := r.Sent[ch.Name()]; !has && what != "cleared" {
|
||||
r.Sent[ch.Name()] = id
|
||||
}
|
||||
h.mu.Unlock()
|
||||
h.record(Sent{At: h.Now(), Channel: ch.Name(), Key: r.Key, What: what, Outcome: "sent"})
|
||||
return true
|
||||
}
|
||||
|
||||
// edit changes the first message on each channel that showed it; a channel that cannot, or whose
|
||||
// edit fails, is sent a new message instead.
|
||||
func (h *Holder) edit(r *Record, what string) {
|
||||
m := h.say(r, what)
|
||||
h.mu.Lock()
|
||||
sent := map[string]string{}
|
||||
for k, v := range r.Sent {
|
||||
sent[k] = v
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, ch := range h.channels() {
|
||||
id, shown := sent[ch.Name()]
|
||||
if !shown {
|
||||
continue
|
||||
}
|
||||
if ch.CanEdit() {
|
||||
err := ch.Edit(id, m)
|
||||
h.noteChannel(ch.Name(), err)
|
||||
if err == nil {
|
||||
h.record(Sent{At: h.Now(), Channel: ch.Name(), Key: r.Key, What: what, Outcome: "edited"})
|
||||
continue
|
||||
}
|
||||
h.Logf("[messenger] could not edit the message for %s on %s (%v); sending a new one", r.Key, ch.Name(), err)
|
||||
}
|
||||
h.sendOn(ch, r, what, m)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) foldOn(channel string, r *Record, what string) {
|
||||
h.mu.Lock()
|
||||
r.Folded = true
|
||||
list := h.folds[channel]
|
||||
replaced := false
|
||||
for i := range list {
|
||||
if list[i].Key == r.Key {
|
||||
list[i].What, list[i].Severity, replaced = what, r.Severity, true
|
||||
}
|
||||
}
|
||||
if !replaced {
|
||||
list = append(list, foldEntry{Key: r.Key, Severity: r.Severity, What: what})
|
||||
}
|
||||
h.folds[channel] = list
|
||||
first := len(list) == 1 && !replaced
|
||||
h.mu.Unlock()
|
||||
if first {
|
||||
h.Logf("[messenger] %s is at its cap of %d messages an hour: holding the rest, to be folded into one", channel, CapPerHour)
|
||||
}
|
||||
h.record(Sent{At: h.Now(), Channel: channel, Key: r.Key, What: what, Outcome: "folded"})
|
||||
}
|
||||
|
||||
func (h *Holder) flushFolds(now time.Time) {
|
||||
for _, ch := range h.channels() {
|
||||
h.mu.Lock()
|
||||
list := append([]foldEntry(nil), h.folds[ch.Name()]...)
|
||||
last := h.lastFold[ch.Name()]
|
||||
h.mu.Unlock()
|
||||
if len(list) == 0 || now.Sub(last) < FoldEvery {
|
||||
continue
|
||||
}
|
||||
urgent := false
|
||||
lines := []string{}
|
||||
for i, e := range list {
|
||||
if i == 40 {
|
||||
lines = append(lines, fmt.Sprintf("and %d more", len(list)-40))
|
||||
break
|
||||
}
|
||||
key := e.Key
|
||||
if _, ok := Check(key); !ok {
|
||||
key = "(a key withheld)"
|
||||
}
|
||||
lines = append(lines, e.Severity+" "+e.What+": "+key)
|
||||
urgent = urgent || e.Severity == Urgent
|
||||
}
|
||||
m := Message{
|
||||
Title: fmt.Sprintf("HELD BACK: %d message(s) over the cap of %d an hour", len(list), CapPerHour),
|
||||
Body: strings.Join(lines, "\n") + "\nmore: conditions (through the mesh)",
|
||||
Urgent: urgent,
|
||||
}
|
||||
if ch.Ready() != nil {
|
||||
continue
|
||||
}
|
||||
_, err := ch.Send(m)
|
||||
h.noteChannel(ch.Name(), err)
|
||||
if err != nil {
|
||||
h.record(Sent{At: now, Channel: ch.Name(), Key: "(folded)", What: "fold", Outcome: "failed: " + err.Error()})
|
||||
continue
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.folds[ch.Name()] = h.folds[ch.Name()][len(list):]
|
||||
h.lastFold[ch.Name()] = now
|
||||
h.mu.Unlock()
|
||||
h.record(Sent{At: now, Channel: ch.Name(), Key: "(folded)", What: fmt.Sprintf("fold of %d", len(list)), Outcome: "sent"})
|
||||
}
|
||||
}
|
||||
|
||||
// allow says whether a channel is under its cap: sends in the last hour, as recorded.
|
||||
func (h *Holder) allow(channel string) bool {
|
||||
return h.sentLastHour(channel) < CapPerHour
|
||||
}
|
||||
|
||||
func (h *Holder) sentLastHour(channel string) int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
since := h.Now().Add(-time.Hour)
|
||||
n := 0
|
||||
for _, s := range h.recent {
|
||||
if s.Channel == channel && s.Outcome == "sent" && s.At.After(since) && s.Key != "(folded)" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// sayOnce sends a message of the holder's own at most once per interval, on every channel ready.
|
||||
func (h *Holder) sayOnce(key string, every time.Duration, m Message) {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
if last, said := h.saidOnce[key]; said && now.Sub(last) < every {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.saidOnce[key] = now
|
||||
h.mu.Unlock()
|
||||
r := &Record{Key: key, Severity: Warning, Raised: now, Sent: map[string]string{}}
|
||||
for _, ch := range h.channels() {
|
||||
if ch.Ready() != nil {
|
||||
continue
|
||||
}
|
||||
if h.sendOn(ch, r, "notice", m) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) record(s Sent) {
|
||||
h.mu.Lock()
|
||||
h.recent = append(h.recent, s)
|
||||
if len(h.recent) > KeptSends {
|
||||
h.recent = h.recent[len(h.recent)-KeptSends:]
|
||||
}
|
||||
recent := append([]Sent(nil), h.recent...)
|
||||
h.mu.Unlock()
|
||||
if h.Store != nil {
|
||||
if err := h.Store.PutRecent(recent); err != nil {
|
||||
h.noteStore(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) persist(r *Record) {
|
||||
if h.Store == nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
c := *r
|
||||
h.mu.Unlock()
|
||||
if err := h.Store.Put(c); err != nil {
|
||||
h.noteStore(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) noteStore(err error) {
|
||||
h.mu.Lock()
|
||||
first := h.storeErr == ""
|
||||
h.storeErr = err.Error()
|
||||
h.mu.Unlock()
|
||||
if first {
|
||||
h.Logf("[messenger] cannot write its state (open messages are held in memory only until it can): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) noteChannel(name string, err error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
now := h.Now()
|
||||
if err == nil {
|
||||
if h.chanErr[name] != "" {
|
||||
h.Logf("[messenger] %s sends again", name)
|
||||
}
|
||||
h.chanErr[name] = ""
|
||||
h.chanOK[name] = now
|
||||
return
|
||||
}
|
||||
// Said in the log when it changes, and at most every ten minutes while it holds.
|
||||
if h.chanErr[name] != err.Error() || now.Sub(h.chanErrAt[name]) >= 10*time.Minute {
|
||||
h.Logf("[messenger] %s cannot send: %v", name, err)
|
||||
h.chanErrAt[name] = now
|
||||
}
|
||||
h.chanErr[name] = err.Error()
|
||||
}
|
||||
|
||||
func (h *Holder) whyNot() string {
|
||||
var parts []string
|
||||
for _, ch := range h.channels() {
|
||||
if err := ch.Ready(); err != nil {
|
||||
parts = append(parts, ch.Name()+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
h.mu.Lock()
|
||||
e := h.chanErr[ch.Name()]
|
||||
h.mu.Unlock()
|
||||
if e != "" {
|
||||
parts = append(parts, ch.Name()+": "+e)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "no channel"
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
func roughly(d time.Duration) string {
|
||||
switch {
|
||||
case d < 0:
|
||||
return "a moment"
|
||||
case d < 2*time.Minute:
|
||||
return fmt.Sprintf("%d s", int(d.Seconds()))
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d min", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%.1f h", d.Hours())
|
||||
}
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeChannel struct {
|
||||
name string
|
||||
notReady error
|
||||
fail error
|
||||
sends []Message
|
||||
edits map[string]Message
|
||||
n int
|
||||
}
|
||||
|
||||
func (f *fakeChannel) Name() string { return f.name }
|
||||
func (f *fakeChannel) Ready() error { return f.notReady }
|
||||
func (f *fakeChannel) CanEdit() bool { return true }
|
||||
func (f *fakeChannel) Send(m Message) (string, error) {
|
||||
if f.fail != nil {
|
||||
return "", f.fail
|
||||
}
|
||||
f.n++
|
||||
f.sends = append(f.sends, m)
|
||||
return fmt.Sprint(f.n), nil
|
||||
}
|
||||
func (f *fakeChannel) Edit(id string, m Message) error {
|
||||
if f.fail != nil {
|
||||
return f.fail
|
||||
}
|
||||
if f.edits == nil {
|
||||
f.edits = map[string]Message{}
|
||||
}
|
||||
f.edits[id] = m
|
||||
return nil
|
||||
}
|
||||
|
||||
type memStore struct {
|
||||
recs map[string]Record
|
||||
recent []Sent
|
||||
}
|
||||
|
||||
func (m *memStore) Put(r Record) error {
|
||||
if m.recs == nil {
|
||||
m.recs = map[string]Record{}
|
||||
}
|
||||
m.recs[r.Key] = r
|
||||
return nil
|
||||
}
|
||||
func (m *memStore) Delete(k string) error { delete(m.recs, k); return nil }
|
||||
func (m *memStore) All() ([]Record, error) {
|
||||
var out []Record
|
||||
for _, r := range m.recs {
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (m *memStore) PutRecent(s []Sent) error { m.recent = s; return nil }
|
||||
func (m *memStore) Recent() ([]Sent, error) { return m.recent, nil }
|
||||
|
||||
type clock struct{ t time.Time }
|
||||
|
||||
func (c *clock) now() time.Time { return c.t }
|
||||
func (c *clock) pass(d time.Duration) { c.t = c.t.Add(d) }
|
||||
func start() *clock { return &clock{t: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)} }
|
||||
func cond(key, sev, summary string) Condition {
|
||||
parts := strings.Split(key, ".")
|
||||
return Condition{Key: key, Scope: parts[0], ID: parts[1], Kind: parts[len(parts)-1], Severity: sev, Summary: summary}
|
||||
}
|
||||
|
||||
func newHolder(t *testing.T) (*Holder, *fakeChannel, *fakeChannel, *clock, *memStore) {
|
||||
t.Helper()
|
||||
c := start()
|
||||
tg, dt := &fakeChannel{name: "telegram"}, &fakeChannel{name: "desktop"}
|
||||
st := &memStore{}
|
||||
var emitted []string
|
||||
h := &Holder{Telegram: tg, Desktop: dt, Store: st, Now: c.now, Logf: t.Logf,
|
||||
Emit: func(e string, _ any) error { emitted = append(emitted, e); return nil }}
|
||||
h.init()
|
||||
return h, tg, dt, c, st
|
||||
}
|
||||
|
||||
func TestARaisedConditionIsSentOnceByItsKey(t *testing.T) {
|
||||
h, tg, dt, c, _ := newHolder(t)
|
||||
k := cond("machine.ace.silent", Urgent, "the home server has not been heard for 15 min")
|
||||
k.Raised = c.now()
|
||||
h.Condition(EventRaised, k)
|
||||
h.Condition(EventRaised, k) // a redelivery
|
||||
c.pass(time.Minute)
|
||||
h.Condition(EventRaised, k) // said again by the controller
|
||||
if len(tg.sends) != 1 || len(dt.sends) != 1 {
|
||||
t.Fatalf("urgent: telegram %d, desktop %d; want one each", len(tg.sends), len(dt.sends))
|
||||
}
|
||||
if !strings.Contains(tg.sends[0].Text(), "machine.ace.silent") || !strings.HasPrefix(tg.sends[0].Title, "URGENT: ") {
|
||||
t.Fatalf("message: %q", tg.sends[0].Text())
|
||||
}
|
||||
if open := h.Open(); len(open) != 1 || open[0].Count != 1 {
|
||||
t.Fatalf("open: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarningGoesToTheDesktopWhenASessionAnswersElseTelegram(t *testing.T) {
|
||||
h, tg, dt, _, _ := newHolder(t)
|
||||
h.Condition(EventRaised, cond("plan.41.stalled", Warning, "plan 41 waits on a build"))
|
||||
if len(dt.sends) != 1 || len(tg.sends) != 0 {
|
||||
t.Fatalf("desktop answered: desktop %d telegram %d", len(dt.sends), len(tg.sends))
|
||||
}
|
||||
dt.fail = errors.New("the account is not logged in")
|
||||
h.Condition(EventRaised, cond("plan.42.stalled", Warning, "plan 42 waits on a build"))
|
||||
if len(tg.sends) != 1 {
|
||||
t.Fatalf("no session: telegram %d", len(tg.sends))
|
||||
}
|
||||
dt.fail, dt.notReady = nil, errors.New("not configured")
|
||||
h.Condition(EventRaised, cond("plan.43.stalled", Warning, "plan 43 waits on a build"))
|
||||
if len(tg.sends) != 2 {
|
||||
t.Fatalf("no desktop configured: telegram %d", len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCapHoldsTheRestAndFoldsThemIntoOneMessage(t *testing.T) {
|
||||
h, tg, _, c, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
for i := 0; i < 25; i++ {
|
||||
h.Condition(EventRaised, cond(fmt.Sprintf("machine.m%d.silent", i), Urgent, "a machine is silent"))
|
||||
}
|
||||
if len(tg.sends) != CapPerHour {
|
||||
t.Fatalf("sent %d, cap %d", len(tg.sends), CapPerHour)
|
||||
}
|
||||
st := h.Status("listening")
|
||||
if st.Channels[0].Held != 5 || st.Channels[0].SentLastHour != CapPerHour {
|
||||
t.Fatalf("status: %+v", st.Channels[0])
|
||||
}
|
||||
// Said, not silent: the fold goes out at the first tick, and again only after ten minutes.
|
||||
h.Tick()
|
||||
if len(tg.sends) != CapPerHour+1 {
|
||||
t.Fatalf("no fold: %d sends", len(tg.sends))
|
||||
}
|
||||
fold := tg.sends[len(tg.sends)-1]
|
||||
if !strings.Contains(fold.Title, "HELD BACK: 5") {
|
||||
t.Fatalf("fold: %q", fold.Text())
|
||||
}
|
||||
for i := 20; i < 25; i++ {
|
||||
if !strings.Contains(fold.Body, fmt.Sprintf("machine.m%d.silent", i)) {
|
||||
t.Fatalf("fold does not name m%d: %q", i, fold.Body)
|
||||
}
|
||||
}
|
||||
h.Condition(EventRaised, cond("machine.late.silent", Urgent, "a machine is silent"))
|
||||
c.pass(time.Minute)
|
||||
h.Tick()
|
||||
if len(tg.sends) != CapPerHour+1 {
|
||||
t.Fatalf("a second fold inside ten minutes")
|
||||
}
|
||||
c.pass(FoldEvery)
|
||||
h.Tick()
|
||||
if last := tg.sends[len(tg.sends)-1]; !strings.Contains(last.Body, "machine.late.silent") {
|
||||
t.Fatalf("the second fold: %q", last.Text())
|
||||
}
|
||||
// An hour on, the window is free again.
|
||||
c.pass(time.Hour)
|
||||
h.Condition(EventRaised, cond("machine.next.silent", Urgent, "a machine is silent"))
|
||||
if last := tg.sends[len(tg.sends)-1]; !strings.Contains(last.Text(), "machine.next.silent") {
|
||||
t.Fatalf("not sent after the window: %q", last.Text())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMessageCarryingAnAddressIsRefusedAndSaidWithItsWordsWithheld(t *testing.T) {
|
||||
h, tg, _, _, _ := newHolder(t)
|
||||
var emitted []string
|
||||
h.Emit = func(e string, _ any) error { emitted = append(emitted, e); return nil }
|
||||
h.Condition(EventRaised, cond("provider.keycloak.ace.failing", Urgent, "cannot reach 192.168.1.20:8443 with token=abc"))
|
||||
if len(tg.sends) != 1 {
|
||||
t.Fatalf("sends: %d", len(tg.sends))
|
||||
}
|
||||
text := tg.sends[0].Text()
|
||||
if strings.Contains(text, "192.168") || strings.Contains(text, "token=") {
|
||||
t.Fatalf("the address left: %q", text)
|
||||
}
|
||||
if !strings.Contains(text, "channel-refused") || !strings.Contains(text, "provider.keycloak.ace.failing") {
|
||||
t.Fatalf("not said as refused, by key: %q", text)
|
||||
}
|
||||
hist := h.History(10)["refusals"].([]RefusalNote)
|
||||
if len(hist) != 1 || hist[0].Class != "address" {
|
||||
t.Fatalf("refusals: %+v", hist)
|
||||
}
|
||||
if len(emitted) != 1 || emitted[0] != "refused" {
|
||||
t.Fatalf("emitted: %v", emitted)
|
||||
}
|
||||
// Its clearing keeps the words withheld.
|
||||
h.Condition(EventCleared, Condition{Key: "provider.keycloak.ace.failing"})
|
||||
for _, m := range tg.edits {
|
||||
if strings.Contains(m.Text(), "192.168") {
|
||||
t.Fatalf("the clearing carried it: %q", m.Text())
|
||||
}
|
||||
}
|
||||
if len(tg.edits) != 1 {
|
||||
t.Fatalf("edits: %d", len(tg.edits))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyThatCarriesAnAddressIsWithheldToo(t *testing.T) {
|
||||
h, tg, _, _, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
h.Condition(EventRaised, cond("machine.10.0.0.7.silent", Urgent, "a machine is silent"))
|
||||
if len(tg.sends) != 1 || strings.Contains(tg.sends[0].Text(), "10.0.0.7") {
|
||||
t.Fatalf("sends: %+v", tg.sends)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStillOpenPastItsBoundItIsSaidOnceMore(t *testing.T) {
|
||||
h, tg, _, c, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
u := cond("bus.controller.slow-consumer", Urgent, "the controller's consumer is far behind")
|
||||
u.Raised = c.now()
|
||||
w := cond("plan.41.stalled", Warning, "plan 41 waits")
|
||||
w.Raised = c.now()
|
||||
h.Condition(EventRaised, u)
|
||||
h.Condition(EventRaised, w)
|
||||
c.pass(59 * time.Minute)
|
||||
h.Tick()
|
||||
if len(tg.sends) != 2 {
|
||||
t.Fatalf("reminded before the hour: %d", len(tg.sends))
|
||||
}
|
||||
c.pass(2 * time.Minute)
|
||||
h.Tick()
|
||||
if len(tg.sends) != 3 || !strings.HasPrefix(tg.sends[2].Title, "STILL OPEN after 61 min") {
|
||||
t.Fatalf("urgent reminder: %d %q", len(tg.sends), tg.sends[len(tg.sends)-1].Title)
|
||||
}
|
||||
c.pass(3 * time.Hour)
|
||||
h.Tick()
|
||||
if len(tg.sends) != 3 {
|
||||
t.Fatalf("reminded twice")
|
||||
}
|
||||
c.pass(9 * time.Hour) // the warning is now 13 h old
|
||||
h.Tick()
|
||||
if len(tg.sends) != 4 || !strings.Contains(tg.sends[3].Text(), "plan.41.stalled") {
|
||||
t.Fatalf("warning reminder: %d", len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearedEditsTheFirstMessageAndReopenedWithinTenMinutesIsNotNew(t *testing.T) {
|
||||
h, tg, dt, c, st := newHolder(t)
|
||||
k := cond("machine.ace.silent", Urgent, "the home server is silent")
|
||||
k.Raised = c.now()
|
||||
h.Condition(EventRaised, k)
|
||||
c.pass(14 * time.Minute)
|
||||
h.Condition(EventCleared, Condition{Key: k.Key})
|
||||
if len(tg.sends) != 1 || len(tg.edits) != 1 || len(dt.edits) != 1 {
|
||||
t.Fatalf("telegram sends %d edits %d, desktop edits %d", len(tg.sends), len(tg.edits), len(dt.edits))
|
||||
}
|
||||
if m := tg.edits["1"]; !strings.HasPrefix(m.Title, "CLEARED after 14 min") {
|
||||
t.Fatalf("edit: %q", m.Title)
|
||||
}
|
||||
if len(h.Open()) != 0 {
|
||||
t.Fatalf("still open")
|
||||
}
|
||||
c.pass(5 * time.Minute)
|
||||
h.Condition(EventRaised, k)
|
||||
if len(tg.sends) != 1 || !strings.Contains(tg.edits["1"].Title, "open again, 2 times") {
|
||||
t.Fatalf("reopened as new: sends %d, edit %q", len(tg.sends), tg.edits["1"].Title)
|
||||
}
|
||||
h.Condition(EventCleared, Condition{Key: k.Key})
|
||||
c.pass(11 * time.Minute)
|
||||
h.Tick()
|
||||
if _, kept := st.recs[k.Key]; kept {
|
||||
t.Fatalf("a cleared message kept past the reopen window")
|
||||
}
|
||||
h.Condition(EventRaised, k)
|
||||
if len(tg.sends) != 2 {
|
||||
t.Fatalf("a raise after the window is a new message: %d", len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilencedConditionSendsNothing(t *testing.T) {
|
||||
h, tg, dt, c, _ := newHolder(t)
|
||||
k := cond("machine.ace.silent", Urgent, "silent")
|
||||
k.SilencedTill = c.now().Add(2 * time.Hour)
|
||||
h.Condition(EventRaised, k)
|
||||
c.pass(90 * time.Minute)
|
||||
h.Tick()
|
||||
h.Condition(EventCleared, Condition{Key: k.Key})
|
||||
if len(tg.sends)+len(dt.sends)+len(tg.edits)+len(dt.edits) != 0 {
|
||||
t.Fatalf("a silenced condition said something")
|
||||
}
|
||||
// Silenced after it was sent: no reminder.
|
||||
k2 := cond("machine.shanks.silent", Urgent, "silent")
|
||||
k2.Raised = c.now()
|
||||
h.Condition(EventRaised, k2)
|
||||
k2.SilencedTill = c.now().Add(3 * time.Hour)
|
||||
h.Condition(EventChanged, k2)
|
||||
c.pass(2 * time.Hour)
|
||||
h.Tick()
|
||||
if len(tg.sends) != 1 {
|
||||
t.Fatalf("reminded while silenced: %d", len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscalationIsSaidOnce(t *testing.T) {
|
||||
h, tg, dt, _, _ := newHolder(t)
|
||||
k := cond("machine.novox.silent", Warning, "the anchor is silent")
|
||||
h.Condition(EventRaised, k)
|
||||
k.Severity = Urgent
|
||||
h.Condition(EventChanged, k)
|
||||
h.Condition(EventChanged, k)
|
||||
if len(dt.sends) != 2 || len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "NOW URGENT") {
|
||||
t.Fatalf("desktop %d telegram %d", len(dt.sends), len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAChannelThatCannotSendSaysSoAndIsTriedAgain(t *testing.T) {
|
||||
h, tg, _, _, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
tg.fail = errors.New("telegram sendMessage: cannot connect (dial)")
|
||||
h.Condition(EventRaised, cond("machine.ace.silent", Urgent, "silent"))
|
||||
st := h.Status("listening")
|
||||
if !strings.HasPrefix(st.Verdict, "CANNOT SEND") || len(st.Unsent) != 1 || st.Channels[0].LastError == "" {
|
||||
t.Fatalf("status: %+v", st)
|
||||
}
|
||||
tg.fail = nil
|
||||
h.Tick()
|
||||
if len(tg.sends) != 1 || h.Status("listening").Verdict != "ok" {
|
||||
t.Fatalf("not retried: %d, %s", len(tg.sends), h.Status("listening").Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingConfiguredIsSaidInTheStatus(t *testing.T) {
|
||||
h, tg, dt, _, _ := newHolder(t)
|
||||
tg.notReady = errors.New("no bot token")
|
||||
dt.notReady = errors.New("no machine")
|
||||
st := h.Status("listening")
|
||||
if !strings.HasPrefix(st.Verdict, "CANNOT SEND") || st.Channels[0].NotReady != "no bot token" {
|
||||
t.Fatalf("status: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARestartForgetsNothing(t *testing.T) {
|
||||
h, tg, _, c, st := newHolder(t)
|
||||
h.Desktop = nil
|
||||
k := cond("machine.ace.silent", Urgent, "silent")
|
||||
k.Raised = c.now()
|
||||
h.Condition(EventRaised, k)
|
||||
again := &Holder{Telegram: tg, Store: st, Now: c.now, Logf: t.Logf}
|
||||
if err := again.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again.Condition(EventRaised, k)
|
||||
if len(tg.sends) != 1 {
|
||||
t.Fatalf("a restart sent it again")
|
||||
}
|
||||
again.Condition(EventCleared, Condition{Key: k.Key})
|
||||
if len(tg.edits) != 1 {
|
||||
t.Fatalf("a restart lost the message to edit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableEventIsToldOnceAnHour(t *testing.T) {
|
||||
h, tg, _, c, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
h.Unreadable("mesh-controller.condition-raised", errors.New("no severity"))
|
||||
h.Unreadable("mesh-controller.condition-raised", errors.New("no severity"))
|
||||
if len(tg.sends) != 1 || h.Status("listening").Unreadable != 2 {
|
||||
t.Fatalf("sends %d", len(tg.sends))
|
||||
}
|
||||
c.pass(61 * time.Minute)
|
||||
h.Unreadable("mesh-controller.condition-raised", errors.New("no severity"))
|
||||
if len(tg.sends) != 2 {
|
||||
t.Fatalf("not said again after an hour")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotifyIsKeptApartFromConditions(t *testing.T) {
|
||||
h, tg, _, _, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
out, err := h.Notify("backup.ace.failed", Warning, "last night's backup of the home server failed", "backup", false)
|
||||
if err != nil || out["key"] != "notify.backup.ace.failed" || len(tg.sends) != 1 {
|
||||
t.Fatalf("%v %v %d", out, err, len(tg.sends))
|
||||
}
|
||||
if _, err := h.Notify("x", "", "s", "", false); err == nil {
|
||||
t.Fatalf("no severity was not refused")
|
||||
}
|
||||
if _, err := h.Notify("x", Urgent, "see /var/lib/x", "", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(tg.sends[len(tg.sends)-1].Text(), "/var/lib") {
|
||||
t.Fatalf("a path left")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATestMessageRespectsTheRule(t *testing.T) {
|
||||
h, tg, dt, _, _ := newHolder(t)
|
||||
if out := h.Test("telegram", "see https://example.org"); out["refused"] == "" || len(tg.sends) != 0 {
|
||||
t.Fatalf("%v", out)
|
||||
}
|
||||
if out := h.Test("", ""); out["telegram"] != "sent" || out["desktop"] != "sent" || len(dt.sends) != 1 {
|
||||
t.Fatalf("%v", out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
// messenger: the holder of the operator-channel seat (novox/hq to-be 45 §5, ADR 0227, research 028).
|
||||
// A Go bundle the node's runtime launches. It consumes the controller's condition events and decides
|
||||
// what is said to the operator, on Telegram and on the desktop notifier of the machine the operator
|
||||
// is at. It keeps its open messages in its own state, so a restart forgets nothing. stdout is the MCP
|
||||
// channel; what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func errorf(format string, a ...any) error { return fmt.Errorf(format, a...) }
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
// Settings are the operator's values for this module, merged by the mesh into one JSON file.
|
||||
type Settings struct {
|
||||
TelegramChatID string `json:"telegram-chat-id"`
|
||||
DesktopMachines []string `json:"desktop-machines"`
|
||||
}
|
||||
|
||||
func readSettings(path string) (Settings, error) {
|
||||
var s Settings
|
||||
if path == "" {
|
||||
return s, errors.New("started without a settings file")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
// The chat id may be given as a number.
|
||||
var loose map[string]any
|
||||
if err := json.Unmarshal(raw, &loose); err != nil {
|
||||
return s, fmt.Errorf("the settings file is not JSON: %v", err)
|
||||
}
|
||||
switch v := loose["telegram-chat-id"].(type) {
|
||||
case string:
|
||||
s.TelegramChatID = strings.TrimSpace(v)
|
||||
case float64:
|
||||
s.TelegramChatID = fmt.Sprintf("%.0f", v)
|
||||
}
|
||||
if list, ok := loose["desktop-machines"].([]any); ok {
|
||||
for _, m := range list {
|
||||
if name, ok := m.(string); ok && strings.TrimSpace(name) != "" {
|
||||
s.DesktopMachines = append(s.DesktopMachines, strings.TrimSpace(name))
|
||||
}
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// stateStore keeps the open messages in the module's declared state `open`, and the recent sends
|
||||
// under one key of `sent` (ADR 0201).
|
||||
type stateStore struct{}
|
||||
|
||||
// kvKey is a condition key as a bucket key: only the characters a key may carry.
|
||||
func kvKey(key string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range key {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '.', r == '=':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteRune('_')
|
||||
}
|
||||
}
|
||||
return strings.Trim(b.String(), ".")
|
||||
}
|
||||
|
||||
func (stateStore) Put(r Record) error {
|
||||
_, err := stdio.State("open").Put(kvKey(r.Key), r)
|
||||
return err
|
||||
}
|
||||
func (stateStore) Delete(key string) error { return stdio.State("open").Delete(kvKey(key)) }
|
||||
func (stateStore) All() ([]Record, error) {
|
||||
keys, err := stdio.State("open").Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []Record
|
||||
for _, k := range keys {
|
||||
e, err := stdio.State("open").Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
var r Record
|
||||
if err := json.Unmarshal(e.Value, &r); err != nil {
|
||||
logf("[messenger] the open message kept as %s cannot be read (%v); left as it is", k, err)
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (stateStore) PutRecent(s []Sent) error {
|
||||
_, err := stdio.State("sent").Put("recent", s)
|
||||
return err
|
||||
}
|
||||
func (stateStore) Recent() ([]Sent, error) {
|
||||
e, err := stdio.State("sent").Get("recent")
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
var s []Sent
|
||||
return s, json.Unmarshal(e.Value, &s)
|
||||
}
|
||||
|
||||
// listening is whether condition events reach this holder, in words.
|
||||
type listening struct {
|
||||
mu sync.Mutex
|
||||
now string
|
||||
}
|
||||
|
||||
func (l *listening) set(s string) { l.mu.Lock(); l.now = s; l.mu.Unlock() }
|
||||
func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.now }
|
||||
|
||||
func main() {
|
||||
settingsFile := os.Getenv("MESH_MESSENGER_SETTINGS")
|
||||
var said sync.Mutex
|
||||
lastSaid := ""
|
||||
settings := func() Settings {
|
||||
s, err := readSettings(settingsFile)
|
||||
said.Lock()
|
||||
defer said.Unlock()
|
||||
if err != nil && err.Error() != lastSaid {
|
||||
logf("[messenger] settings cannot be read: %v", err)
|
||||
}
|
||||
lastSaid = ""
|
||||
if err != nil {
|
||||
lastSaid = err.Error()
|
||||
}
|
||||
return s
|
||||
}
|
||||
h := &Holder{
|
||||
Telegram: NewTelegram(TelegramConfig{
|
||||
TokenFile: os.Getenv("MESH_MESSENGER_TELEGRAM_TOKEN_FILE"),
|
||||
ChatID: func() string { return settings().TelegramChatID },
|
||||
}),
|
||||
Desktop: &Desktop{
|
||||
Machines: func() []string { return settings().DesktopMachines },
|
||||
Ask: stdio.Ask,
|
||||
},
|
||||
Store: stateStore{},
|
||||
Logf: logf,
|
||||
Emit: func(event string, body any) error { return stdio.Emit(event, body) },
|
||||
}
|
||||
h.init()
|
||||
l := &listening{now: "not yet: starting"}
|
||||
go run(h, l)
|
||||
if err := stdio.Serve("", tools(h, l)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// run reads back the state, listens for condition events, and keeps time — each retried, and each
|
||||
// failure said, never given up on quietly.
|
||||
func run(h *Holder, l *listening) {
|
||||
time.Sleep(500 * time.Millisecond) // Serve first: the state is reached through it
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := h.Load()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
logf("[messenger] cannot read back its open messages yet (%v); asking again in %s", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
for _, ch := range h.channels() {
|
||||
if err := ch.Ready(); err != nil {
|
||||
logf("[messenger] %s cannot send: %v", ch.Name(), err)
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
for range time.Tick(time.Minute) {
|
||||
h.Tick()
|
||||
}
|
||||
}()
|
||||
handle := func(e stdio.Envelope) error {
|
||||
event, ok := eventOf(e.Key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
c, err := DecodeCondition(event, e.Body)
|
||||
if err != nil {
|
||||
h.Unreadable(e.Key, err)
|
||||
return nil
|
||||
}
|
||||
h.Condition(event, c)
|
||||
return nil
|
||||
}
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := stdio.Subscribe(ControllerSeat+".*", handle)
|
||||
if err == nil {
|
||||
l.set("listening")
|
||||
logf("[messenger] listening for the controller's condition events")
|
||||
return
|
||||
}
|
||||
l.set("not yet: " + err.Error())
|
||||
logf("[messenger] not hearing condition events yet (%v); asking again in %s", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
func str(description string) map[string]any {
|
||||
return map[string]any{"type": "string", "description": description}
|
||||
}
|
||||
|
||||
func strArg(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) }
|
||||
|
||||
func limitArg(a map[string]any, def int) int {
|
||||
if v, ok := a["limit"].(float64); ok && v >= 1 {
|
||||
return min(int(v), KeptSends)
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func tools(h *Holder, l *listening) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: "operator-channel.open",
|
||||
Description: "What is open now: every message the operator was sent about something still wrong, urgent first, " +
|
||||
"oldest first — its key, severity, summary, since when, the channels it went to, whether it is silenced, " +
|
||||
"reminded, held by the cap, refused, or not sent yet.",
|
||||
Run: func(map[string]any) (any, error) { return h.Open(), nil }},
|
||||
{Name: "operator-channel.history",
|
||||
Description: "What was said to the operator lately, newest first — each send, edit, fold and failure with its " +
|
||||
"channel, key and outcome — and every message refused for carrying an address, a path or a secret.",
|
||||
Input: map[string]any{"limit": map[string]any{"type": "integer", "description": "at most this many of each (default 50)"}},
|
||||
Run: func(a map[string]any) (any, error) { return h.History(limitArg(a, 50)), nil }},
|
||||
{Name: "operator-channel.notify",
|
||||
Description: "Tell the operator something, as a module that uses the seat: a key (the same key is the same " +
|
||||
"message), urgent or warning, one line in the mesh's words, and what it is about. clear says it is over. " +
|
||||
"Roles and words only: an address, a path or a secret is refused. Deduplicated, capped and routed " +
|
||||
"like the controller's conditions.",
|
||||
Input: map[string]any{
|
||||
"key": str("what makes it the same message the next time, e.g. backup.ace.failed"),
|
||||
"severity": map[string]any{"type": "string", "enum": []string{Urgent, Warning}},
|
||||
"summary": str("one line in the mesh's words"),
|
||||
"subject": str("what it is about: a machine's role, a module, a plan"),
|
||||
"clear": map[string]any{"type": "boolean", "description": "it is over: the message is edited to say so"},
|
||||
},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
clear, _ := a["clear"].(bool)
|
||||
return h.Notify(strArg(a, "key"), strArg(a, "severity"), strArg(a, "summary"), strArg(a, "subject"), clear)
|
||||
}},
|
||||
{Name: "messenger_status",
|
||||
Description: "Whether the operator can be reached, and why not: each channel — can it send, what it lacks " +
|
||||
"(the Telegram bot token and chat id, the desktop machines), its last error, how many it sent in the " +
|
||||
"last hour against the cap, how many it holds — the open and silenced count, messages not sent yet, " +
|
||||
"refusals, unreadable events, whether condition events arrive, and the rules it applies. check asks " +
|
||||
"Telegram who the bot is, sending nothing.",
|
||||
Input: map[string]any{"check": map[string]any{"type": "boolean", "description": "also ask Telegram whether the token works"}},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
st := h.Status(l.get())
|
||||
if check, _ := a["check"].(bool); check {
|
||||
if t, ok := h.Telegram.(*Telegram); ok {
|
||||
who, err := t.Who()
|
||||
if err != nil {
|
||||
return map[string]any{"status": st, "telegram_check": "failed: " + err.Error()}, nil
|
||||
}
|
||||
return map[string]any{"status": st, "telegram_check": "the token works: the bot is @" + who}, nil
|
||||
}
|
||||
}
|
||||
return st, nil
|
||||
}},
|
||||
{Name: "messenger_recent",
|
||||
Description: "The recent sends — each message, edit, fold and failure on each channel, newest first — and the refusals.",
|
||||
Input: map[string]any{"limit": map[string]any{"type": "integer", "description": "at most this many (default 20)"}},
|
||||
Run: func(a map[string]any) (any, error) { return h.History(limitArg(a, 20)), nil }},
|
||||
{Name: "messenger_test",
|
||||
Description: "Send a test message now, to telegram, desktop or both (default both), through the content rule " +
|
||||
"and the cap: proves a channel reaches the operator. Answers per channel: sent, or why not.",
|
||||
Input: map[string]any{
|
||||
"channel": map[string]any{"type": "string", "enum": []string{"telegram", "desktop", "both"}},
|
||||
"text": str("the words (default: a line saying it is a test)"),
|
||||
},
|
||||
Run: func(a map[string]any) (any, error) { return h.Test(strArg(a, "channel"), strArg(a, "text")), nil }},
|
||||
{Name: "messenger_check",
|
||||
Description: "Would these words be allowed to leave the mesh? Runs the content rule — no address, path or " +
|
||||
"secret — and answers allowed, or the class of what it carried. Sends nothing.",
|
||||
Input: map[string]any{"text": str("the words to judge")},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if r, ok := Check(strArg(a, "text")); !ok {
|
||||
return map[string]any{"allowed": false, "class": r.Class, "carried": r.What}, nil
|
||||
}
|
||||
return map[string]any{"allowed": true}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The manifest says what the code does: the seat it declares and holds and the verbs it serves, the
|
||||
// events it consumes (the controller's three), the one it emits, the tool it calls, its state, its
|
||||
// secret, and its own tools — and names nothing of one installation.
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Seats []seat `json:"seats"`
|
||||
Claims []seat `json:"claims"`
|
||||
Consumes []string
|
||||
Emits []string
|
||||
Invokes []string
|
||||
State []string
|
||||
Own map[string]string `json:"own-secrets"`
|
||||
Tools []string
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
type seat struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T) (manifest, string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, string(raw)
|
||||
}
|
||||
|
||||
func TestItDeclaresAndHoldsTheOperatorChannel(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
want := seat{Name: "operator-channel", Scope: "mesh", Serves: []string{"open", "history", "notify"}}
|
||||
if len(m.Seats) != 1 || !reflect.DeepEqual(m.Seats[0], want) || len(m.Claims) != 1 || !reflect.DeepEqual(m.Claims[0], want) {
|
||||
t.Fatalf("seats %+v claims %+v", m.Seats, m.Claims)
|
||||
}
|
||||
if !reflect.DeepEqual(m.Consumes, []string{
|
||||
ControllerSeat + "." + EventRaised, ControllerSeat + "." + EventChanged, ControllerSeat + "." + EventCleared}) {
|
||||
t.Fatalf("consumes %v", m.Consumes)
|
||||
}
|
||||
if !reflect.DeepEqual(m.Emits, []string{"refused"}) || !reflect.DeepEqual(m.Invokes, []string{"seat:node-notifier.send"}) {
|
||||
t.Fatalf("emits %v invokes %v", m.Emits, m.Invokes)
|
||||
}
|
||||
if !reflect.DeepEqual(m.State, []string{"open", "sent"}) {
|
||||
t.Fatalf("state %v", m.State)
|
||||
}
|
||||
if m.Own["telegram-token"] == "" {
|
||||
t.Fatalf("own secrets %v", m.Own)
|
||||
}
|
||||
env, _ := m.Build.Artifacts[0]["env"].(map[string]any)
|
||||
if env["MESH_MESSENGER_TELEGRAM_TOKEN_FILE"] != m.Own["telegram-token"] {
|
||||
t.Fatalf("the bundle reads its token from %v, the secret is at %v", env["MESH_MESSENGER_TELEGRAM_TOKEN_FILE"], m.Own["telegram-token"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsAgreeWithTheManifest(t *testing.T) {
|
||||
m, raw := readManifest(t)
|
||||
var own, verbs []string
|
||||
for _, tool := range tools(&Holder{}, &listening{}) {
|
||||
if strings.TrimSpace(tool.Description) == "" {
|
||||
t.Errorf("%s has no description", tool.Name)
|
||||
}
|
||||
if seatName, verb, ok := strings.Cut(tool.Name, "."); ok {
|
||||
if seatName != "operator-channel" {
|
||||
t.Errorf("%s is a verb of a seat this does not hold", tool.Name)
|
||||
}
|
||||
verbs = append(verbs, verb)
|
||||
continue
|
||||
}
|
||||
own = append(own, tool.Name)
|
||||
}
|
||||
listed := append([]string(nil), m.Tools...)
|
||||
sort.Strings(own)
|
||||
sort.Strings(listed)
|
||||
if !reflect.DeepEqual(own, listed) {
|
||||
t.Fatalf("serves %v, lists %v", own, listed)
|
||||
}
|
||||
if !reflect.DeepEqual(verbs, m.Claims[0].Serves) {
|
||||
t.Fatalf("verbs %v, claim %v", verbs, m.Claims[0].Serves)
|
||||
}
|
||||
for _, never := range []string{"/home/", "jochen", "g14", "shanks", "novox", "zurag", "api.telegram"} {
|
||||
if strings.Contains(strings.ToLower(raw), never) {
|
||||
t.Errorf("module.json names %q", never)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ChannelStatus is one channel as the status says it.
|
||||
type ChannelStatus struct {
|
||||
Name string `json:"name"`
|
||||
CanSend bool `json:"can_send"`
|
||||
NotReady string `json:"not_ready,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
LastSent string `json:"last_sent,omitempty"`
|
||||
SentLastHour int `json:"sent_last_hour"`
|
||||
Cap int `json:"cap_per_hour"`
|
||||
Held int `json:"held_by_cap"`
|
||||
}
|
||||
|
||||
// Status is the holder's whole account of itself.
|
||||
type Status struct {
|
||||
Verdict string `json:"verdict"`
|
||||
Channels []ChannelStatus `json:"channels"`
|
||||
Open int `json:"open"`
|
||||
Silenced int `json:"silenced"`
|
||||
Unsent []string `json:"unsent,omitempty"`
|
||||
Refused int `json:"refused_since_start"`
|
||||
Unreadable int `json:"unreadable_events_since_start"`
|
||||
LastBad string `json:"last_unreadable,omitempty"`
|
||||
Heard map[string]int `json:"events_heard_since_start"`
|
||||
LastHeard string `json:"last_event_heard,omitempty"`
|
||||
Listening string `json:"listening"`
|
||||
StateProblem string `json:"state_problem,omitempty"`
|
||||
Rules []string `json:"rules"`
|
||||
}
|
||||
|
||||
// Status answers at once from what the holder keeps.
|
||||
func (h *Holder) Status(listening string) Status {
|
||||
now := h.Now()
|
||||
var st Status
|
||||
for _, ch := range h.channels() {
|
||||
cs := ChannelStatus{Name: ch.Name(), Cap: CapPerHour, SentLastHour: h.sentLastHour(ch.Name())}
|
||||
if err := ch.Ready(); err != nil {
|
||||
cs.NotReady = err.Error()
|
||||
} else {
|
||||
cs.CanSend = true
|
||||
}
|
||||
h.mu.Lock()
|
||||
cs.LastError = h.chanErr[ch.Name()]
|
||||
if t, ok := h.chanOK[ch.Name()]; ok {
|
||||
cs.LastSent = t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
cs.Held = len(h.folds[ch.Name()])
|
||||
h.mu.Unlock()
|
||||
if cs.LastError != "" {
|
||||
cs.CanSend = false
|
||||
}
|
||||
st.Channels = append(st.Channels, cs)
|
||||
}
|
||||
h.mu.Lock()
|
||||
for _, r := range h.open {
|
||||
if !r.Cleared.IsZero() {
|
||||
continue
|
||||
}
|
||||
st.Open++
|
||||
if r.silenced(now) {
|
||||
st.Silenced++
|
||||
}
|
||||
if r.Pending != "" {
|
||||
st.Unsent = append(st.Unsent, r.Key+" ("+r.Pending+")")
|
||||
}
|
||||
}
|
||||
st.Refused = len(h.refusals)
|
||||
st.Unreadable = h.unreadable
|
||||
st.LastBad = h.lastBad
|
||||
st.Heard = map[string]int{}
|
||||
for k, v := range h.heard {
|
||||
st.Heard[k] = v
|
||||
}
|
||||
if !h.lastHeard.IsZero() {
|
||||
st.LastHeard = h.lastHeard.UTC().Format(time.RFC3339)
|
||||
}
|
||||
st.StateProblem = h.storeErr
|
||||
h.mu.Unlock()
|
||||
sort.Strings(st.Unsent)
|
||||
st.Listening = listening
|
||||
st.Rules = []string{
|
||||
"urgent: Telegram and the desktop; warning: the desktop where a session answers, otherwise Telegram",
|
||||
"deduplicated by the condition's key; reminded once after 1 h (urgent) or 12 h (warning); cleared by editing the first message",
|
||||
"at most 20 messages an hour per channel; the rest folded into one message, at most every 10 min",
|
||||
"refused: anything carrying an address, a path or a secret; channel-refused is sent with the words withheld",
|
||||
"silenced conditions send nothing; silence is `conditions silence` through the mesh",
|
||||
}
|
||||
anyCan := false
|
||||
for _, c := range st.Channels {
|
||||
anyCan = anyCan || c.CanSend
|
||||
}
|
||||
switch {
|
||||
case !anyCan:
|
||||
st.Verdict = "CANNOT SEND: no channel can reach the operator — see channels"
|
||||
case len(st.Unsent) > 0:
|
||||
st.Verdict = "BEHIND: some messages could not be sent yet and are tried every minute"
|
||||
case listening != "listening":
|
||||
st.Verdict = "NOT LISTENING: condition events are not reaching this holder — " + listening
|
||||
case st.StateProblem != "":
|
||||
st.Verdict = "STATE: open messages are held in memory only — " + st.StateProblem
|
||||
default:
|
||||
st.Verdict = "ok"
|
||||
for _, c := range st.Channels {
|
||||
if !c.CanSend {
|
||||
st.Verdict = "ok on one channel: " + c.Name + " cannot send"
|
||||
}
|
||||
}
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// OpenMessage is one open message as `open` answers it.
|
||||
type OpenMessage struct {
|
||||
Key string `json:"key"`
|
||||
Severity string `json:"severity"`
|
||||
Summary string `json:"summary"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Since string `json:"since"`
|
||||
Origin string `json:"origin"`
|
||||
SentOn []string `json:"sent_on,omitempty"`
|
||||
Silenced string `json:"silenced_until,omitempty"`
|
||||
Reminded bool `json:"reminded"`
|
||||
Unsent string `json:"unsent,omitempty"`
|
||||
Held bool `json:"held_by_cap,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Count int `json:"times_opened"`
|
||||
}
|
||||
|
||||
// Open is what is unresolved now, urgent first, oldest first.
|
||||
func (h *Holder) Open() []OpenMessage {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
var out []OpenMessage
|
||||
for _, r := range h.open {
|
||||
if !r.Cleared.IsZero() {
|
||||
continue
|
||||
}
|
||||
o := OpenMessage{Key: r.Key, Severity: r.Severity, Summary: r.Summary, Subject: r.Subject,
|
||||
Since: r.Raised.UTC().Format(time.RFC3339), Origin: r.Origin, Reminded: r.Reminded,
|
||||
Unsent: r.Pending, Held: r.Folded, Refused: r.Refused, Count: r.Count}
|
||||
for ch := range r.Sent {
|
||||
o.SentOn = append(o.SentOn, ch)
|
||||
}
|
||||
sort.Strings(o.SentOn)
|
||||
if r.silenced(now) {
|
||||
o.Silenced = r.SilencedTill.UTC().Format(time.RFC3339)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Severity != out[j].Severity {
|
||||
return out[i].Severity == Urgent
|
||||
}
|
||||
return out[i].Since < out[j].Since
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// History is what went out, newest first, and the refusals.
|
||||
func (h *Holder) History(limit int) map[string]any {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
sends := []Sent{}
|
||||
for i := len(h.recent) - 1; i >= 0 && len(sends) < limit; i-- {
|
||||
sends = append(sends, h.recent[i])
|
||||
}
|
||||
refusals := []RefusalNote{}
|
||||
for i := len(h.refusals) - 1; i >= 0 && len(refusals) < limit; i-- {
|
||||
refusals = append(refusals, h.refusals[i])
|
||||
}
|
||||
return map[string]any{"sends": sends, "refusals": refusals}
|
||||
}
|
||||
|
||||
// Test sends a test message on one channel or both, through the content rule and the cap.
|
||||
func (h *Holder) Test(channel, text string) map[string]string {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
if text == "" {
|
||||
text = "a test from the operator-channel's holder: this channel reaches you"
|
||||
}
|
||||
out := map[string]string{}
|
||||
if refusal, ok := Check(text); !ok {
|
||||
out["refused"] = "it carried " + refusal.String() + "; nothing was sent"
|
||||
return out
|
||||
}
|
||||
r := &Record{Key: "messenger.test", Severity: Warning, Raised: h.Now(), Sent: map[string]string{}}
|
||||
m := Message{Title: "TEST: " + text, Body: "nothing is wrong; this was asked for"}
|
||||
for _, ch := range h.channels() {
|
||||
if channel != "" && channel != "both" && channel != ch.Name() {
|
||||
continue
|
||||
}
|
||||
if err := ch.Ready(); err != nil {
|
||||
out[ch.Name()] = "not sent: " + err.Error()
|
||||
continue
|
||||
}
|
||||
if !h.allow(ch.Name()) {
|
||||
out[ch.Name()] = "not sent: the channel is at its cap of 20 an hour"
|
||||
continue
|
||||
}
|
||||
if h.sendOn(ch, r, "test", m) {
|
||||
out[ch.Name()] = "sent"
|
||||
} else {
|
||||
h.mu.Lock()
|
||||
out[ch.Name()] = "failed: " + h.chanErr[ch.Name()]
|
||||
h.mu.Unlock()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Notify takes a message from a module that uses the seat (research 028 Q5, modules as sources):
|
||||
// the same shape, rule, cap and deduplication as a condition. Its key is put under `notify.` so it
|
||||
// can never be taken for one of the controller's conditions.
|
||||
func (h *Holder) Notify(key, severity, summary, subject string, clear bool) (map[string]any, error) {
|
||||
if key == "" {
|
||||
return nil, errorf("key is required: the same key is the same message")
|
||||
}
|
||||
if len(key) < 7 || key[:7] != "notify." {
|
||||
key = "notify." + key
|
||||
}
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
h.mu.Unlock()
|
||||
if clear {
|
||||
h.cleared(key)
|
||||
return map[string]any{"key": key, "cleared": true}, nil
|
||||
}
|
||||
switch severity {
|
||||
case Urgent, Warning:
|
||||
case "":
|
||||
return nil, errorf("severity is required: urgent or warning")
|
||||
default:
|
||||
return nil, errorf("severity %q is neither urgent nor warning", severity)
|
||||
}
|
||||
if summary == "" {
|
||||
return nil, errorf("summary is required: one line in the mesh's words")
|
||||
}
|
||||
h.raised(Record{Key: key, Kind: "notice", Subject: subject, Severity: severity, Summary: summary,
|
||||
Origin: "notify", More: "operator-channel.open"})
|
||||
h.mu.Lock()
|
||||
r := h.open[key]
|
||||
var sent []string
|
||||
refused, pending := "", ""
|
||||
if r != nil {
|
||||
for ch := range r.Sent {
|
||||
sent = append(sent, ch)
|
||||
}
|
||||
refused, pending = r.Refused, r.Pending
|
||||
}
|
||||
h.mu.Unlock()
|
||||
sort.Strings(sent)
|
||||
return map[string]any{"key": key, "sent_on": sent, "refused": refused, "unsent": pending}, nil
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package main
|
||||
|
||||
// The Telegram channel: a bot to the operator's chat (novox/hq to-be 45 §5, research 028/02). Sending
|
||||
// needs only outbound HTTPS. The bot token is this module's own secret, accepted from the operator; the
|
||||
// chat id is a setting. Neither is ever said: an error from the HTTP client carries the URL, and the
|
||||
// URL carries the token, so every error is rebuilt here from its kind before it leaves this file.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TelegramAPI is where the bot API answers; a test points it elsewhere.
|
||||
var TelegramAPI = "https://api.telegram.org"
|
||||
|
||||
var (
|
||||
tokenShape = regexp.MustCompile(`^\d{5,}:[A-Za-z0-9_-]{30,}$`)
|
||||
chatIDShape = regexp.MustCompile(`^(-?\d{1,20}|@[A-Za-z][A-Za-z0-9_]{4,})$`)
|
||||
)
|
||||
|
||||
// TelegramConfig is where the token is and what the chat is; read each time it is used, so a secret
|
||||
// accepted or a setting changed takes effect at the next message without a restart.
|
||||
type TelegramConfig struct {
|
||||
TokenFile string
|
||||
ChatID func() string
|
||||
}
|
||||
|
||||
// Telegram sends to one chat.
|
||||
type Telegram struct {
|
||||
Config TelegramConfig
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func NewTelegram(cfg TelegramConfig) *Telegram {
|
||||
return &Telegram{Config: cfg, Client: &http.Client{Timeout: 20 * time.Second}}
|
||||
}
|
||||
|
||||
func (t *Telegram) Name() string { return "telegram" }
|
||||
|
||||
// Ready says whether the channel can send, in words.
|
||||
func (t *Telegram) Ready() error {
|
||||
_, _, err := t.ready()
|
||||
return err
|
||||
}
|
||||
|
||||
// ready says whether the channel can send, and when not, what the operator must give. The words name
|
||||
// the setting and the secret, never a value.
|
||||
func (t *Telegram) ready() (string, string, error) {
|
||||
token, err := t.token()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
chat := strings.TrimSpace(t.Config.ChatID())
|
||||
if chat == "" {
|
||||
return "", "", errors.New("no chat to send to: the setting telegram-chat-id is not given " +
|
||||
"(`settings` for messenger with {\"telegram-chat-id\": \"<the chat's id>\"})")
|
||||
}
|
||||
if !chatIDShape.MatchString(chat) {
|
||||
return "", "", errors.New("the setting telegram-chat-id is not a chat id (a number, or @name of a channel)")
|
||||
}
|
||||
return token, chat, nil
|
||||
}
|
||||
|
||||
func (t *Telegram) token() (string, error) {
|
||||
if t.Config.TokenFile == "" {
|
||||
return "", errors.New("no bot token: this module was started without a token file")
|
||||
}
|
||||
raw, err := os.ReadFile(t.Config.TokenFile)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", errors.New("no bot token: the own secret telegram-token is not on this machine yet " +
|
||||
"(`secret accept <machine> messenger telegram-token`, then push the machine)")
|
||||
}
|
||||
if err != nil {
|
||||
return "", errors.New("the own secret telegram-token cannot be read: " + plainError(err))
|
||||
}
|
||||
token := strings.TrimSpace(string(raw))
|
||||
if token == "" {
|
||||
return "", errors.New("no bot token: the own secret telegram-token is empty")
|
||||
}
|
||||
if !tokenShape.MatchString(token) {
|
||||
// The mesh mints a random value for an own secret nobody accepted; that is not a bot token.
|
||||
return "", errors.New("the own secret telegram-token is not a bot token (digits, a colon, then the key " +
|
||||
"BotFather gave) — most likely the mesh made it because none was accepted: " +
|
||||
"`secret accept <machine> messenger telegram-token`, then push the machine")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// Send posts a message and answers its message id.
|
||||
func (t *Telegram) Send(m Message) (string, error) {
|
||||
text := m.Text()
|
||||
token, chat, err := t.ready()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var out struct {
|
||||
MessageID int64 `json:"message_id"`
|
||||
}
|
||||
if err := t.call(token, "sendMessage", map[string]any{
|
||||
"chat_id": chat, "text": text, "disable_web_page_preview": true,
|
||||
}, &out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprint(out.MessageID), nil
|
||||
}
|
||||
|
||||
// Edit replaces the text of a message sent before: how a cleared condition is said (to-be 45 §5).
|
||||
func (t *Telegram) Edit(id string, m Message) error {
|
||||
text := m.Text()
|
||||
token, chat, err := t.ready()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return t.call(token, "editMessageText", map[string]any{
|
||||
"chat_id": chat, "message_id": json.Number(id), "text": text, "disable_web_page_preview": true,
|
||||
}, nil)
|
||||
}
|
||||
|
||||
// CanEdit: Telegram edits a message in place.
|
||||
func (t *Telegram) CanEdit() bool { return true }
|
||||
|
||||
// Who asks the bot API who it is: the check that the token works, with no message sent.
|
||||
func (t *Telegram) Who() (string, error) {
|
||||
token, _, err := t.ready()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var me struct {
|
||||
Username string `json:"username"`
|
||||
}
|
||||
if err := t.call(token, "getMe", map[string]any{}, &me); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return me.Username, nil
|
||||
}
|
||||
|
||||
func (t *Telegram) call(token, method string, body map[string]any, into any) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
req, err := http.NewRequest(http.MethodPost, TelegramAPI+"/bot"+token+"/"+method, bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return errors.New("telegram " + method + ": the request could not be made")
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("telegram %s: %s", method, plainError(err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var answer struct {
|
||||
OK bool `json:"ok"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
Description string `json:"description"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&answer); err != nil {
|
||||
return fmt.Errorf("telegram %s: HTTP %d with an answer that is not the bot API's", method, resp.StatusCode)
|
||||
}
|
||||
if !answer.OK {
|
||||
d := strings.ReplaceAll(answer.Description, token, "<token>")
|
||||
return fmt.Errorf("telegram %s refused: %d %s", method, answer.ErrorCode, d)
|
||||
}
|
||||
if into != nil && len(answer.Result) > 0 {
|
||||
_ = json.Unmarshal(answer.Result, into)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// plainError is an error in words, without the URL a transport error carries.
|
||||
func plainError(err error) string {
|
||||
var ue *url.Error
|
||||
if errors.As(err, &ue) {
|
||||
err = ue.Err
|
||||
}
|
||||
var ne net.Error
|
||||
switch {
|
||||
case errors.As(err, &ne) && ne.Timeout():
|
||||
return "no answer in time"
|
||||
case errors.Is(err, os.ErrPermission):
|
||||
return "permission denied"
|
||||
}
|
||||
var dns *net.DNSError
|
||||
if errors.As(err, &dns) {
|
||||
return "the bot API's name does not resolve"
|
||||
}
|
||||
var op *net.OpError
|
||||
if errors.As(err, &op) {
|
||||
return "cannot connect (" + op.Op + ")"
|
||||
}
|
||||
s := err.Error()
|
||||
if strings.Contains(s, "/bot") || strings.Contains(s, "://") {
|
||||
return "the request failed"
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user