Merge pull request 'The operator-channel and the watcher's watcher (hq to-be 45 phase 1)' (#86) from feat/the-mesh-says-when-it-is-wrong into main
This commit was merged in pull request #86.
This commit is contained in:
@@ -4,3 +4,5 @@ dist/
|
||||
# Go tool bundles built in place (go build in a module's cmd/<name>-tools) are build output.
|
||||
modules/slack/cmd/slack-tools/slack-tools
|
||||
modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools
|
||||
modules/messenger/cmd/messenger/messenger
|
||||
modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# mesh-watcher
|
||||
|
||||
The watcher's watcher (novox/hq to-be 45 §5, signal S10, ADR 0227 rule 6): what tells the operator
|
||||
when the parts that would tell them are what failed.
|
||||
|
||||
- **Assigned to one machine that is not the control node.** It reads where the controller and the
|
||||
bus run (`mesh-controller.seats`) every ten minutes; on the same machine its status says
|
||||
`MISPLACED`.
|
||||
- **Watches two signals.**
|
||||
- **The self-check:** the controller's `doctor` heartbeat, `mesh-controller.doctor-heartbeat`.
|
||||
Bound: twice the interval the heartbeat says doctor runs at (five minutes when it says none).
|
||||
Heard by the time the heartbeat says it was made, so a backlog delivered after a restart is not a
|
||||
sign of life. The heartbeat is read in one place, `cmd/mesh-watcher/heartbeat.go`, which states
|
||||
every assumption it makes about its shape.
|
||||
- **The bus:** a round trip through the bus server — its own `watcher_ping` on its own machine —
|
||||
every minute. Bound: three minutes.
|
||||
- **Silent past its bound:** it sends to Telegram **directly over HTTPS, not through the bus**, once;
|
||||
once more an hour later if still silent; and again when the signal returns. Before a signal is
|
||||
first heard it counts from the watcher's start: a heartbeat that never comes is what this is for.
|
||||
- **Its own health is visible:** `watcher_status` leads with whether it can tell the operator anything
|
||||
at all (`BLIND` without a token or chat id, or while Telegram fails), whether it is misplaced,
|
||||
and whether heartbeats reach it. A message it could not send is owed and tried every minute.
|
||||
|
||||
## What the operator gives
|
||||
|
||||
1. **The bot token**, as this module's own secret: `secret accept <machine> mesh-watcher telegram-token`,
|
||||
then push that machine. The same bot as the operator-channel's is fine; it is one more machine
|
||||
holding it (ADR 0227, accepted for this one case).
|
||||
2. **The chat id**, as a setting: `settings` for `mesh-watcher` with `{"telegram-chat-id": "<id>"}`.
|
||||
|
||||
## Tools
|
||||
|
||||
| tool | does |
|
||||
|---|---|
|
||||
| `watcher_status` | its own health, then each signal: last heard, bound, silent, owed |
|
||||
| `watcher_last_heard` | when each signal was last heard, and what it sent lately |
|
||||
| `watcher_test` | a test message to Telegram now, directly |
|
||||
| `watcher_ping` | the bus round trip's other end |
|
||||
@@ -0,0 +1,114 @@
|
||||
package main
|
||||
|
||||
// The self-check's heartbeat, read in this one place (novox/hq to-be 45 §4, S10).
|
||||
//
|
||||
// **The contract this reads, and every assumption it makes**, because the controller's side was built
|
||||
// at the same time from the same design, which says only "every run ends with a heartbeat event
|
||||
// carrying the run's id and counts":
|
||||
//
|
||||
// - The event is the mesh-controller seat's own, `doctor-heartbeat`, consumed as
|
||||
// `mesh-controller.doctor-heartbeat` (subject `mesh.seat.mesh-controller.event.doctor-heartbeat`).
|
||||
// - The body is one JSON object: `run` (the run's id), `at` (when the run ended, RFC 3339),
|
||||
// `interval-seconds` (how often doctor runs), and `counts` ({pass, fail, failed-to-run}).
|
||||
// `finished`/`time` are read for `at`; `interval_seconds`, `interval` (seconds, or a duration such
|
||||
// as "5m") for the interval.
|
||||
// - Without a time, the heartbeat is taken as made when it arrived (said in the status as such).
|
||||
// Without an interval, the design's five minutes stand.
|
||||
// - The counts are kept for the status only. A run that found failures is still a heartbeat: the
|
||||
// watcher watches that the checker runs; what it finds is the controller's to raise.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// HeartbeatEvent is the event's local name under the controller's seat.
|
||||
const (
|
||||
HeartbeatEvent = "doctor-heartbeat"
|
||||
ControllerSeat = "mesh-controller"
|
||||
)
|
||||
|
||||
// Beat is one heartbeat.
|
||||
type Beat struct {
|
||||
Run string
|
||||
At time.Time
|
||||
Interval time.Duration
|
||||
Counts map[string]int
|
||||
}
|
||||
|
||||
// isHeartbeat says whether an envelope's key is the controller's heartbeat.
|
||||
func isHeartbeat(key string) bool { return key == ControllerSeat+"."+HeartbeatEvent }
|
||||
|
||||
// DecodeBeat reads one heartbeat's body.
|
||||
func DecodeBeat(body []byte) (Beat, error) {
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &raw); err != nil || raw == nil {
|
||||
return Beat{}, fmt.Errorf("%s: the body is not a JSON object", HeartbeatEvent)
|
||||
}
|
||||
var b Beat
|
||||
if v, ok := raw["run"]; ok {
|
||||
var s string
|
||||
var n float64
|
||||
switch {
|
||||
case json.Unmarshal(v, &s) == nil:
|
||||
b.Run = s
|
||||
case json.Unmarshal(v, &n) == nil:
|
||||
b.Run = fmt.Sprintf("%.0f", n)
|
||||
default:
|
||||
return Beat{}, fmt.Errorf("%s: run is neither a string nor a number", HeartbeatEvent)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"at", "finished", "time"} {
|
||||
v, ok := raw[name]
|
||||
if !ok || string(v) == "null" {
|
||||
continue
|
||||
}
|
||||
var s string
|
||||
if json.Unmarshal(v, &s) != nil {
|
||||
return Beat{}, fmt.Errorf("%s: %s is not a string", HeartbeatEvent, name)
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339Nano, s)
|
||||
if err != nil {
|
||||
return Beat{}, fmt.Errorf("%s: %s is not an RFC 3339 time: %q", HeartbeatEvent, name, s)
|
||||
}
|
||||
b.At = t
|
||||
break
|
||||
}
|
||||
for _, name := range []string{"interval-seconds", "interval_seconds", "interval"} {
|
||||
v, ok := raw[name]
|
||||
if !ok || string(v) == "null" {
|
||||
continue
|
||||
}
|
||||
var n float64
|
||||
var s string
|
||||
switch {
|
||||
case json.Unmarshal(v, &n) == nil:
|
||||
b.Interval = time.Duration(n * float64(time.Second))
|
||||
case json.Unmarshal(v, &s) == nil:
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return Beat{}, fmt.Errorf("%s: %s is not a duration: %q", HeartbeatEvent, name, s)
|
||||
}
|
||||
b.Interval = d
|
||||
default:
|
||||
return Beat{}, fmt.Errorf("%s: %s is neither seconds nor a duration", HeartbeatEvent, name)
|
||||
}
|
||||
if b.Interval < 0 || b.Interval > 24*time.Hour {
|
||||
return Beat{}, fmt.Errorf("%s: an interval of %s is not one doctor runs at", HeartbeatEvent, b.Interval)
|
||||
}
|
||||
break
|
||||
}
|
||||
if v, ok := raw["counts"]; ok && string(v) != "null" {
|
||||
var counts map[string]float64
|
||||
if json.Unmarshal(v, &counts) != nil {
|
||||
return Beat{}, fmt.Errorf("%s: counts is not an object of numbers", HeartbeatEvent)
|
||||
}
|
||||
b.Counts = map[string]int{}
|
||||
for k, n := range counts {
|
||||
b.Counts[k] = int(n)
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
// mesh-watcher: the watcher's watcher (novox/hq to-be 45 §5, S10, ADR 0227 rule 6). A Go bundle the
|
||||
// node's runtime launches on one machine that is not the control node. It hears the controller's
|
||||
// self-check heartbeat and makes a round trip through the bus every minute; when either goes silent
|
||||
// past its bound, it tells the operator on Telegram directly over HTTPS — the one sender that does not
|
||||
// pass through the control node — and tells them again when it returns. stdout is the MCP channel;
|
||||
// what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
func readChatID(path string) (string, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var s map[string]any
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return "", fmt.Errorf("the settings file is not JSON: %v", err)
|
||||
}
|
||||
switch v := s["telegram-chat-id"].(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(v), nil
|
||||
case float64:
|
||||
return fmt.Sprintf("%.0f", v), nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
type listening struct {
|
||||
mu sync.Mutex
|
||||
now string
|
||||
}
|
||||
|
||||
func (l *listening) set(s string) { l.mu.Lock(); l.now = s; l.mu.Unlock() }
|
||||
func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.now }
|
||||
|
||||
func main() {
|
||||
settings := os.Getenv("MESH_WATCHER_SETTINGS")
|
||||
var said sync.Mutex
|
||||
lastSaid := ""
|
||||
tg := NewTelegram(TelegramConfig{
|
||||
TokenFile: os.Getenv("MESH_WATCHER_TELEGRAM_TOKEN_FILE"),
|
||||
ChatID: func() string {
|
||||
id, err := readChatID(settings)
|
||||
said.Lock()
|
||||
defer said.Unlock()
|
||||
if err != nil && err.Error() != lastSaid {
|
||||
logf("[mesh-watcher] settings cannot be read: %v", err)
|
||||
}
|
||||
lastSaid = ""
|
||||
if err != nil {
|
||||
lastSaid = err.Error()
|
||||
}
|
||||
return id
|
||||
},
|
||||
})
|
||||
node := os.Getenv("MESH_NODE")
|
||||
w := NewWatcher(tg, time.Now, logf, node)
|
||||
l := &listening{now: "not yet: starting"}
|
||||
go run(w, l, node)
|
||||
if err := stdio.Serve("", tools(w, l)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// run keeps time, makes the round trips, reads where the controller is, and listens for heartbeats.
|
||||
// The clock runs whatever the bus does: it is what notices the bus is gone.
|
||||
func run(w *Watcher, l *listening, node string) {
|
||||
if err := w.Telegram.Ready(); err != nil {
|
||||
logf("[mesh-watcher] BLIND until given: %v", err)
|
||||
}
|
||||
go func() {
|
||||
for range time.Tick(time.Minute) {
|
||||
go func() { w.BusAnswered(roundTrip(node)) }()
|
||||
w.Tick()
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
time.Sleep(2 * time.Second)
|
||||
for {
|
||||
if nodes, err := controlNodes(); err == nil {
|
||||
w.Placed(nodes)
|
||||
} else {
|
||||
logf("[mesh-watcher] cannot read where the controller runs (%v); asking again in ten minutes", err)
|
||||
}
|
||||
time.Sleep(10 * time.Minute)
|
||||
}
|
||||
}()
|
||||
handle := func(e stdio.Envelope) error {
|
||||
if !isHeartbeat(e.Key) {
|
||||
return nil
|
||||
}
|
||||
b, err := DecodeBeat(e.Body)
|
||||
if err != nil {
|
||||
w.BadHeartbeat(err)
|
||||
return nil
|
||||
}
|
||||
w.Heartbeat(b)
|
||||
return nil
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := stdio.Subscribe(ControllerSeat+".*", handle)
|
||||
if err == nil {
|
||||
l.set("listening")
|
||||
logf("[mesh-watcher] listening for the self-check's heartbeat; watching from %s", node)
|
||||
return
|
||||
}
|
||||
l.set("not yet: " + err.Error())
|
||||
logf("[mesh-watcher] not hearing heartbeats yet (%v); asking again in %s", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
// roundTrip asks this module's own ping on this machine, through the bus server: an answer is the bus
|
||||
// carrying a request and its reply.
|
||||
func roundTrip(node string) error {
|
||||
key := "mesh-watcher.watcher_ping"
|
||||
if node != "" {
|
||||
key += "@" + node
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := stdio.Ask(key, map[string]any{})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-time.After(30 * time.Second):
|
||||
return fmt.Errorf("no answer in 30 s")
|
||||
}
|
||||
}
|
||||
|
||||
// controlNodes reads which machines hold the controller and the bus.
|
||||
func controlNodes() ([]string, error) {
|
||||
raw, err := stdio.Ask("seat:mesh-controller.seats", map[string]any{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return holdersOf(raw, "mesh-controller", "mesh-broker")
|
||||
}
|
||||
|
||||
// holdersOf reads the seats verb's answer, bare or inside a tool reply, for the named seats' machines.
|
||||
func holdersOf(raw json.RawMessage, seats ...string) ([]string, error) {
|
||||
var answer struct {
|
||||
Seats []struct {
|
||||
Seat string `json:"seat"`
|
||||
Holders []struct {
|
||||
Node string `json:"node"`
|
||||
} `json:"holders"`
|
||||
} `json:"seats"`
|
||||
Output string `json:"output"`
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
var s string
|
||||
if json.Unmarshal(raw, &s) == nil {
|
||||
return holdersOf(json.RawMessage(s), seats...)
|
||||
}
|
||||
return nil, fmt.Errorf("the seats answer is not JSON")
|
||||
}
|
||||
if len(answer.Seats) == 0 {
|
||||
switch {
|
||||
case answer.Output != "":
|
||||
return holdersOf(json.RawMessage(answer.Output), seats...)
|
||||
case len(answer.Content) > 0:
|
||||
return holdersOf(json.RawMessage(answer.Content[0].Text), seats...)
|
||||
}
|
||||
return nil, fmt.Errorf("the seats answer lists no seat")
|
||||
}
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
for _, s := range answer.Seats {
|
||||
for _, want := range seats {
|
||||
if s.Seat != want {
|
||||
continue
|
||||
}
|
||||
for _, h := range s.Holders {
|
||||
if !seen[h.Node] {
|
||||
seen[h.Node] = true
|
||||
out = append(out, h.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func tools(w *Watcher, l *listening) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: "watcher_status",
|
||||
Description: "The watcher's own health first — whether it can tell the operator anything (the Telegram token and " +
|
||||
"chat id), whether it runs on the machine it watches, whether heartbeats reach it — then each watched " +
|
||||
"signal (the controller's self-check heartbeat, a round trip through the bus): last heard, how long ago, " +
|
||||
"its bound, whether it is said silent, and any message not sent yet.",
|
||||
Run: func(map[string]any) (any, error) { return w.Status(l.get()), nil }},
|
||||
{Name: "watcher_last_heard",
|
||||
Description: "When each watched signal was last heard, and what the watcher sent to Telegram lately, newest first.",
|
||||
Run: func(map[string]any) (any, error) { return w.LastHeard(), nil }},
|
||||
{Name: "watcher_test",
|
||||
Description: "Send a test message to Telegram now, directly: proves the watcher can reach the operator when the " +
|
||||
"mesh cannot. Answers sent, or why not.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
err := w.send("test", Message{Title: "TEST: mesh-watcher on " + w.Node + " reaches you directly",
|
||||
Body: "nothing is wrong; this was asked for"})
|
||||
if err != nil {
|
||||
return map[string]string{"telegram": "not sent: " + err.Error()}, nil
|
||||
}
|
||||
return map[string]string{"telegram": "sent"}, nil
|
||||
}},
|
||||
{Name: "watcher_ping",
|
||||
Description: "Answers at once: the round trip the watcher makes through the bus every minute to know the bus carries a request and its reply.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return map[string]string{"pong": time.Now().UTC().Format(time.RFC3339)}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The manifest says what the code does: it consumes the heartbeat and nothing else, calls its own
|
||||
// ping and the controller's seats verb, holds no seat (it must not be the controller's), keeps its
|
||||
// Telegram token as its own secret, lists its own tools — and names nothing of one installation.
|
||||
|
||||
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Claims []any `json:"claims"`
|
||||
Own map[string]string `json:"own-secrets"`
|
||||
Tools []string `json:"tools"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(m.Consumes, []string{ControllerSeat + "." + HeartbeatEvent}) {
|
||||
t.Fatalf("consumes %v", m.Consumes)
|
||||
}
|
||||
if !reflect.DeepEqual(m.Invokes, []string{m.Module + ".watcher_ping", "seat:mesh-controller.seats"}) {
|
||||
t.Fatalf("invokes %v", m.Invokes)
|
||||
}
|
||||
if len(m.Claims) != 0 {
|
||||
t.Fatalf("a watcher holds no seat: %v", m.Claims)
|
||||
}
|
||||
env, _ := m.Build.Artifacts[0]["env"].(map[string]any)
|
||||
if m.Own["telegram-token"] == "" || env["MESH_WATCHER_TELEGRAM_TOKEN_FILE"] != m.Own["telegram-token"] {
|
||||
t.Fatalf("token: own %v, env %v", m.Own, env)
|
||||
}
|
||||
var served []string
|
||||
for _, tool := range tools(NewWatcher(&fakeTelegram{}, time.Now, t.Logf, ""), &listening{}) {
|
||||
served = append(served, tool.Name)
|
||||
if strings.TrimSpace(tool.Description) == "" {
|
||||
t.Errorf("%s has no description", tool.Name)
|
||||
}
|
||||
}
|
||||
listed := append([]string(nil), m.Tools...)
|
||||
sort.Strings(served)
|
||||
sort.Strings(listed)
|
||||
if !reflect.DeepEqual(served, listed) {
|
||||
t.Fatalf("serves %v, lists %v", served, listed)
|
||||
}
|
||||
for _, never := range []string{"/home/", "jochen", "g14", "shanks", "novox", "zurag", "api.telegram"} {
|
||||
if strings.Contains(strings.ToLower(string(raw)), never) {
|
||||
t.Errorf("module.json names %q", never)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package main
|
||||
|
||||
// Telegram, sent to directly over HTTPS — never through the bus or the control node (novox/hq to-be 45
|
||||
// §5): the one sender that does not pass through the control node. The same client as the
|
||||
// operator-channel's holder (module messenger), kept here so the watcher depends on nothing it
|
||||
// watches. The bot token is this module's own secret, accepted from the operator; the
|
||||
// chat id is a setting. Neither is ever said: an error from the HTTP client carries the URL, and the
|
||||
// URL carries the token, so every error is rebuilt here from its kind before it leaves this file.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TelegramAPI is where the bot API answers; a test points it elsewhere.
|
||||
var TelegramAPI = "https://api.telegram.org"
|
||||
|
||||
var (
|
||||
tokenShape = regexp.MustCompile(`^\d{5,}:[A-Za-z0-9_-]{30,}$`)
|
||||
chatIDShape = regexp.MustCompile(`^(-?\d{1,20}|@[A-Za-z][A-Za-z0-9_]{4,})$`)
|
||||
)
|
||||
|
||||
// TelegramConfig is where the token is and what the chat is; read each time it is used, so a secret
|
||||
// accepted or a setting changed takes effect at the next message without a restart.
|
||||
type TelegramConfig struct {
|
||||
TokenFile string
|
||||
ChatID func() string
|
||||
}
|
||||
|
||||
// Telegram sends to one chat.
|
||||
type Telegram struct {
|
||||
Config TelegramConfig
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func NewTelegram(cfg TelegramConfig) *Telegram {
|
||||
return &Telegram{Config: cfg, Client: &http.Client{Timeout: 20 * time.Second}}
|
||||
}
|
||||
|
||||
func (t *Telegram) Name() string { return "telegram" }
|
||||
|
||||
// Ready says whether the channel can send, in words.
|
||||
func (t *Telegram) Ready() error {
|
||||
_, _, err := t.ready()
|
||||
return err
|
||||
}
|
||||
|
||||
// ready says whether the channel can send, and when not, what the operator must give. The words name
|
||||
// the setting and the secret, never a value.
|
||||
func (t *Telegram) ready() (string, string, error) {
|
||||
token, err := t.token()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
chat := strings.TrimSpace(t.Config.ChatID())
|
||||
if chat == "" {
|
||||
return "", "", errors.New("no chat to send to: the setting telegram-chat-id is not given " +
|
||||
"(`settings` for mesh-watcher with {\"telegram-chat-id\": \"<the chat's id>\"})")
|
||||
}
|
||||
if !chatIDShape.MatchString(chat) {
|
||||
return "", "", errors.New("the setting telegram-chat-id is not a chat id (a number, or @name of a channel)")
|
||||
}
|
||||
return token, chat, nil
|
||||
}
|
||||
|
||||
func (t *Telegram) token() (string, error) {
|
||||
if t.Config.TokenFile == "" {
|
||||
return "", errors.New("no bot token: this module was started without a token file")
|
||||
}
|
||||
raw, err := os.ReadFile(t.Config.TokenFile)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", errors.New("no bot token: the own secret telegram-token is not on this machine yet " +
|
||||
"(`secret accept <machine> mesh-watcher telegram-token`, then push the machine)")
|
||||
}
|
||||
if err != nil {
|
||||
return "", errors.New("the own secret telegram-token cannot be read: " + plainError(err))
|
||||
}
|
||||
token := strings.TrimSpace(string(raw))
|
||||
if token == "" {
|
||||
return "", errors.New("no bot token: the own secret telegram-token is empty")
|
||||
}
|
||||
if !tokenShape.MatchString(token) {
|
||||
// The mesh mints a random value for an own secret nobody accepted; that is not a bot token.
|
||||
return "", errors.New("the own secret telegram-token is not a bot token (digits, a colon, then the key " +
|
||||
"BotFather gave) — most likely the mesh made it because none was accepted: " +
|
||||
"`secret accept <machine> mesh-watcher telegram-token`, then push the machine")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// Send posts a message and answers its message id.
|
||||
func (t *Telegram) Send(m Message) (string, error) {
|
||||
text := m.Text()
|
||||
token, chat, err := t.ready()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var out struct {
|
||||
MessageID int64 `json:"message_id"`
|
||||
}
|
||||
if err := t.call(token, "sendMessage", map[string]any{
|
||||
"chat_id": chat, "text": text, "disable_web_page_preview": true,
|
||||
}, &out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprint(out.MessageID), nil
|
||||
}
|
||||
|
||||
// Edit replaces the text of a message sent before: how a cleared condition is said (to-be 45 §5).
|
||||
func (t *Telegram) Edit(id string, m Message) error {
|
||||
text := m.Text()
|
||||
token, chat, err := t.ready()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return t.call(token, "editMessageText", map[string]any{
|
||||
"chat_id": chat, "message_id": json.Number(id), "text": text, "disable_web_page_preview": true,
|
||||
}, nil)
|
||||
}
|
||||
|
||||
// CanEdit: Telegram edits a message in place.
|
||||
func (t *Telegram) CanEdit() bool { return true }
|
||||
|
||||
// Who asks the bot API who it is: the check that the token works, with no message sent.
|
||||
func (t *Telegram) Who() (string, error) {
|
||||
token, _, err := t.ready()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var me struct {
|
||||
Username string `json:"username"`
|
||||
}
|
||||
if err := t.call(token, "getMe", map[string]any{}, &me); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return me.Username, nil
|
||||
}
|
||||
|
||||
func (t *Telegram) call(token, method string, body map[string]any, into any) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
req, err := http.NewRequest(http.MethodPost, TelegramAPI+"/bot"+token+"/"+method, bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return errors.New("telegram " + method + ": the request could not be made")
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("telegram %s: %s", method, plainError(err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var answer struct {
|
||||
OK bool `json:"ok"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
Description string `json:"description"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&answer); err != nil {
|
||||
return fmt.Errorf("telegram %s: HTTP %d with an answer that is not the bot API's", method, resp.StatusCode)
|
||||
}
|
||||
if !answer.OK {
|
||||
d := strings.ReplaceAll(answer.Description, token, "<token>")
|
||||
return fmt.Errorf("telegram %s refused: %d %s", method, answer.ErrorCode, d)
|
||||
}
|
||||
if into != nil && len(answer.Result) > 0 {
|
||||
_ = json.Unmarshal(answer.Result, into)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// plainError is an error in words, without the URL a transport error carries.
|
||||
func plainError(err error) string {
|
||||
var ue *url.Error
|
||||
if errors.As(err, &ue) {
|
||||
err = ue.Err
|
||||
}
|
||||
var ne net.Error
|
||||
switch {
|
||||
case errors.As(err, &ne) && ne.Timeout():
|
||||
return "no answer in time"
|
||||
case errors.Is(err, os.ErrPermission):
|
||||
return "permission denied"
|
||||
}
|
||||
var dns *net.DNSError
|
||||
if errors.As(err, &dns) {
|
||||
return "the bot API's name does not resolve"
|
||||
}
|
||||
var op *net.OpError
|
||||
if errors.As(err, &op) {
|
||||
return "cannot connect (" + op.Op + ")"
|
||||
}
|
||||
s := err.Error()
|
||||
if strings.Contains(s, "/bot") || strings.Contains(s, "://") {
|
||||
return "the request failed"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const goodToken = "123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsawQ"
|
||||
|
||||
func tokenFile(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "telegram-token")
|
||||
if content != "" {
|
||||
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestTelegramSaysWhatItLacks(t *testing.T) {
|
||||
for _, c := range []struct{ token, chat, says string }{
|
||||
{"", "42", "not on this machine yet"},
|
||||
{"r4nd0mlyMadeByTheMeshBecauseNobodyAcceptedOne", "42", "not a bot token"},
|
||||
{goodToken, "", "telegram-chat-id is not given"},
|
||||
{goodToken, "not a chat", "is not a chat id"},
|
||||
} {
|
||||
tg := NewTelegram(TelegramConfig{TokenFile: tokenFile(t, c.token), ChatID: func() string { return c.chat }})
|
||||
err := tg.Ready()
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%+v: %v", c, err)
|
||||
}
|
||||
if err != nil && strings.Contains(err.Error(), goodToken) {
|
||||
t.Errorf("the token is in the words")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramSendsEditsAndNeverSaysItsToken(t *testing.T) {
|
||||
var asked []string
|
||||
var bodies []map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
asked = append(asked, r.URL.Path)
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
var b map[string]any
|
||||
_ = json.Unmarshal(raw, &b)
|
||||
bodies = append(bodies, b)
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/sendMessage"):
|
||||
_, _ = w.Write([]byte(`{"ok":true,"result":{"message_id":77}}`))
|
||||
case strings.HasSuffix(r.URL.Path, "/editMessageText"):
|
||||
_, _ = w.Write([]byte(`{"ok":false,"error_code":400,"description":"Bad Request: message to edit not found"}`))
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"ok":true,"result":{"username":"mesh_bot"}}`))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
old := TelegramAPI
|
||||
TelegramAPI = srv.URL
|
||||
defer func() { TelegramAPI = old }()
|
||||
tg := NewTelegram(TelegramConfig{TokenFile: tokenFile(t, goodToken+"\n"), ChatID: func() string { return "-1001" }})
|
||||
id, err := tg.Send(Message{Title: "URGENT: x", Body: "key: a.b.c"})
|
||||
if err != nil || id != "77" {
|
||||
t.Fatalf("%q %v", id, err)
|
||||
}
|
||||
if asked[0] != "/bot"+goodToken+"/sendMessage" || bodies[0]["chat_id"] != "-1001" || bodies[0]["text"] != "URGENT: x\nkey: a.b.c" {
|
||||
t.Fatalf("%v %v", asked, bodies[0])
|
||||
}
|
||||
err = tg.Edit("77", Message{Title: "CLEARED"})
|
||||
if err == nil || !strings.Contains(err.Error(), "message to edit not found") || strings.Contains(err.Error(), goodToken) {
|
||||
t.Fatalf("edit: %v", err)
|
||||
}
|
||||
if bodies[1]["message_id"] != float64(77) {
|
||||
t.Fatalf("message id: %v", bodies[1]["message_id"])
|
||||
}
|
||||
if who, err := tg.Who(); err != nil || who != "mesh_bot" {
|
||||
t.Fatalf("%q %v", who, err)
|
||||
}
|
||||
// Nothing answers: the error is in words, without the URL that carries the token.
|
||||
srv.Close()
|
||||
_, err = tg.Send(Message{Title: "x"})
|
||||
if err == nil || strings.Contains(err.Error(), goodToken) || strings.Contains(err.Error(), "/bot") {
|
||||
t.Fatalf("unreachable: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
package main
|
||||
|
||||
// The watcher's watcher (novox/hq to-be 45 §5, signal S10, ADR 0227 rule 6): on a machine that is not
|
||||
// the control node, it listens for the self-check's heartbeat and for the bus itself. When either has
|
||||
// been silent past its bound it sends to Telegram directly over HTTPS — not through the bus — says so
|
||||
// once more an hour later if it still is, and says so again when it returns.
|
||||
//
|
||||
// Two signals:
|
||||
//
|
||||
// - self-check: the controller's `doctor` heartbeat. Bound: twice the interval the heartbeat says
|
||||
// it runs at, else twice five minutes (to-be 45 §3, S10). Heard by the time the heartbeat says it
|
||||
// was made, not by when it arrived: a backlog delivered after a restart is old news, not a sign of
|
||||
// life.
|
||||
// - bus: a round trip through the bus server — this module asking its own tool on its own machine
|
||||
// — every minute. Bound: three minutes.
|
||||
//
|
||||
// Before a signal is first heard it counts from when the watcher started: a heartbeat that never comes
|
||||
// is exactly what this is for.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
SelfCheck = "self-check"
|
||||
Bus = "bus"
|
||||
DefaultInterval = 5 * time.Minute
|
||||
BusBound = 3 * time.Minute
|
||||
RemindAfter = time.Hour
|
||||
// Skew is how far in the future a heartbeat's time may be and still be believed.
|
||||
Skew = 2 * time.Minute
|
||||
)
|
||||
|
||||
// Message is one thing said to the operator.
|
||||
type Message struct {
|
||||
Title string
|
||||
Body string
|
||||
}
|
||||
|
||||
func (m Message) Text() string {
|
||||
if m.Body == "" {
|
||||
return m.Title
|
||||
}
|
||||
return m.Title + "\n" + m.Body
|
||||
}
|
||||
|
||||
// Sender is the Telegram channel.
|
||||
type Sender interface {
|
||||
Ready() error
|
||||
Send(Message) (string, error)
|
||||
}
|
||||
|
||||
type signal struct {
|
||||
name string
|
||||
lastHeard time.Time
|
||||
bound time.Duration
|
||||
silentAt time.Time // when it was said silent; zero while it is heard
|
||||
heardBefore time.Time // the last word before it went silent
|
||||
reminded bool
|
||||
owed *Message // a message that could not be sent yet
|
||||
lastRun string // the heartbeat's run id, for the status
|
||||
lastErr string // the bus probe's last error
|
||||
}
|
||||
|
||||
// Watcher watches.
|
||||
type Watcher struct {
|
||||
Telegram Sender
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
Node string
|
||||
|
||||
mu sync.Mutex
|
||||
started time.Time
|
||||
signals map[string]*signal
|
||||
sent []sentNote
|
||||
sendErr string
|
||||
sendOK time.Time
|
||||
misplace string // why this machine is the wrong one to watch from, or ""
|
||||
control string // the machine the controller is on, as last read
|
||||
badBeats int
|
||||
lastBad string
|
||||
}
|
||||
|
||||
type sentNote struct {
|
||||
At time.Time `json:"at"`
|
||||
Signal string `json:"signal"`
|
||||
What string `json:"what"`
|
||||
Outcome string `json:"outcome"`
|
||||
}
|
||||
|
||||
func NewWatcher(tg Sender, now func() time.Time, logf func(string, ...any), node string) *Watcher {
|
||||
w := &Watcher{Telegram: tg, Now: now, Logf: logf, Node: node, started: now(), signals: map[string]*signal{
|
||||
SelfCheck: {name: SelfCheck, bound: 2 * DefaultInterval},
|
||||
Bus: {name: Bus, bound: BusBound},
|
||||
}}
|
||||
return w
|
||||
}
|
||||
|
||||
// Heartbeat takes one self-check heartbeat.
|
||||
func (w *Watcher) Heartbeat(b Beat) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
now := w.Now()
|
||||
s := w.signals[SelfCheck]
|
||||
at := b.At
|
||||
if at.IsZero() {
|
||||
at = now
|
||||
}
|
||||
if at.After(now.Add(Skew)) {
|
||||
w.badBeats++
|
||||
w.lastBad = fmt.Sprintf("a heartbeat from the future (%s), not believed", at.UTC().Format(time.RFC3339))
|
||||
w.Logf("[mesh-watcher] %s", w.lastBad)
|
||||
return
|
||||
}
|
||||
if at.After(s.lastHeard) {
|
||||
s.lastHeard = at
|
||||
s.lastRun = b.Run
|
||||
}
|
||||
if b.Interval > 0 {
|
||||
s.bound = 2 * b.Interval
|
||||
}
|
||||
}
|
||||
|
||||
// BadHeartbeat counts a heartbeat that could not be read: said, never read as a sign of life.
|
||||
func (w *Watcher) BadHeartbeat(err error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.badBeats++
|
||||
w.lastBad = err.Error()
|
||||
w.Logf("[mesh-watcher] a heartbeat could not be read (not counted as heard): %v", err)
|
||||
}
|
||||
|
||||
// BusAnswered takes the outcome of one round trip through the bus.
|
||||
func (w *Watcher) BusAnswered(err error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
s := w.signals[Bus]
|
||||
if err != nil {
|
||||
if s.lastErr != err.Error() {
|
||||
w.Logf("[mesh-watcher] the bus did not answer a round trip: %v", err)
|
||||
}
|
||||
s.lastErr = err.Error()
|
||||
return
|
||||
}
|
||||
s.lastErr = ""
|
||||
s.lastHeard = w.Now()
|
||||
}
|
||||
|
||||
// Placed records where the controller runs, and says when that is this machine.
|
||||
func (w *Watcher) Placed(controlNodes []string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.control = ""
|
||||
was := w.misplace
|
||||
w.misplace = ""
|
||||
for _, n := range controlNodes {
|
||||
if w.control != "" {
|
||||
w.control += ", "
|
||||
}
|
||||
w.control += n
|
||||
if n == w.Node && w.Node != "" {
|
||||
w.misplace = "this machine holds " + n + "'s controller or bus: a watcher here goes down with what it watches; assign mesh-watcher to another machine"
|
||||
}
|
||||
}
|
||||
if w.misplace != "" && was == "" {
|
||||
w.Logf("[mesh-watcher] %s", w.misplace)
|
||||
}
|
||||
}
|
||||
|
||||
// Tick decides what is silent, what returned, and sends what is owed.
|
||||
func (w *Watcher) Tick() {
|
||||
w.mu.Lock()
|
||||
now := w.Now()
|
||||
var out []struct {
|
||||
s *signal
|
||||
m Message
|
||||
w string
|
||||
}
|
||||
for _, name := range []string{SelfCheck, Bus} {
|
||||
s := w.signals[name]
|
||||
since := s.lastHeard
|
||||
if since.IsZero() {
|
||||
since = w.started
|
||||
}
|
||||
silent := now.Sub(since)
|
||||
switch {
|
||||
case s.silentAt.IsZero() && silent > s.bound:
|
||||
s.silentAt, s.reminded, s.heardBefore = now, false, since
|
||||
m := w.silentMessage(s, silent, false)
|
||||
s.owed = &m
|
||||
case !s.silentAt.IsZero() && silent <= s.bound:
|
||||
m := Message{
|
||||
Title: "CLEARED: " + w.what(s) + " heard again",
|
||||
Body: "silent for about " + roughly(s.lastHeard.Sub(s.heardBefore)) + "; told by mesh-watcher on " +
|
||||
w.Node + ", directly over HTTPS",
|
||||
}
|
||||
s.silentAt = time.Time{}
|
||||
s.owed = &m
|
||||
case !s.silentAt.IsZero() && !s.reminded && now.Sub(s.silentAt) >= RemindAfter:
|
||||
s.reminded = true
|
||||
m := w.silentMessage(s, silent, true)
|
||||
s.owed = &m
|
||||
}
|
||||
if s.owed != nil {
|
||||
out = append(out, struct {
|
||||
s *signal
|
||||
m Message
|
||||
w string
|
||||
}{s, *s.owed, name})
|
||||
}
|
||||
}
|
||||
w.mu.Unlock()
|
||||
for _, o := range out {
|
||||
err := w.send(o.w, o.m)
|
||||
w.mu.Lock()
|
||||
if err == nil {
|
||||
o.s.owed = nil
|
||||
}
|
||||
w.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) what(s *signal) string {
|
||||
if s.name == SelfCheck {
|
||||
return "the controller's self-check (doctor)"
|
||||
}
|
||||
return "the bus"
|
||||
}
|
||||
|
||||
func (w *Watcher) silentMessage(s *signal, silent time.Duration, again bool) Message {
|
||||
title := "URGENT: self-check-silent: " + w.what(s) + " has not been heard for " + roughly(silent)
|
||||
if s.name == Bus {
|
||||
title = "URGENT: bus-silent: " + w.what(s) + " has not answered a round trip for " + roughly(silent)
|
||||
}
|
||||
if again {
|
||||
title = "STILL SILENT: " + title[len("URGENT: "):]
|
||||
}
|
||||
body := "bound: " + roughly(s.bound) + "; told by mesh-watcher on " + w.Node + ", directly over HTTPS, not through the mesh"
|
||||
if s.lastHeard.IsZero() {
|
||||
body += "\nnot heard once since the watcher started"
|
||||
} else {
|
||||
body += "\nlast heard: " + s.lastHeard.UTC().Format("2006-01-02 15:04") + " UTC"
|
||||
}
|
||||
if s.name == SelfCheck {
|
||||
body += "\nthe controller, the control node or the bus may be down; the mesh's own messages pass through them"
|
||||
}
|
||||
return Message{Title: title, Body: body}
|
||||
}
|
||||
|
||||
func (w *Watcher) send(signalName string, m Message) error {
|
||||
_, err := w.Telegram.Send(m)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
note := sentNote{At: w.Now(), Signal: signalName, What: m.Title, Outcome: "sent"}
|
||||
if err != nil {
|
||||
note.Outcome = "failed: " + err.Error()
|
||||
if w.sendErr != err.Error() {
|
||||
w.Logf("[mesh-watcher] cannot send to Telegram (tried again every minute): %v", err)
|
||||
}
|
||||
w.sendErr = err.Error()
|
||||
} else {
|
||||
if w.sendErr != "" {
|
||||
w.Logf("[mesh-watcher] Telegram sends again")
|
||||
}
|
||||
w.sendErr = ""
|
||||
w.sendOK = w.Now()
|
||||
w.Logf("[mesh-watcher] told the operator: %s", m.Title)
|
||||
}
|
||||
w.sent = append(w.sent, note)
|
||||
if len(w.sent) > 100 {
|
||||
w.sent = w.sent[len(w.sent)-100:]
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// SignalStatus is one signal as the status says it.
|
||||
type SignalStatus struct {
|
||||
Signal string `json:"signal"`
|
||||
LastHeard string `json:"last_heard"`
|
||||
Ago string `json:"ago"`
|
||||
Bound string `json:"bound"`
|
||||
Silent bool `json:"silent"`
|
||||
SaidSilent string `json:"said_silent_at,omitempty"`
|
||||
Owed string `json:"not_sent_yet,omitempty"`
|
||||
LastRun string `json:"last_run,omitempty"`
|
||||
LastFailure string `json:"last_failure,omitempty"`
|
||||
}
|
||||
|
||||
// Status is the watcher's account of itself, its own health first.
|
||||
type Status struct {
|
||||
Verdict string `json:"verdict"`
|
||||
Node string `json:"watching_from"`
|
||||
Control string `json:"controller_and_bus_on,omitempty"`
|
||||
Misplaced string `json:"misplaced,omitempty"`
|
||||
Telegram string `json:"telegram"`
|
||||
LastSent string `json:"last_sent,omitempty"`
|
||||
Signals []SignalStatus `json:"signals"`
|
||||
BadBeats int `json:"unreadable_heartbeats"`
|
||||
LastBad string `json:"last_unreadable,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Listening string `json:"listening"`
|
||||
}
|
||||
|
||||
func (w *Watcher) Status(listening string) Status {
|
||||
ready := w.Telegram.Ready()
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
now := w.Now()
|
||||
st := Status{Node: w.Node, Control: w.control, Misplaced: w.misplace, Started: w.started.UTC().Format(time.RFC3339),
|
||||
BadBeats: w.badBeats, LastBad: w.lastBad, Listening: listening}
|
||||
switch {
|
||||
case ready != nil:
|
||||
st.Telegram = "CANNOT SEND: " + ready.Error()
|
||||
case w.sendErr != "":
|
||||
st.Telegram = "FAILING: " + w.sendErr
|
||||
default:
|
||||
st.Telegram = "ready"
|
||||
}
|
||||
if !w.sendOK.IsZero() {
|
||||
st.LastSent = w.sendOK.UTC().Format(time.RFC3339)
|
||||
}
|
||||
anySilent := false
|
||||
for _, name := range []string{SelfCheck, Bus} {
|
||||
s := w.signals[name]
|
||||
ss := SignalStatus{Signal: name, Bound: roughly(s.bound), Silent: !s.silentAt.IsZero(), LastRun: s.lastRun, LastFailure: s.lastErr}
|
||||
if s.lastHeard.IsZero() {
|
||||
ss.LastHeard = "never since start"
|
||||
ss.Ago = roughly(now.Sub(w.started)) + " since start"
|
||||
} else {
|
||||
ss.LastHeard = s.lastHeard.UTC().Format(time.RFC3339)
|
||||
ss.Ago = roughly(now.Sub(s.lastHeard))
|
||||
}
|
||||
if ss.Silent {
|
||||
ss.SaidSilent = s.silentAt.UTC().Format(time.RFC3339)
|
||||
anySilent = true
|
||||
}
|
||||
if s.owed != nil {
|
||||
ss.Owed = s.owed.Title
|
||||
}
|
||||
st.Signals = append(st.Signals, ss)
|
||||
}
|
||||
switch {
|
||||
case st.Telegram != "ready":
|
||||
st.Verdict = "BLIND: the watcher cannot tell the operator anything — " + st.Telegram
|
||||
case w.misplace != "":
|
||||
st.Verdict = "MISPLACED: " + w.misplace
|
||||
case listening != "listening":
|
||||
st.Verdict = "NOT LISTENING: heartbeats cannot reach the watcher — " + listening
|
||||
case anySilent:
|
||||
st.Verdict = "ALARM: a watched signal is silent — see signals"
|
||||
default:
|
||||
st.Verdict = "ok: watching"
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// LastHeard is each signal's last word, and the recent sends.
|
||||
func (w *Watcher) LastHeard() map[string]any {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
now := w.Now()
|
||||
out := map[string]any{}
|
||||
for name, s := range w.signals {
|
||||
if s.lastHeard.IsZero() {
|
||||
out[name] = "never since the watcher started " + roughly(now.Sub(w.started)) + " ago"
|
||||
} else {
|
||||
out[name] = s.lastHeard.UTC().Format(time.RFC3339) + " (" + roughly(now.Sub(s.lastHeard)) + " ago)"
|
||||
}
|
||||
}
|
||||
sent := make([]sentNote, 0, len(w.sent))
|
||||
for i := len(w.sent) - 1; i >= 0; i-- {
|
||||
sent = append(sent, w.sent[i])
|
||||
}
|
||||
out["sent"] = sent
|
||||
return out
|
||||
}
|
||||
|
||||
func roughly(d time.Duration) string {
|
||||
switch {
|
||||
case d < 0:
|
||||
return "a moment"
|
||||
case d < 2*time.Minute:
|
||||
return fmt.Sprintf("%d s", int(d.Seconds()))
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d min", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%.1f h", d.Hours())
|
||||
}
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeTelegram struct {
|
||||
notReady error
|
||||
fail error
|
||||
sent []Message
|
||||
}
|
||||
|
||||
func (f *fakeTelegram) Ready() error { return f.notReady }
|
||||
func (f *fakeTelegram) Send(m Message) (string, error) {
|
||||
if f.fail != nil {
|
||||
return "", f.fail
|
||||
}
|
||||
f.sent = append(f.sent, m)
|
||||
return "1", nil
|
||||
}
|
||||
|
||||
type clock struct{ t time.Time }
|
||||
|
||||
func (c *clock) now() time.Time { return c.t }
|
||||
func (c *clock) pass(d time.Duration) { c.t = c.t.Add(d) }
|
||||
|
||||
func watcher(t *testing.T) (*Watcher, *fakeTelegram, *clock) {
|
||||
c := &clock{t: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)}
|
||||
tg := &fakeTelegram{}
|
||||
return NewWatcher(tg, c.now, t.Logf, "ace"), tg, c
|
||||
}
|
||||
|
||||
// beatEvery has the controller's doctor run, and the bus answer, every five minutes up to d.
|
||||
func beatEvery(w *Watcher, c *clock, d time.Duration) {
|
||||
for end := c.t.Add(d); c.t.Before(end); {
|
||||
c.pass(time.Minute)
|
||||
if c.t.Minute()%5 == 0 {
|
||||
w.Heartbeat(Beat{Run: "r", At: c.t, Interval: 5 * time.Minute})
|
||||
}
|
||||
w.BusAnswered(nil)
|
||||
w.Tick()
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeardSignalsSayNothing(t *testing.T) {
|
||||
w, tg, c := watcher(t)
|
||||
beatEvery(w, c, 3*time.Hour)
|
||||
if len(tg.sent) != 0 {
|
||||
t.Fatalf("sent %v", tg.sent)
|
||||
}
|
||||
if st := w.Status("listening"); st.Verdict != "ok: watching" {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSelfCheckSilentPastTwiceItsIntervalIsSentOnceThenOnceMoreThenItsReturn(t *testing.T) {
|
||||
w, tg, c := watcher(t)
|
||||
beatEvery(w, c, 30*time.Minute)
|
||||
last := c.t
|
||||
// The controller stops; the bus still answers.
|
||||
for i := 0; i < 9; i++ {
|
||||
c.pass(time.Minute)
|
||||
w.BusAnswered(nil)
|
||||
w.Tick()
|
||||
}
|
||||
if len(tg.sent) != 0 {
|
||||
t.Fatalf("said silent within its bound (9 min of 10): %v", tg.sent)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
c.pass(time.Minute)
|
||||
w.BusAnswered(nil)
|
||||
w.Tick()
|
||||
}
|
||||
if len(tg.sent) != 1 || !strings.Contains(tg.sent[0].Title, "self-check-silent") {
|
||||
t.Fatalf("not said silent past twice the interval: %v", tg.sent)
|
||||
}
|
||||
if !strings.Contains(tg.sent[0].Body, "directly over HTTPS") || !strings.Contains(tg.sent[0].Body, last.Format("15:04")) {
|
||||
t.Fatalf("body: %q", tg.sent[0].Body)
|
||||
}
|
||||
if st := w.Status("listening"); !strings.HasPrefix(st.Verdict, "ALARM") {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
for i := 0; i < 70; i++ {
|
||||
c.pass(time.Minute)
|
||||
w.BusAnswered(nil)
|
||||
w.Tick()
|
||||
}
|
||||
if len(tg.sent) != 2 || !strings.HasPrefix(tg.sent[1].Title, "STILL SILENT") {
|
||||
t.Fatalf("not said once more after an hour: %v", tg.sent)
|
||||
}
|
||||
beatEvery(w, c, 10*time.Minute)
|
||||
if len(tg.sent) != 3 || !strings.HasPrefix(tg.sent[2].Title, "CLEARED") {
|
||||
t.Fatalf("its return not said: %v", tg.sent)
|
||||
}
|
||||
beatEvery(w, c, time.Hour)
|
||||
if len(tg.sent) != 3 {
|
||||
t.Fatalf("said again after it returned: %v", tg.sent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeartbeatNeverHeardIsSilenceFromTheStart(t *testing.T) {
|
||||
w, tg, c := watcher(t)
|
||||
for i := 0; i < 11; i++ {
|
||||
c.pass(time.Minute)
|
||||
w.BusAnswered(nil)
|
||||
w.Tick()
|
||||
}
|
||||
if len(tg.sent) != 1 || !strings.Contains(tg.sent[0].Body, "not heard once since the watcher started") {
|
||||
t.Fatalf("%v", tg.sent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplayedOldHeartbeatIsNotASignOfLife(t *testing.T) {
|
||||
w, tg, c := watcher(t)
|
||||
beatEvery(w, c, 10*time.Minute)
|
||||
old := c.t
|
||||
for i := 0; i < 11; i++ {
|
||||
c.pass(time.Minute)
|
||||
w.BusAnswered(nil)
|
||||
w.Tick()
|
||||
}
|
||||
// The watcher's consumer delivers a backlog: heartbeats made before the silence.
|
||||
w.Heartbeat(Beat{Run: "old", At: old.Add(-time.Minute)})
|
||||
w.Heartbeat(Beat{Run: "older", At: old.Add(-10 * time.Minute)})
|
||||
c.pass(time.Minute)
|
||||
w.Tick()
|
||||
if len(tg.sent) != 1 {
|
||||
t.Fatalf("a replay read as a return: %v", tg.sent)
|
||||
}
|
||||
w.Heartbeat(Beat{Run: "future", At: c.t.Add(time.Hour)})
|
||||
c.pass(time.Minute)
|
||||
w.Tick()
|
||||
if len(tg.sent) != 1 || w.Status("listening").BadBeats != 1 {
|
||||
t.Fatalf("a heartbeat from the future believed: %v", tg.sent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBoundFollowsTheIntervalTheHeartbeatSays(t *testing.T) {
|
||||
w, tg, c := watcher(t)
|
||||
w.Heartbeat(Beat{At: c.t, Interval: 15 * time.Minute})
|
||||
for i := 0; i < 29; i++ {
|
||||
c.pass(time.Minute)
|
||||
w.BusAnswered(nil)
|
||||
w.Tick()
|
||||
}
|
||||
if len(tg.sent) != 0 {
|
||||
t.Fatalf("said silent inside 2 × 15 min: %v", tg.sent)
|
||||
}
|
||||
c.pass(2 * time.Minute)
|
||||
w.BusAnswered(nil)
|
||||
w.Tick()
|
||||
if len(tg.sent) != 1 {
|
||||
t.Fatalf("not said past 2 × 15 min")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBusSilentPastThreeMinutesIsSent(t *testing.T) {
|
||||
w, tg, c := watcher(t)
|
||||
beatEvery(w, c, 10*time.Minute)
|
||||
for i := 0; i < 4; i++ {
|
||||
c.pass(time.Minute)
|
||||
w.BusAnswered(errors.New("timeout"))
|
||||
w.Tick()
|
||||
}
|
||||
if len(tg.sent) != 1 || !strings.Contains(tg.sent[0].Title, "bus-silent") {
|
||||
t.Fatalf("%v", tg.sent)
|
||||
}
|
||||
if st := w.Status("listening"); st.Signals[1].LastFailure != "timeout" {
|
||||
t.Fatalf("%+v", st.Signals[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestATelegramThatCannotSendIsSaidAndTheMessageIsOwed(t *testing.T) {
|
||||
w, tg, c := watcher(t)
|
||||
tg.fail = errors.New("telegram sendMessage: cannot connect (dial)")
|
||||
for i := 0; i < 11; i++ {
|
||||
c.pass(time.Minute)
|
||||
w.BusAnswered(nil)
|
||||
w.Tick()
|
||||
}
|
||||
st := w.Status("listening")
|
||||
if !strings.HasPrefix(st.Verdict, "BLIND") || st.Signals[0].Owed == "" {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
tg.fail = nil
|
||||
c.pass(time.Minute)
|
||||
w.Tick()
|
||||
if len(tg.sent) != 1 || w.Status("listening").Signals[0].Owed != "" {
|
||||
t.Fatalf("the owed message was not sent: %v", tg.sent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotGivenATokenIsBlind(t *testing.T) {
|
||||
w, tg, _ := watcher(t)
|
||||
tg.notReady = errors.New("no bot token")
|
||||
if st := w.Status("listening"); st.Verdict != "BLIND: the watcher cannot tell the operator anything — CANNOT SEND: no bot token" {
|
||||
t.Fatalf("%q", st.Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnTheControlNodeItSaysItIsMisplaced(t *testing.T) {
|
||||
w, _, _ := watcher(t)
|
||||
w.Placed([]string{"novox"})
|
||||
if st := w.Status("listening"); st.Verdict != "ok: watching" || st.Control != "novox" {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
w.Placed([]string{"ace"})
|
||||
if st := w.Status("listening"); !strings.HasPrefix(st.Verdict, "MISPLACED") {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSeatsAnswerIsReadInEitherShape(t *testing.T) {
|
||||
bare := `{"seats":[{"seat":"mesh-controller","holders":[{"node":"n1","module":"mesh-controller"}]},
|
||||
{"seat":"mesh-broker","holders":[{"node":"n1"},{"node":"n2"}]},{"seat":"git","holders":[{"node":"n3"}]}]}`
|
||||
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]string{{"type": "text", "text": bare}}})
|
||||
output, _ := json.Marshal(map[string]any{"ok": true, "output": bare})
|
||||
for _, raw := range []string{bare, string(wrapped), string(output)} {
|
||||
got, err := holdersOf(json.RawMessage(raw), "mesh-controller", "mesh-broker")
|
||||
if err != nil || strings.Join(got, ",") != "n1,n2" {
|
||||
t.Errorf("%s: %v %v", raw, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheHeartbeatAsTheDesignSaysIt(t *testing.T) {
|
||||
b, err := DecodeBeat([]byte(`{"run":"doctor-41","at":"2026-10-06T12:05:00Z","interval-seconds":300,
|
||||
"counts":{"pass":10,"fail":1,"failed-to-run":0}}`))
|
||||
if err != nil || b.Run != "doctor-41" || b.Interval != 5*time.Minute || b.Counts["fail"] != 1 ||
|
||||
!b.At.Equal(time.Date(2026, 10, 6, 12, 5, 0, 0, time.UTC)) {
|
||||
t.Fatalf("%+v %v", b, err)
|
||||
}
|
||||
b, err = DecodeBeat([]byte(`{"run":7,"finished":"2026-10-06T12:05:00Z","interval":"5m"}`))
|
||||
if err != nil || b.Run != "7" || b.Interval != 5*time.Minute || b.At.IsZero() {
|
||||
t.Fatalf("%+v %v", b, err)
|
||||
}
|
||||
if b, err := DecodeBeat([]byte(`{}`)); err != nil || !b.At.IsZero() {
|
||||
t.Fatalf("an empty heartbeat: %+v %v", b, err)
|
||||
}
|
||||
for body, says := range map[string]string{
|
||||
`[]`: "not a JSON object",
|
||||
`{"at":"yesterday"}`: "not an RFC 3339 time",
|
||||
`{"interval":"often"}`: "not a duration",
|
||||
`{"interval-seconds":-5}`: "not one doctor runs at",
|
||||
`{"counts":"many"}`: "counts",
|
||||
} {
|
||||
if _, err := DecodeBeat([]byte(body)); err == nil || !strings.Contains(err.Error(), says) {
|
||||
t.Errorf("%s: %v", body, err)
|
||||
}
|
||||
}
|
||||
if !isHeartbeat("mesh-controller.doctor-heartbeat") || isHeartbeat("mesh-controller.applied") {
|
||||
t.Fatalf("the heartbeat's key")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module mesh-watcher
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.7
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
Executable
BIN
Binary file not shown.
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"module": "mesh-watcher",
|
||||
"version": "1",
|
||||
"slug": "watch",
|
||||
"consumes": [
|
||||
"mesh-controller.doctor-heartbeat"
|
||||
],
|
||||
"invokes": [
|
||||
"mesh-watcher.watcher_ping",
|
||||
"seat:mesh-controller.seats"
|
||||
],
|
||||
"own-secrets": {
|
||||
"telegram-token": "${dir:state}/telegram-token"
|
||||
},
|
||||
"tools": [
|
||||
"watcher_status",
|
||||
"watcher_last_heard",
|
||||
"watcher_test",
|
||||
"watcher_ping"
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": "."
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"merge": "json",
|
||||
"content": "{\n \"telegram-chat-id\": \"\"\n}\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-watcher",
|
||||
"binary": "mesh-watcher",
|
||||
"loads": [
|
||||
"mesh-watcher"
|
||||
],
|
||||
"env": {
|
||||
"MESH_WATCHER_SETTINGS": "${dir:state}/settings.json",
|
||||
"MESH_WATCHER_TELEGRAM_TOKEN_FILE": "${dir:state}/telegram-token"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
# messenger
|
||||
|
||||
The holder of the `operator-channel` seat: how the mesh tells its operator what it noticed (novox/hq
|
||||
to-be 45 §5, ADR 0227, research 028 — the minimal form, Q1a, Q5a, Q8).
|
||||
|
||||
- **Declares and holds `operator-channel`**, held once for the mesh, serving `open`, `history` and
|
||||
`notify`.
|
||||
- **Consumes the controller's condition events** — `mesh-controller.condition-raised`, `-changed`,
|
||||
`-cleared` — and decides what is sent. The controller calls nobody. The events are read in one
|
||||
place, `cmd/messenger/condition.go`, which states every assumption it makes about their shape.
|
||||
- **Two channels:** Telegram (a bot to the operator's chat) and the desktop notifier — the
|
||||
`node-notifier` seat's `send` verb on the machine the operator sits at (ADR 0208), asked through the
|
||||
mesh. Nothing new runs on that machine.
|
||||
- **Keeps its open messages in its own state** (`open`; the recent sends in `sent`), so a restart
|
||||
forgets nothing (ADR 0201).
|
||||
|
||||
## What is sent, and when
|
||||
|
||||
| When | What |
|
||||
|---|---|
|
||||
| `condition-raised` | one message, deduplicated by the condition's key |
|
||||
| still open after 1 h (urgent) or 12 h (warning) | once more |
|
||||
| `condition-changed` from warning to urgent | once more, to both channels |
|
||||
| `condition-cleared` | the first message edited to say so (both channels can) |
|
||||
| cleared and raised again within 10 min | the same message, edited back to open — not a new one |
|
||||
| silenced | nothing, its clearing included |
|
||||
|
||||
- **Routing:** urgent to Telegram and the desktop; warning to the desktop when a session there
|
||||
answers, otherwise to Telegram. The notifier answering is how "the operator's session is there" is
|
||||
read until presence is decided (research 028 Q4).
|
||||
- **Rate:** at most 20 messages an hour per channel. The rest are held and folded into one message
|
||||
naming them all, sent at most every ten minutes — the cap is said, never silent.
|
||||
- **What may leave the mesh:** roles and words. A message carrying an address (IP, host name, URL,
|
||||
mail address), a path, or anything shaped like a secret (a token, a key block, a long random or
|
||||
hexadecimal string, `password=…`) is refused, logged, stated as the `refused` event, and sent in its
|
||||
place as `channel-refused` with the words that carried it withheld. The rule is `cmd/messenger/content.go`.
|
||||
- **Nothing silent:** a channel that cannot send says so in `messenger_status` and in the log, and
|
||||
the message is tried again every minute while the condition is open. An event that cannot be read
|
||||
is refused by name, counted, and told to the operator once an hour.
|
||||
- **No answering back.** Acknowledging is `conditions silence`, through the mesh.
|
||||
|
||||
## What the operator gives
|
||||
|
||||
Nothing is sent until these are given; `messenger_status` says which is missing.
|
||||
|
||||
1. **The bot token**, as this module's own secret: `secret accept <machine> messenger telegram-token`,
|
||||
then push that machine. A value the mesh minted because none was accepted is recognised as not a
|
||||
bot token and said so.
|
||||
2. **The chat id**, as a setting: `settings` for `messenger` with `{"telegram-chat-id": "<id>"}`.
|
||||
3. **The desktop machines**, as a setting: `{"desktop-machines": ["<the machine the operator sits at>", …]}`.
|
||||
Without it, warnings go to Telegram.
|
||||
|
||||
## Tools
|
||||
|
||||
| tool | does |
|
||||
|---|---|
|
||||
| `operator-channel.open` | what is open now, urgent first, with where it went, silenced, reminded, held, refused, unsent |
|
||||
| `operator-channel.history` | what was said lately, and the refusals |
|
||||
| `operator-channel.notify` | a message from a module using the seat: key, severity, summary; `clear` to end it |
|
||||
| `messenger_status` | whether the operator can be reached and why not; `check` asks Telegram whether the token works |
|
||||
| `messenger_recent` | the recent sends, edits, folds and failures |
|
||||
| `messenger_test` | a test message now, to telegram, desktop or both |
|
||||
| `messenger_check` | whether some words may leave the mesh |
|
||||
|
||||
## Where it runs
|
||||
|
||||
Held once for the mesh: assign it to one machine whose tool runtime runs as root, since its secret
|
||||
and settings are root's files at 0600 (as every runtime-carried module's are).
|
||||
|
||||
## Not yet
|
||||
|
||||
- **`notify` is a served verb, not a work queue.** The design has the seat *accept* `notify` so a
|
||||
message waits for a holder; the node's tool runtime does not yet hand a bundle its seat's queue,
|
||||
and an accepted queue nobody takes would hold messages silently. It is answered request-and-reply
|
||||
until the runtime takes a seat's queue for a bundle (mesh-tools).
|
||||
- **Channels are inside the holder**, not modules contributing to the seat: a seat a module declares
|
||||
cannot receive contributions yet (ADR 0212 kinds are compiled for the mesh's own seats).
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const goodToken = "123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsawQ"
|
||||
|
||||
func tokenFile(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "telegram-token")
|
||||
if content != "" {
|
||||
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestTelegramSaysWhatItLacks(t *testing.T) {
|
||||
for _, c := range []struct{ token, chat, says string }{
|
||||
{"", "42", "not on this machine yet"},
|
||||
{"r4nd0mlyMadeByTheMeshBecauseNobodyAcceptedOne", "42", "not a bot token"},
|
||||
{goodToken, "", "telegram-chat-id is not given"},
|
||||
{goodToken, "not a chat", "is not a chat id"},
|
||||
} {
|
||||
tg := NewTelegram(TelegramConfig{TokenFile: tokenFile(t, c.token), ChatID: func() string { return c.chat }})
|
||||
err := tg.Ready()
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%+v: %v", c, err)
|
||||
}
|
||||
if err != nil && strings.Contains(err.Error(), goodToken) {
|
||||
t.Errorf("the token is in the words")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramSendsEditsAndNeverSaysItsToken(t *testing.T) {
|
||||
var asked []string
|
||||
var bodies []map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
asked = append(asked, r.URL.Path)
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
var b map[string]any
|
||||
_ = json.Unmarshal(raw, &b)
|
||||
bodies = append(bodies, b)
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/sendMessage"):
|
||||
_, _ = w.Write([]byte(`{"ok":true,"result":{"message_id":77}}`))
|
||||
case strings.HasSuffix(r.URL.Path, "/editMessageText"):
|
||||
_, _ = w.Write([]byte(`{"ok":false,"error_code":400,"description":"Bad Request: message to edit not found"}`))
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"ok":true,"result":{"username":"mesh_bot"}}`))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
old := TelegramAPI
|
||||
TelegramAPI = srv.URL
|
||||
defer func() { TelegramAPI = old }()
|
||||
tg := NewTelegram(TelegramConfig{TokenFile: tokenFile(t, goodToken+"\n"), ChatID: func() string { return "-1001" }})
|
||||
id, err := tg.Send(Message{Title: "URGENT: x", Body: "key: a.b.c"})
|
||||
if err != nil || id != "77" {
|
||||
t.Fatalf("%q %v", id, err)
|
||||
}
|
||||
if asked[0] != "/bot"+goodToken+"/sendMessage" || bodies[0]["chat_id"] != "-1001" || bodies[0]["text"] != "URGENT: x\nkey: a.b.c" {
|
||||
t.Fatalf("%v %v", asked, bodies[0])
|
||||
}
|
||||
err = tg.Edit("77", Message{Title: "CLEARED"})
|
||||
if err == nil || !strings.Contains(err.Error(), "message to edit not found") || strings.Contains(err.Error(), goodToken) {
|
||||
t.Fatalf("edit: %v", err)
|
||||
}
|
||||
if bodies[1]["message_id"] != float64(77) {
|
||||
t.Fatalf("message id: %v", bodies[1]["message_id"])
|
||||
}
|
||||
if who, err := tg.Who(); err != nil || who != "mesh_bot" {
|
||||
t.Fatalf("%q %v", who, err)
|
||||
}
|
||||
// Nothing answers: the error is in words, without the URL that carries the token.
|
||||
srv.Close()
|
||||
_, err = tg.Send(Message{Title: "x"})
|
||||
if err == nil || strings.Contains(err.Error(), goodToken) || strings.Contains(err.Error(), "/bot") {
|
||||
t.Fatalf("unreachable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheDesktopAsksTheNotifierSeatOnEachMachine(t *testing.T) {
|
||||
var asked []string
|
||||
d := &Desktop{
|
||||
Machines: func() []string { return []string{"one", "two"} },
|
||||
Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
args := body.(map[string]any)
|
||||
if args["body"] != "a <b>" || args["urgency"] != "critical" {
|
||||
t.Errorf("args: %v", args)
|
||||
}
|
||||
if strings.HasSuffix(key, "@two") {
|
||||
return nil, errors.New("the account is not logged in")
|
||||
}
|
||||
return json.RawMessage(`{"id":12}`), nil
|
||||
},
|
||||
}
|
||||
id, err := d.Send(Message{Title: "t", Body: "a <b>", Urgent: true})
|
||||
if err != nil || id != "one=12" {
|
||||
t.Fatalf("%q %v", id, err)
|
||||
}
|
||||
if asked[0] != "seat:node-notifier.send@one" || asked[1] != "seat:node-notifier.send@two" {
|
||||
t.Fatalf("%v", asked)
|
||||
}
|
||||
if d.LastAnswers()["two"] == "" || d.LastAnswers()["one"] != "" {
|
||||
t.Fatalf("%v", d.LastAnswers())
|
||||
}
|
||||
asked = nil
|
||||
if err := d.Edit("one=12", Message{Title: "t", Body: "a <b>", Urgent: true}); err != nil || len(asked) != 1 {
|
||||
t.Fatalf("edit asked %v: %v", asked, err)
|
||||
}
|
||||
none := &Desktop{Machines: func() []string { return nil }}
|
||||
if err := none.Ready(); err == nil || !strings.Contains(err.Error(), "desktop-machines") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNotifiersAnswerIsReadInEitherShape(t *testing.T) {
|
||||
for raw, want := range map[string]int{
|
||||
`{"id":5}`: 5,
|
||||
`{"content":[{"type":"text","text":"{\"id\":6}"}]}`: 6,
|
||||
`"{\"id\":7}"`: 7,
|
||||
} {
|
||||
if got, err := notificationID(json.RawMessage(raw)); err != nil || got != want {
|
||||
t.Errorf("%s: %d %v", raw, got, err)
|
||||
}
|
||||
}
|
||||
if _, err := notificationID(json.RawMessage(`{"content":[{"text":"no session"}],"isError":true}`)); err == nil {
|
||||
t.Errorf("an error answer read as an id")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSettingsAreReadAsTheMeshMergesThem(t *testing.T) {
|
||||
p := filepath.Join(t.TempDir(), "settings.json")
|
||||
_ = os.WriteFile(p, []byte(`{"telegram-chat-id": 123456, "desktop-machines": ["a", " ", "b"]}`), 0o600)
|
||||
s, err := readSettings(p)
|
||||
if err != nil || s.TelegramChatID != "123456" || len(s.DesktopMachines) != 2 {
|
||||
t.Fatalf("%+v %v", s, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
package main
|
||||
|
||||
// The controller's condition events, read in this one place (novox/hq to-be 45 §2).
|
||||
//
|
||||
// **The contract this reads, and every assumption it makes about it**, because the controller's
|
||||
// side was built at the same time from the same design and the design names the events and the
|
||||
// fields but not their spelling on the wire:
|
||||
//
|
||||
// - The events are the mesh-controller seat's own: `condition-raised`, `condition-changed` and
|
||||
// `condition-cleared`, consumed as `mesh-controller.<event>` (subject
|
||||
// `mesh.seat.mesh-controller.event.<event>`), exactly as `applied` and `built-before` are.
|
||||
// - The body is the condition as the store holds it, one JSON object, with the field names of the
|
||||
// design's table in kebab-case: key, kind, subject, severity, summary, evidence, source, raised,
|
||||
// last-observed, observations, tried, resolver, silenced, epoch. snake_case and camelCase
|
||||
// spellings of the two-word names are read too.
|
||||
// - `subject` is an object {scope, id, machine}; a plain string is read as well.
|
||||
// - `severity` is `urgent` or `warning`. Anything else is not guessed: the event is unreadable.
|
||||
// - `silenced` is absent, null, or an object {until, by, why}; `until` in the past is not silenced.
|
||||
// - Times are RFC 3339.
|
||||
// - `key` is `<scope>.<id>.<kind>`. When it is absent it is made from subject and kind; when
|
||||
// neither gives one the event is unreadable.
|
||||
// - A cleared event carries the condition as last held, and may add `cleared` (its time).
|
||||
//
|
||||
// An event this cannot read is refused by name — which event, which field, why — counted, said in
|
||||
// the status and in the log, and told to the operator; never read as an empty condition.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The events, by their local names under the controller's seat.
|
||||
const (
|
||||
EventRaised = "condition-raised"
|
||||
EventChanged = "condition-changed"
|
||||
EventCleared = "condition-cleared"
|
||||
// ControllerSeat is the role the events are stated under.
|
||||
ControllerSeat = "mesh-controller"
|
||||
)
|
||||
|
||||
// Severities: two levels, no more (to-be 45 §2).
|
||||
const (
|
||||
Urgent = "urgent"
|
||||
Warning = "warning"
|
||||
)
|
||||
|
||||
// Condition is what this holder needs of one.
|
||||
type Condition struct {
|
||||
Key string
|
||||
Kind string
|
||||
Scope string
|
||||
ID string
|
||||
Machine string
|
||||
Severity string
|
||||
Summary string
|
||||
Source string
|
||||
Resolver string
|
||||
Raised time.Time
|
||||
LastObserved time.Time
|
||||
Observations int
|
||||
SilencedTill time.Time
|
||||
SilencedWhy string
|
||||
Cleared time.Time
|
||||
}
|
||||
|
||||
// SubjectWords is the subject as a person reads it: "machine ace", "plan 41", "provider keycloak".
|
||||
func (c Condition) SubjectWords() string {
|
||||
parts := []string{}
|
||||
if c.Scope != "" {
|
||||
parts = append(parts, c.Scope)
|
||||
}
|
||||
if c.ID != "" {
|
||||
parts = append(parts, c.ID)
|
||||
}
|
||||
if c.Machine != "" && c.Machine != c.ID {
|
||||
parts = append(parts, "on "+c.Machine)
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// SilencedAt says whether the condition's messages are stopped at that moment.
|
||||
func (c Condition) SilencedAt(now time.Time) bool {
|
||||
return !c.SilencedTill.IsZero() && now.Before(c.SilencedTill)
|
||||
}
|
||||
|
||||
// eventOf is the local event name of an envelope's key: `mesh-controller.condition-raised` is
|
||||
// `condition-raised`. Anything not of the controller's seat is not a condition event.
|
||||
func eventOf(key string) (string, bool) {
|
||||
emitter, event, ok := strings.Cut(key, ".")
|
||||
if !ok || emitter != ControllerSeat {
|
||||
return "", false
|
||||
}
|
||||
switch event {
|
||||
case EventRaised, EventChanged, EventCleared:
|
||||
return event, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// DecodeCondition reads one condition event's body.
|
||||
func DecodeCondition(event string, body []byte) (Condition, error) {
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return Condition{}, fmt.Errorf("%s: the body is not a JSON object: %v", event, err)
|
||||
}
|
||||
if raw == nil {
|
||||
return Condition{}, fmt.Errorf("%s: the body is null", event)
|
||||
}
|
||||
var c Condition
|
||||
var err error
|
||||
str := func(names ...string) string {
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, n := range names {
|
||||
v, ok := raw[n]
|
||||
if !ok || string(v) == "null" {
|
||||
continue
|
||||
}
|
||||
var s string
|
||||
if e := json.Unmarshal(v, &s); e != nil {
|
||||
err = fmt.Errorf("%s: %s is not a string", event, n)
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
when := func(names ...string) time.Time {
|
||||
s := str(names...)
|
||||
if s == "" || err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
t, e := time.Parse(time.RFC3339Nano, s)
|
||||
if e != nil {
|
||||
err = fmt.Errorf("%s: %s is not an RFC 3339 time: %q", event, names[0], s)
|
||||
}
|
||||
return t
|
||||
}
|
||||
c.Key = str("key")
|
||||
c.Kind = str("kind")
|
||||
c.Severity = str("severity")
|
||||
c.Summary = str("summary")
|
||||
c.Resolver = str("resolver")
|
||||
c.Raised = when("raised")
|
||||
c.LastObserved = when("last-observed", "last_observed", "lastObserved")
|
||||
c.Cleared = when("cleared")
|
||||
if err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
// source: a string, or an object naming the row, probe or event.
|
||||
if v, ok := raw["source"]; ok && string(v) != "null" {
|
||||
var s string
|
||||
if json.Unmarshal(v, &s) == nil {
|
||||
c.Source = s
|
||||
} else {
|
||||
var o map[string]any
|
||||
if json.Unmarshal(v, &o) == nil {
|
||||
for _, k := range []string{"row", "probe", "event", "name", "id"} {
|
||||
if s, ok := o[k].(string); ok && s != "" {
|
||||
c.Source = s
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"observations", "count"} {
|
||||
if v, ok := raw[n]; ok && string(v) != "null" {
|
||||
var f float64
|
||||
if json.Unmarshal(v, &f) != nil {
|
||||
return Condition{}, fmt.Errorf("%s: %s is not a number", event, n)
|
||||
}
|
||||
c.Observations = int(f)
|
||||
break
|
||||
}
|
||||
}
|
||||
if v, ok := raw["subject"]; ok && string(v) != "null" {
|
||||
var s string
|
||||
if json.Unmarshal(v, &s) == nil {
|
||||
c.ID = strings.TrimSpace(s)
|
||||
} else {
|
||||
var o struct {
|
||||
Scope string `json:"scope"`
|
||||
ID string `json:"id"`
|
||||
Machine string `json:"machine"`
|
||||
Node string `json:"node"`
|
||||
}
|
||||
if e := json.Unmarshal(v, &o); e != nil {
|
||||
return Condition{}, fmt.Errorf("%s: subject is neither a string nor {scope, id, machine}", event)
|
||||
}
|
||||
c.Scope, c.ID, c.Machine = o.Scope, o.ID, o.Machine
|
||||
if c.Machine == "" {
|
||||
c.Machine = o.Node
|
||||
}
|
||||
}
|
||||
}
|
||||
if v, ok := raw["silenced"]; ok && string(v) != "null" && string(v) != "{}" && string(v) != `""` && string(v) != "false" {
|
||||
var o struct {
|
||||
Until string `json:"until"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
if e := json.Unmarshal(v, &o); e != nil {
|
||||
return Condition{}, fmt.Errorf("%s: silenced is not {until, by, why}", event)
|
||||
}
|
||||
if o.Until != "" {
|
||||
t, e := time.Parse(time.RFC3339Nano, o.Until)
|
||||
if e != nil {
|
||||
return Condition{}, fmt.Errorf("%s: silenced.until is not an RFC 3339 time: %q", event, o.Until)
|
||||
}
|
||||
c.SilencedTill = t
|
||||
}
|
||||
c.SilencedWhy = o.Why
|
||||
}
|
||||
if c.Key == "" && c.Scope != "" && c.ID != "" && c.Kind != "" {
|
||||
c.Key = c.Scope + "." + c.ID + "." + c.Kind
|
||||
}
|
||||
if c.Key == "" {
|
||||
return Condition{}, fmt.Errorf("%s: no key, and no subject and kind to make one from", event)
|
||||
}
|
||||
if c.Scope == "" || c.ID == "" || c.Kind == "" {
|
||||
// The key names them (`<scope>.<id>.<kind>`, the id itself possibly dotted).
|
||||
parts := strings.Split(c.Key, ".")
|
||||
if len(parts) >= 3 {
|
||||
if c.Scope == "" {
|
||||
c.Scope = parts[0]
|
||||
}
|
||||
if c.Kind == "" {
|
||||
c.Kind = parts[len(parts)-1]
|
||||
}
|
||||
if c.ID == "" {
|
||||
c.ID = strings.Join(parts[1:len(parts)-1], ".")
|
||||
}
|
||||
}
|
||||
}
|
||||
switch c.Severity {
|
||||
case Urgent, Warning:
|
||||
case "":
|
||||
if event != EventCleared {
|
||||
return Condition{}, fmt.Errorf("%s %s: no severity", event, c.Key)
|
||||
}
|
||||
default:
|
||||
return Condition{}, fmt.Errorf("%s %s: severity %q is neither urgent nor warning", event, c.Key, c.Severity)
|
||||
}
|
||||
if c.Summary == "" && event != EventCleared {
|
||||
return Condition{}, fmt.Errorf("%s %s: no summary", event, c.Key)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The contract with the controller's condition events (to-be 45 §2), as condition.go states it.
|
||||
|
||||
func TestTheEventsAreTheControllersSeat(t *testing.T) {
|
||||
for key, want := range map[string]string{
|
||||
"mesh-controller.condition-raised": EventRaised,
|
||||
"mesh-controller.condition-changed": EventChanged,
|
||||
"mesh-controller.condition-cleared": EventCleared,
|
||||
} {
|
||||
if got, ok := eventOf(key); !ok || got != want {
|
||||
t.Errorf("%s: %q %v", key, got, ok)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"mesh-controller.applied", "gitea.condition-raised", "condition-raised"} {
|
||||
if _, ok := eventOf(key); ok {
|
||||
t.Errorf("%s read as a condition event", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConditionAsTheStoreHoldsIt(t *testing.T) {
|
||||
body := `{
|
||||
"key": "machine.ace.silent", "kind": "silent",
|
||||
"subject": {"scope": "machine", "id": "ace", "machine": "ace"},
|
||||
"severity": "urgent", "summary": "the home server has not been heard for 15 min",
|
||||
"evidence": [{"at": "2026-10-06T12:00:00Z", "what": "last heartbeat"}],
|
||||
"source": "S1", "raised": "2026-10-06T12:15:00Z", "last-observed": "2026-10-06T12:16:00Z",
|
||||
"observations": 3, "tried": [], "resolver": "self",
|
||||
"silenced": {"until": "2026-10-06T14:00:00Z", "by": "operator", "why": "moving it"},
|
||||
"epoch": 57
|
||||
}`
|
||||
c, err := DecodeCondition(EventRaised, []byte(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Key != "machine.ace.silent" || c.Kind != "silent" || c.Scope != "machine" || c.ID != "ace" ||
|
||||
c.Severity != Urgent || c.Source != "S1" || c.Observations != 3 || c.Resolver != "self" {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
if !c.Raised.Equal(time.Date(2026, 10, 6, 12, 15, 0, 0, time.UTC)) || c.LastObserved.IsZero() {
|
||||
t.Fatalf("times: %+v", c)
|
||||
}
|
||||
if !c.SilencedAt(time.Date(2026, 10, 6, 13, 0, 0, 0, time.UTC)) || c.SilencedAt(time.Date(2026, 10, 6, 15, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("silenced: %v", c.SilencedTill)
|
||||
}
|
||||
if c.SubjectWords() != "machine ace" {
|
||||
t.Fatalf("subject words: %q", c.SubjectWords())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOtherSpellingsAndAKeyMadeFromItsParts(t *testing.T) {
|
||||
c, err := DecodeCondition(EventChanged, []byte(`{"kind":"stalled","subject":{"scope":"plan","id":"41"},
|
||||
"severity":"warning","summary":"plan 41 waits","last_observed":"2026-10-06T12:00:00Z","silenced":null,
|
||||
"source":{"row":"S3"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Key != "plan.41.stalled" || c.Source != "S3" || c.LastObserved.IsZero() || !c.SilencedTill.IsZero() {
|
||||
t.Fatalf("%+v", c)
|
||||
}
|
||||
c, err = DecodeCondition(EventRaised, []byte(`{"key":"provider.keycloak.ace.gitea.failing","subject":"keycloak",
|
||||
"severity":"warning","summary":"the identity provider fails gitea"}`))
|
||||
if err != nil || c.Scope != "provider" || c.Kind != "failing" || c.ID != "keycloak" {
|
||||
t.Fatalf("%+v %v", c, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAClearedEventNeedsOnlyItsKey(t *testing.T) {
|
||||
c, err := DecodeCondition(EventCleared, []byte(`{"key":"machine.ace.silent","cleared":"2026-10-06T12:30:00Z"}`))
|
||||
if err != nil || c.Key != "machine.ace.silent" || c.Cleared.IsZero() {
|
||||
t.Fatalf("%+v %v", c, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatCannotBeReadIsRefusedByName(t *testing.T) {
|
||||
for body, says := range map[string]string{
|
||||
`not json`: "not a JSON object",
|
||||
`null`: "null",
|
||||
`{"severity":"urgent","summary":"x"}`: "no key",
|
||||
`{"key":"a.b.c","summary":"x"}`: "no severity",
|
||||
`{"key":"a.b.c","severity":"critical","summary":"x"}`: "neither urgent nor warning",
|
||||
`{"key":"a.b.c","severity":"urgent"}`: "no summary",
|
||||
`{"key":"a.b.c","severity":"urgent","summary":"x","raised":"yesterday"}`: "not an RFC 3339 time",
|
||||
`{"key":7}`: "key is not a string",
|
||||
`{"key":"a.b.c","severity":"urgent","summary":"x","silenced":{"until":"soon"}}`: "silenced.until",
|
||||
} {
|
||||
_, err := DecodeCondition(EventRaised, []byte(body))
|
||||
if err == nil || !strings.Contains(err.Error(), says) {
|
||||
t.Errorf("%s: %v, want %q", body, err, says)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
// What may leave the mesh (novox/hq to-be 45 §5, research 028 Q8, ADR 0227): roles and words. A
|
||||
// message carrying an address, a path or anything shaped like a secret is refused here, by the holder,
|
||||
// because Telegram is not end-to-end encrypted for bots and this rule is the only thing between a
|
||||
// condition's words and someone else's server. It is not trusted to each source.
|
||||
//
|
||||
// Deliberately wider than it must be: a commit id or a long random name is refused too. A source
|
||||
// that wants its message through says it in words; a refusal is said, never silent.
|
||||
|
||||
import (
|
||||
"math"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Refusal says why a text may not leave: the class of what it carried, never the text itself.
|
||||
type Refusal struct {
|
||||
Class string // address, path or secret
|
||||
What string // a few words: "an IPv4 address", "a URL", …
|
||||
}
|
||||
|
||||
func (r Refusal) String() string { return r.Class + " (" + r.What + ")" }
|
||||
|
||||
var (
|
||||
reURL = regexp.MustCompile(`(?i)\b[a-z][a-z0-9+.-]*://`)
|
||||
reEmail = regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}`)
|
||||
reIPv4 = regexp.MustCompile(`\b\d{1,3}(\.\d{1,3}){3}\b`)
|
||||
reIPv6 = regexp.MustCompile(`(?i)(^|[^0-9a-z:])(([0-9a-f]{1,4}:){4,7}[0-9a-f]{1,4}|([0-9a-f]{1,4}:)*[0-9a-f]{0,4}::([0-9a-f]{1,4}:)*[0-9a-f]{0,4})([^0-9a-z:]|$)`)
|
||||
reMAC = regexp.MustCompile(`(?i)\b([0-9a-f]{2}[:-]){5}[0-9a-f]{2}\b`)
|
||||
rePEM = regexp.MustCompile(`-----BEGIN [A-Z ]+-----`)
|
||||
reJWT = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`)
|
||||
reBotToken = regexp.MustCompile(`\b\d{6,}:[A-Za-z0-9_-]{30,}`)
|
||||
reKnown = regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{16,}|sk-[A-Za-z0-9_-]{16,}|xox[abprs]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16})`)
|
||||
reAssigned = regexp.MustCompile(`(?i)\b(password|passwd|passphrase|secret|token|api[_-]?key|apikey|credential|private[_-]?key)\s*[=:]\s*\S`)
|
||||
reHex = regexp.MustCompile(`(?i)\b[0-9a-f]{32,}\b`)
|
||||
reRun = regexp.MustCompile(`[A-Za-z0-9+/=_]{20,}`)
|
||||
reWinPath = regexp.MustCompile(`(?i)\b[a-z]:\\`)
|
||||
)
|
||||
|
||||
// topLevel are names that end a host name — the generic and country ones a mesh's names use, and the
|
||||
// private ones (.internal, .lan, .home, .local) a mesh is likelier to.
|
||||
var topLevel = map[string]bool{}
|
||||
|
||||
func init() {
|
||||
for _, t := range strings.Fields(`com net org edu gov mil int io dev app cloud ai co me info biz xyz
|
||||
site online tech page link
|
||||
be nl de fr uk lu eu ch at it es pt se no dk fi pl cz us ca au nz jp cn ru in br ie
|
||||
internal lan home local localdomain corp intranet private arpa test example invalid localhost`) {
|
||||
topLevel[t] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Check says whether a text may leave the mesh, and when not, why.
|
||||
func Check(text string) (Refusal, bool) {
|
||||
switch {
|
||||
case reURL.MatchString(text):
|
||||
return Refusal{"address", "a URL"}, false
|
||||
case reEmail.MatchString(text):
|
||||
return Refusal{"address", "a mail address"}, false
|
||||
case reIPv4.MatchString(text):
|
||||
return Refusal{"address", "an IPv4 address"}, false
|
||||
case reMAC.MatchString(text):
|
||||
return Refusal{"address", "a hardware address"}, false
|
||||
case reIPv6.MatchString(text):
|
||||
return Refusal{"address", "an IPv6 address"}, false
|
||||
case rePEM.MatchString(text):
|
||||
return Refusal{"secret", "a key block"}, false
|
||||
case reJWT.MatchString(text):
|
||||
return Refusal{"secret", "a signed token"}, false
|
||||
case reBotToken.MatchString(text):
|
||||
return Refusal{"secret", "a bot token"}, false
|
||||
case reKnown.MatchString(text):
|
||||
return Refusal{"secret", "a known token shape"}, false
|
||||
case reAssigned.MatchString(text):
|
||||
return Refusal{"secret", "a value given to a secret's name"}, false
|
||||
case reHex.MatchString(text):
|
||||
return Refusal{"secret", "a long hexadecimal string"}, false
|
||||
case reWinPath.MatchString(text):
|
||||
return Refusal{"path", "a drive path"}, false
|
||||
}
|
||||
for _, run := range reRun.FindAllString(text, -1) {
|
||||
if looksRandom(run) {
|
||||
return Refusal{"secret", "a long random-looking string"}, false
|
||||
}
|
||||
}
|
||||
for _, word := range strings.FieldsFunc(text, func(r rune) bool {
|
||||
return unicode.IsSpace(r) || strings.ContainsRune("\"'`()[]{}<>,;|", r)
|
||||
}) {
|
||||
w := strings.TrimRight(word, ".:!?")
|
||||
if isPath(w) {
|
||||
return Refusal{"path", "a file path"}, false
|
||||
}
|
||||
if isHostName(w) {
|
||||
return Refusal{"address", "a host name"}, false
|
||||
}
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
// isPath: absolute, home-relative or dot-relative, or two separators deep. A mesh address names one
|
||||
// machine and one tool (`ace/postgres.query`) and has one; a ratio ("3/4") has digits only.
|
||||
func isPath(w string) bool {
|
||||
if w == "" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(w, "/") && len(w) > 1 {
|
||||
return true
|
||||
}
|
||||
for _, p := range []string{"~/", "./", "../", "$HOME", "${"} {
|
||||
if strings.HasPrefix(w, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return strings.Count(w, "/") >= 2 || strings.Contains(w, "\\")
|
||||
}
|
||||
|
||||
// isHostName: two names or more, the last a top-level one. A condition key's last name is its kind
|
||||
// (`machine.ace.silent`), which none of these is.
|
||||
func isHostName(w string) bool {
|
||||
w = strings.ToLower(w)
|
||||
if w == "localhost" {
|
||||
return true
|
||||
}
|
||||
parts := strings.Split(w, ".")
|
||||
if len(parts) < 2 {
|
||||
return false
|
||||
}
|
||||
for _, p := range parts {
|
||||
if p == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return topLevel[parts[len(parts)-1]]
|
||||
}
|
||||
|
||||
// looksRandom: letters and digits mixed, and the characters spread as a random string's are. A
|
||||
// sentence's words are separated, so only an unbroken run reaches here.
|
||||
func looksRandom(s string) bool {
|
||||
var letters, digits int
|
||||
counts := map[rune]int{}
|
||||
for _, r := range s {
|
||||
counts[r]++
|
||||
switch {
|
||||
case unicode.IsLetter(r):
|
||||
letters++
|
||||
case unicode.IsDigit(r):
|
||||
digits++
|
||||
}
|
||||
}
|
||||
if letters == 0 || digits == 0 {
|
||||
// One class only: a word, or a number. A long number of digits alone is a count or a time.
|
||||
return letters > 0 && hasUpperAndLower(s) && entropy(counts, len(s)) >= 4.0
|
||||
}
|
||||
return entropy(counts, len(s)) >= 3.3
|
||||
}
|
||||
|
||||
func hasUpperAndLower(s string) bool {
|
||||
return strings.IndexFunc(s, unicode.IsUpper) >= 0 && strings.IndexFunc(s, unicode.IsLower) >= 0
|
||||
}
|
||||
|
||||
func entropy(counts map[rune]int, n int) float64 {
|
||||
e := 0.0
|
||||
for _, c := range counts {
|
||||
p := float64(c) / float64(n)
|
||||
e -= p * math.Log2(p)
|
||||
}
|
||||
return e
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRolesAndWordsMayLeave(t *testing.T) {
|
||||
for _, s := range []string{
|
||||
"URGENT: the home server has not been heard for 15 min",
|
||||
"key: provider.keycloak.ace.gitea.failing",
|
||||
"more: conditions show machine.ace.silent",
|
||||
"since: 2026-10-06 12:30 UTC",
|
||||
"call call-1759752000123456789-12 has run past its bound of 10 min",
|
||||
"ace/postgres.query answers no more",
|
||||
"3/4 machines answered; plan 41 waits on build 7f3a9c1e",
|
||||
"the bus advisory slow-consumer for ace_records",
|
||||
"core.controller.novox.rolled-back",
|
||||
"seat node-notifier has no live holder",
|
||||
"HELD BACK: 5 message(s) over the cap of 20 an hour",
|
||||
"STILL OPEN after 1.5 h: the controller's event loop takes no message",
|
||||
"node-engine and node tools builds differ from the plan's",
|
||||
} {
|
||||
if r, ok := Check(s); !ok {
|
||||
t.Errorf("refused %q: %s", s, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAddressAPathOrASecretMayNot(t *testing.T) {
|
||||
for s, class := range map[string]string{
|
||||
"cannot reach 192.168.1.20": "address",
|
||||
"listening on 10.0.0.7:5432": "address",
|
||||
"route fd00:1234:5678::1 is gone": "address",
|
||||
"fe80::1 answered": "address",
|
||||
"2001:db8:0:0:0:0:2:1 answered": "address",
|
||||
"see https://git.example.org/x": "address",
|
||||
"git.novox.internal does not answer": "address",
|
||||
"the mail for admin@example.org bounced": "address",
|
||||
"zurag.be is down": "address",
|
||||
"localhost refused": "address",
|
||||
"the card aa:bb:cc:dd:ee:ff went away": "address",
|
||||
"/var/lib/mesh-controller is full": "path",
|
||||
"~/.config/hal/env changed": "path",
|
||||
"read ./grants.json": "path",
|
||||
"services/postgres/data/pg_hba.conf": "path",
|
||||
"C:\\Users\\x": "path",
|
||||
"token 123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw": "secret",
|
||||
"ghp_abcdefghijklmnopqrstuvwxyz0123456789": "secret",
|
||||
"password=hunter2": "secret",
|
||||
"api_key: x": "secret",
|
||||
"-----BEGIN OPENSSH PRIVATE KEY-----": "secret",
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0": "secret",
|
||||
"commit 3f2a9c1e5b7d4f6a8c0e2b4d6f8a0c2e4b6d8f0a": "secret",
|
||||
"value Zx8Qp2Lm9Rt4Vb7Nc1Kd5Hs3Jf6": "secret",
|
||||
} {
|
||||
r, ok := Check(s)
|
||||
if ok {
|
||||
t.Errorf("let %q leave", s)
|
||||
continue
|
||||
}
|
||||
if r.Class != class {
|
||||
t.Errorf("%q refused as %s, want %s", s, r, class)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package main
|
||||
|
||||
// The desktop notifier (novox/hq to-be 45 §5, ADR 0208): the `node-notifier` seat's `send` verb on
|
||||
// the machine the operator is at, asked through the mesh. Nothing new runs on that machine — the
|
||||
// seat's holder (dunst) already answers `send` in the operator's session, and answers with an error
|
||||
// when no session is there. That error is what "the operator's session is there" is read from until
|
||||
// presence is decided (research 028 Q4): a machine whose notifier answers has a session.
|
||||
//
|
||||
// Which machines to try is the setting desktop-machines, in order. None given: the channel is not
|
||||
// configured, the status says so, and a warning goes to Telegram.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Asker calls a tool through the mesh, as stdio.Ask does.
|
||||
type Asker func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
// Desktop sends to the notifier of each configured machine.
|
||||
type Desktop struct {
|
||||
Machines func() []string
|
||||
Ask Asker
|
||||
|
||||
mu sync.Mutex
|
||||
last map[string]string // machine -> the last error, or "" after a success
|
||||
}
|
||||
|
||||
func (d *Desktop) Name() string { return "desktop" }
|
||||
func (d *Desktop) CanEdit() bool { return true }
|
||||
|
||||
func (d *Desktop) Ready() error {
|
||||
if len(d.Machines()) == 0 {
|
||||
return errors.New("no machine to show it on: the setting desktop-machines is not given " +
|
||||
"(`settings` for messenger with {\"desktop-machines\": [\"<the machine the operator sits at>\"]})")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Send shows the message on every configured machine that has a session, and answers
|
||||
// `<machine>=<id>` for each one that took it. It fails only when none did.
|
||||
func (d *Desktop) Send(m Message) (string, error) {
|
||||
return d.show(m, nil)
|
||||
}
|
||||
|
||||
// Edit replaces the notification shown before on each machine that showed it.
|
||||
func (d *Desktop) Edit(id string, m Message) error {
|
||||
shown := map[string]int{}
|
||||
for _, part := range strings.Split(id, ",") {
|
||||
machine, n, ok := strings.Cut(part, "=")
|
||||
var i int
|
||||
if ok {
|
||||
if _, err := fmt.Sscan(n, &i); err == nil {
|
||||
shown[machine] = i
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(shown) == 0 {
|
||||
return errors.New("no notification on record to replace")
|
||||
}
|
||||
_, err := d.show(m, shown)
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *Desktop) show(m Message, replace map[string]int) (string, error) {
|
||||
if err := d.Ready(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
urgency := "normal"
|
||||
switch {
|
||||
case m.Quiet:
|
||||
urgency = "low"
|
||||
case m.Urgent:
|
||||
urgency = "critical"
|
||||
}
|
||||
var took []string
|
||||
var failed []string
|
||||
for _, machine := range d.Machines() {
|
||||
if replace != nil {
|
||||
if _, shown := replace[machine]; !shown {
|
||||
continue
|
||||
}
|
||||
}
|
||||
args := map[string]any{
|
||||
"summary": m.Title,
|
||||
"body": html.EscapeString(m.Body),
|
||||
"urgency": urgency,
|
||||
"app_name": "mesh",
|
||||
}
|
||||
if replace != nil {
|
||||
args["replace_id"] = replace[machine]
|
||||
}
|
||||
raw, err := d.Ask("seat:node-notifier.send@"+machine, args)
|
||||
if err == nil {
|
||||
var id int
|
||||
id, err = notificationID(raw)
|
||||
if err == nil {
|
||||
took = append(took, fmt.Sprintf("%s=%d", machine, id))
|
||||
}
|
||||
}
|
||||
d.note(machine, err)
|
||||
if err != nil {
|
||||
failed = append(failed, machine+": "+err.Error())
|
||||
}
|
||||
}
|
||||
if len(took) == 0 {
|
||||
return "", errors.New("no machine showed it — " + strings.Join(failed, "; "))
|
||||
}
|
||||
return strings.Join(took, ","), nil
|
||||
}
|
||||
|
||||
func (d *Desktop) note(machine string, err error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.last == nil {
|
||||
d.last = map[string]string{}
|
||||
}
|
||||
if err != nil {
|
||||
d.last[machine] = err.Error()
|
||||
} else {
|
||||
d.last[machine] = ""
|
||||
}
|
||||
}
|
||||
|
||||
// Machines' last answers, for the status: "" is a machine that took the last message shown to it.
|
||||
func (d *Desktop) LastAnswers() map[string]string {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
out := map[string]string{}
|
||||
for k, v := range d.last {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// notificationID reads the notifier's answer, {"id": N}, whether the runtime hands it over bare or in
|
||||
// the tool reply's text content.
|
||||
func notificationID(raw json.RawMessage) (int, error) {
|
||||
var direct struct {
|
||||
ID *int `json:"id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &direct) == nil && direct.ID != nil {
|
||||
return *direct.ID, nil
|
||||
}
|
||||
var wrapped struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 {
|
||||
if wrapped.IsError {
|
||||
return 0, errors.New(wrapped.Content[0].Text)
|
||||
}
|
||||
if json.Unmarshal([]byte(wrapped.Content[0].Text), &direct) == nil && direct.ID != nil {
|
||||
return *direct.ID, nil
|
||||
}
|
||||
}
|
||||
var s string
|
||||
if json.Unmarshal(raw, &s) == nil && json.Unmarshal([]byte(s), &direct) == nil && direct.ID != nil {
|
||||
return *direct.ID, nil
|
||||
}
|
||||
return 0, fmt.Errorf("the notifier answered no id: %.80s", string(raw))
|
||||
}
|
||||
@@ -0,0 +1,775 @@
|
||||
package main
|
||||
|
||||
// The holder of the operator-channel seat (novox/hq to-be 45 §5, ADR 0227): what is sent, to whom,
|
||||
// when, and how often. The controller decides what is wrong; this decides what is said.
|
||||
//
|
||||
// - On `condition-raised`: one message, deduplicated by the condition's key. Said again with the
|
||||
// same key while open, it is the same message, not a second.
|
||||
// - Once more if still open after 1 hour (urgent) or 12 hours (warning).
|
||||
// - On `condition-cleared`: the first message is edited where the channel can (both can);
|
||||
// otherwise a new one says it. Cleared and raised again within ten minutes, it is the same
|
||||
// message, edited back to open — not a new one.
|
||||
// - A silenced condition sends nothing.
|
||||
// - Urgent to both channels; warning to the desktop when the operator's session is there,
|
||||
// otherwise to Telegram.
|
||||
// - At most twenty messages an hour per channel; the excess is held and folded into one message
|
||||
// naming them all, sent at most every ten minutes — the cap is said, never silent.
|
||||
// - A message carrying an address, a path or a secret is refused (content.go); what is sent in its
|
||||
// place says `channel-refused`, with the offending part withheld.
|
||||
// - A channel that cannot send says so in the status and the log, and the message is tried again
|
||||
// every minute while the condition is open.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
RemindUrgent = time.Hour
|
||||
RemindWarning = 12 * time.Hour
|
||||
ReopenWindow = 10 * time.Minute
|
||||
CapPerHour = 20
|
||||
FoldEvery = 10 * time.Minute
|
||||
KeptSends = 200
|
||||
KeptRefusals = 50
|
||||
)
|
||||
|
||||
// Message is what a channel shows: a title line and a body.
|
||||
type Message struct {
|
||||
Title string
|
||||
Body string
|
||||
Urgent bool
|
||||
Quiet bool // a clearing: shown without urgency
|
||||
}
|
||||
|
||||
func (m Message) Text() string {
|
||||
if m.Body == "" {
|
||||
return m.Title
|
||||
}
|
||||
return m.Title + "\n" + m.Body
|
||||
}
|
||||
|
||||
// Channel is one way to the operator.
|
||||
type Channel interface {
|
||||
Name() string
|
||||
Ready() error
|
||||
Send(Message) (string, error)
|
||||
Edit(id string, m Message) error
|
||||
CanEdit() bool
|
||||
}
|
||||
|
||||
// Record is one open message, kept in the module's own state so a restart forgets nothing.
|
||||
type Record struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Severity string `json:"severity"`
|
||||
Summary string `json:"summary"`
|
||||
Origin string `json:"origin"`
|
||||
More string `json:"more"`
|
||||
Raised time.Time `json:"raised"`
|
||||
SilencedTill time.Time `json:"silenced_till,omitempty"`
|
||||
Sent map[string]string `json:"sent,omitempty"` // channel -> the first message's id
|
||||
FirstSent time.Time `json:"first_sent,omitempty"`
|
||||
Reminded bool `json:"reminded,omitempty"`
|
||||
Pending string `json:"pending,omitempty"` // what is still to be said: raised, reminder, …
|
||||
Folded bool `json:"folded,omitempty"`
|
||||
Cleared time.Time `json:"cleared,omitempty"`
|
||||
Count int `json:"count"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
func (r *Record) silenced(now time.Time) bool {
|
||||
return !r.SilencedTill.IsZero() && now.Before(r.SilencedTill)
|
||||
}
|
||||
|
||||
// Sent is one message that went out, or was held, for the history.
|
||||
type Sent struct {
|
||||
At time.Time `json:"at"`
|
||||
Channel string `json:"channel"`
|
||||
Key string `json:"key"`
|
||||
What string `json:"what"`
|
||||
Outcome string `json:"outcome"` // sent, edited, folded, failed: <why>
|
||||
}
|
||||
|
||||
// RefusalNote is one refused message: never its text.
|
||||
type RefusalNote struct {
|
||||
At time.Time `json:"at"`
|
||||
Key string `json:"key"`
|
||||
Class string `json:"class"`
|
||||
What string `json:"what"`
|
||||
}
|
||||
|
||||
// Store is the module's own state (ADR 0201): the open messages, and the recent sends.
|
||||
type Store interface {
|
||||
Put(r Record) error
|
||||
Delete(key string) error
|
||||
All() ([]Record, error)
|
||||
PutRecent([]Sent) error
|
||||
Recent() ([]Sent, error)
|
||||
}
|
||||
|
||||
type foldEntry struct {
|
||||
Key, Severity, What string
|
||||
}
|
||||
|
||||
// Holder is the seat's holder.
|
||||
type Holder struct {
|
||||
Telegram Channel
|
||||
Desktop Channel
|
||||
Store Store
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
// Emit states a fact as this module (refused); nil states nothing.
|
||||
Emit func(event string, body any) error
|
||||
|
||||
work sync.Mutex // one event, call or tick at a time
|
||||
mu sync.Mutex // what the status reads
|
||||
|
||||
open map[string]*Record
|
||||
recent []Sent
|
||||
refusals []RefusalNote
|
||||
folds map[string][]foldEntry
|
||||
lastFold map[string]time.Time
|
||||
chanErr map[string]string
|
||||
chanErrAt map[string]time.Time
|
||||
chanOK map[string]time.Time
|
||||
unreadable int
|
||||
lastBad string
|
||||
lastBadAt time.Time
|
||||
saidOnce map[string]time.Time
|
||||
storeErr string
|
||||
heard map[string]int
|
||||
lastHeard time.Time
|
||||
}
|
||||
|
||||
func (h *Holder) init() {
|
||||
if h.open == nil {
|
||||
h.open = map[string]*Record{}
|
||||
h.folds = map[string][]foldEntry{}
|
||||
h.lastFold = map[string]time.Time{}
|
||||
h.chanErr = map[string]string{}
|
||||
h.chanErrAt = map[string]time.Time{}
|
||||
h.chanOK = map[string]time.Time{}
|
||||
h.saidOnce = map[string]time.Time{}
|
||||
h.heard = map[string]int{}
|
||||
}
|
||||
if h.Now == nil {
|
||||
h.Now = time.Now
|
||||
}
|
||||
if h.Logf == nil {
|
||||
h.Logf = func(string, ...any) {}
|
||||
}
|
||||
}
|
||||
|
||||
// Load reads back what was open and what was sent before a restart.
|
||||
func (h *Holder) Load() error {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if h.Store == nil {
|
||||
return nil
|
||||
}
|
||||
recs, err := h.Store.All()
|
||||
if err != nil {
|
||||
h.storeErr = err.Error()
|
||||
return err
|
||||
}
|
||||
for i := range recs {
|
||||
r := recs[i]
|
||||
h.open[r.Key] = &r
|
||||
}
|
||||
if sent, err := h.Store.Recent(); err == nil {
|
||||
h.recent = sent
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Condition takes one of the controller's condition events.
|
||||
func (h *Holder) Condition(event string, c Condition) {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
h.heard[event]++
|
||||
h.lastHeard = h.Now()
|
||||
h.mu.Unlock()
|
||||
rec := Record{
|
||||
Key: c.Key, Kind: c.Kind, Subject: c.SubjectWords(), Severity: c.Severity, Summary: c.Summary,
|
||||
Origin: "condition", More: "conditions show " + c.Key, Raised: c.Raised, SilencedTill: c.SilencedTill,
|
||||
}
|
||||
switch event {
|
||||
case EventRaised:
|
||||
h.raised(rec)
|
||||
case EventChanged:
|
||||
h.changed(rec)
|
||||
case EventCleared:
|
||||
h.cleared(rec.Key)
|
||||
}
|
||||
}
|
||||
|
||||
// Unreadable records an event that could not be read, and tells the operator — once an hour.
|
||||
func (h *Holder) Unreadable(key string, err error) {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
h.unreadable++
|
||||
h.lastBad = err.Error()
|
||||
h.lastBadAt = h.Now()
|
||||
n := h.unreadable
|
||||
h.mu.Unlock()
|
||||
h.Logf("[messenger] refused %s: %v (unreadable events since start: %d)", key, err, n)
|
||||
h.sayOnce("messenger.unreadable-event", time.Hour, Message{
|
||||
Title: "WARNING: a condition event could not be read",
|
||||
Body: fmt.Sprintf("the operator-channel's holder could not read %d condition event(s) from the controller; "+
|
||||
"what was wrong is in messenger_status. A condition may be open that was not said.", n),
|
||||
})
|
||||
}
|
||||
|
||||
func (h *Holder) raised(rec Record) {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
old := h.open[rec.Key]
|
||||
h.mu.Unlock()
|
||||
if old != nil && old.Cleared.IsZero() {
|
||||
// Said again while open: the same message. Its words are kept current; nothing is sent.
|
||||
h.mu.Lock()
|
||||
old.Summary, old.Subject, old.Kind, old.SilencedTill = rec.Summary, rec.Subject, rec.Kind, rec.SilencedTill
|
||||
if rec.Severity != "" {
|
||||
old.Severity = rec.Severity
|
||||
}
|
||||
h.mu.Unlock()
|
||||
h.persist(old)
|
||||
return
|
||||
}
|
||||
if old != nil && now.Sub(old.Cleared) < ReopenWindow {
|
||||
// Cleared and raised again within ten minutes: the same message, back to open (to-be 45 §2).
|
||||
h.mu.Lock()
|
||||
old.Cleared = time.Time{}
|
||||
old.Count++
|
||||
old.Summary, old.Severity, old.SilencedTill = rec.Summary, rec.Severity, rec.SilencedTill
|
||||
h.mu.Unlock()
|
||||
if !old.silenced(now) && len(old.Sent) > 0 {
|
||||
h.edit(old, "reopened")
|
||||
}
|
||||
h.persist(old)
|
||||
return
|
||||
}
|
||||
r := rec
|
||||
r.Count = 1
|
||||
if r.Raised.IsZero() {
|
||||
r.Raised = now
|
||||
}
|
||||
r.Sent = map[string]string{}
|
||||
h.mu.Lock()
|
||||
h.open[r.Key] = &r
|
||||
h.mu.Unlock()
|
||||
if r.silenced(now) {
|
||||
h.Logf("[messenger] %s raised while silenced until %s: nothing sent", r.Key, r.SilencedTill.Format(time.RFC3339))
|
||||
h.persist(&r)
|
||||
return
|
||||
}
|
||||
h.deliver(&r, "raised")
|
||||
h.persist(&r)
|
||||
}
|
||||
|
||||
func (h *Holder) changed(rec Record) {
|
||||
h.mu.Lock()
|
||||
old := h.open[rec.Key]
|
||||
h.mu.Unlock()
|
||||
if old == nil || !old.Cleared.IsZero() {
|
||||
// A change to something this holder never heard raised: read as a raise, so it is said.
|
||||
h.Logf("[messenger] %s changed and was not open here; taken as raised", rec.Key)
|
||||
h.raised(rec)
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
escalated := old.Severity == Warning && rec.Severity == Urgent
|
||||
old.Summary, old.Subject, old.SilencedTill = rec.Summary, rec.Subject, rec.SilencedTill
|
||||
if rec.Severity != "" {
|
||||
old.Severity = rec.Severity
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if escalated && !old.silenced(h.Now()) {
|
||||
// Routing differs for urgent: said once more, to both channels.
|
||||
h.deliver(old, "escalated")
|
||||
}
|
||||
h.persist(old)
|
||||
}
|
||||
|
||||
func (h *Holder) cleared(key string) {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
old := h.open[key]
|
||||
h.mu.Unlock()
|
||||
if old == nil || !old.Cleared.IsZero() {
|
||||
h.Logf("[messenger] %s cleared and was not open here: nothing to say", key)
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
old.Cleared = now
|
||||
pending := old.Pending
|
||||
old.Pending = ""
|
||||
sent := len(old.Sent) > 0
|
||||
folded := old.Folded
|
||||
h.mu.Unlock()
|
||||
switch {
|
||||
case old.silenced(now):
|
||||
// A silenced condition sends nothing, its clearing included.
|
||||
case sent:
|
||||
h.edit(old, "cleared")
|
||||
case folded:
|
||||
// Held by the cap and never sent on its own: its clearing is said like any message.
|
||||
h.deliver(old, "cleared")
|
||||
case pending != "":
|
||||
h.Logf("[messenger] %s cleared before it could be sent: nothing to unsay", key)
|
||||
}
|
||||
h.persist(old)
|
||||
}
|
||||
|
||||
// Tick does what time asks: reminders, retries, the folded message, and forgetting what cleared
|
||||
// long enough ago that a new raise is a new message.
|
||||
func (h *Holder) Tick() {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
now := h.Now()
|
||||
var recs []*Record
|
||||
for _, r := range h.open {
|
||||
recs = append(recs, r)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
sort.Slice(recs, func(i, j int) bool { return recs[i].Raised.Before(recs[j].Raised) })
|
||||
for _, r := range recs {
|
||||
switch {
|
||||
case !r.Cleared.IsZero():
|
||||
if now.Sub(r.Cleared) >= ReopenWindow {
|
||||
h.mu.Lock()
|
||||
delete(h.open, r.Key)
|
||||
h.mu.Unlock()
|
||||
if h.Store != nil {
|
||||
if err := h.Store.Delete(r.Key); err != nil {
|
||||
h.noteStore(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
case r.silenced(now):
|
||||
case r.Pending != "":
|
||||
h.deliver(r, r.Pending)
|
||||
h.persist(r)
|
||||
case !r.Reminded && !r.FirstSent.IsZero() && now.Sub(r.Raised) >= remindAfter(r.Severity):
|
||||
h.mu.Lock()
|
||||
r.Reminded = true
|
||||
h.mu.Unlock()
|
||||
h.deliver(r, "reminder")
|
||||
h.persist(r)
|
||||
}
|
||||
}
|
||||
h.flushFolds(now)
|
||||
}
|
||||
|
||||
func remindAfter(severity string) time.Duration {
|
||||
if severity == Urgent {
|
||||
return RemindUrgent
|
||||
}
|
||||
return RemindWarning
|
||||
}
|
||||
|
||||
// compose is the message for a record. withhold names what the content rule refused, so the words
|
||||
// that carried it are not sent.
|
||||
func compose(r *Record, what string, now time.Time, withhold int) Message {
|
||||
sev := strings.ToUpper(r.Severity)
|
||||
if sev == "" {
|
||||
sev = "WARNING"
|
||||
}
|
||||
summary := r.Summary
|
||||
if withhold > 0 {
|
||||
summary = "channel-refused: this message carried " + r.Refused + ", so its words are withheld"
|
||||
}
|
||||
var title string
|
||||
switch what {
|
||||
case "raised":
|
||||
title = sev + ": " + summary
|
||||
case "reminder":
|
||||
title = "STILL OPEN after " + roughly(now.Sub(r.Raised)) + ": " + summary
|
||||
case "escalated":
|
||||
title = "NOW URGENT: " + summary
|
||||
case "reopened":
|
||||
title = sev + " (open again, " + fmt.Sprint(r.Count) + " times): " + summary
|
||||
case "cleared":
|
||||
title = "CLEARED after " + roughly(r.Cleared.Sub(r.Raised)) + ": " + summary
|
||||
default:
|
||||
title = sev + ": " + summary
|
||||
}
|
||||
lines := []string{}
|
||||
if withhold < 3 && r.Subject != "" {
|
||||
about := "about: " + r.Subject
|
||||
if r.Kind != "" {
|
||||
about += " (" + r.Kind + ")"
|
||||
}
|
||||
lines = append(lines, about)
|
||||
}
|
||||
lines = append(lines, "since: "+r.Raised.UTC().Format("2006-01-02 15:04")+" UTC")
|
||||
if withhold < 2 {
|
||||
lines = append(lines, "key: "+r.Key)
|
||||
if r.More != "" {
|
||||
lines = append(lines, "more: "+r.More)
|
||||
}
|
||||
} else {
|
||||
lines = append(lines, "more: conditions (the open ones, through the mesh)")
|
||||
}
|
||||
return Message{Title: title, Body: strings.Join(lines, "\n"), Urgent: r.Severity == Urgent, Quiet: what == "cleared"}
|
||||
}
|
||||
|
||||
// say composes a record's message under the content rule: refused, it is composed again with less of
|
||||
// it, until what remains may leave. The refusal is recorded and stated once per record.
|
||||
func (h *Holder) say(r *Record, what string) Message {
|
||||
now := h.Now()
|
||||
if r.Refused != "" {
|
||||
// Refused before: its words stay withheld in every later message too.
|
||||
for w := 1; w <= 3; w++ {
|
||||
m := compose(r, what, now, w)
|
||||
if _, ok := Check(m.Text()); ok {
|
||||
return m
|
||||
}
|
||||
}
|
||||
}
|
||||
m := compose(r, what, now, 0)
|
||||
refusal, ok := Check(m.Text())
|
||||
if ok {
|
||||
return m
|
||||
}
|
||||
h.mu.Lock()
|
||||
r.Refused = refusal.What
|
||||
h.mu.Unlock()
|
||||
key := r.Key
|
||||
if _, keyOK := Check(key); !keyOK {
|
||||
key = "(withheld)"
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.refusals = append(h.refusals, RefusalNote{At: now, Key: key, Class: refusal.Class, What: refusal.What})
|
||||
if len(h.refusals) > KeptRefusals {
|
||||
h.refusals = h.refusals[len(h.refusals)-KeptRefusals:]
|
||||
}
|
||||
h.mu.Unlock()
|
||||
h.Logf("[messenger] refused the message for %s: it carried %s; sending channel-refused with its words withheld", key, refusal)
|
||||
if h.Emit != nil {
|
||||
if err := h.Emit("refused", map[string]any{"key": key, "class": refusal.Class, "what": refusal.What}); err != nil {
|
||||
h.Logf("[messenger] could not state the refusal on the bus: %v", err)
|
||||
}
|
||||
}
|
||||
for w := 1; w <= 3; w++ {
|
||||
m := compose(r, what, now, w)
|
||||
if _, ok := Check(m.Text()); ok {
|
||||
return m
|
||||
}
|
||||
}
|
||||
return Message{Title: "WARNING: channel-refused: a message carried " + refusal.What + " and was withheld",
|
||||
Body: "more: conditions (the open ones, through the mesh)", Urgent: r.Severity == Urgent}
|
||||
}
|
||||
|
||||
// deliver sends a record's message where its severity routes it.
|
||||
func (h *Holder) deliver(r *Record, what string) {
|
||||
m := h.say(r, what)
|
||||
now := h.Now()
|
||||
delivered := false
|
||||
if r.Severity == Urgent {
|
||||
for _, ch := range h.channels() {
|
||||
if h.sendOn(ch, r, what, m) {
|
||||
delivered = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if h.Desktop != nil && h.Desktop.Ready() == nil {
|
||||
delivered = h.sendOn(h.Desktop, r, what, m)
|
||||
}
|
||||
if !delivered && h.Telegram != nil {
|
||||
delivered = h.sendOn(h.Telegram, r, what, m)
|
||||
}
|
||||
}
|
||||
h.mu.Lock()
|
||||
if delivered {
|
||||
r.Pending = ""
|
||||
if r.FirstSent.IsZero() && (what == "raised" || what == "escalated") {
|
||||
r.FirstSent = now
|
||||
}
|
||||
} else {
|
||||
// Nothing took it: said in the status and the log, tried again next minute.
|
||||
r.Pending = what
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if !delivered {
|
||||
h.Logf("[messenger] could not send %s %s on any channel; trying again every minute: %s", what, r.Key, h.whyNot())
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) channels() []Channel {
|
||||
var out []Channel
|
||||
for _, c := range []Channel{h.Telegram, h.Desktop} {
|
||||
if c != nil {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sendOn sends one message on one channel under its cap; held by the cap, it is folded, which counts
|
||||
// as delivered — the fold will say it.
|
||||
func (h *Holder) sendOn(ch Channel, r *Record, what string, m Message) bool {
|
||||
if err := ch.Ready(); err != nil {
|
||||
h.noteChannel(ch.Name(), err)
|
||||
return false
|
||||
}
|
||||
if !h.allow(ch.Name()) {
|
||||
h.foldOn(ch.Name(), r, what)
|
||||
return true
|
||||
}
|
||||
id, err := ch.Send(m)
|
||||
h.noteChannel(ch.Name(), err)
|
||||
if err != nil {
|
||||
h.record(Sent{At: h.Now(), Channel: ch.Name(), Key: r.Key, What: what, Outcome: "failed: " + err.Error()})
|
||||
return false
|
||||
}
|
||||
h.mu.Lock()
|
||||
if r.Sent == nil {
|
||||
r.Sent = map[string]string{}
|
||||
}
|
||||
if _, has := r.Sent[ch.Name()]; !has && what != "cleared" {
|
||||
r.Sent[ch.Name()] = id
|
||||
}
|
||||
h.mu.Unlock()
|
||||
h.record(Sent{At: h.Now(), Channel: ch.Name(), Key: r.Key, What: what, Outcome: "sent"})
|
||||
return true
|
||||
}
|
||||
|
||||
// edit changes the first message on each channel that showed it; a channel that cannot, or whose
|
||||
// edit fails, is sent a new message instead.
|
||||
func (h *Holder) edit(r *Record, what string) {
|
||||
m := h.say(r, what)
|
||||
h.mu.Lock()
|
||||
sent := map[string]string{}
|
||||
for k, v := range r.Sent {
|
||||
sent[k] = v
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, ch := range h.channels() {
|
||||
id, shown := sent[ch.Name()]
|
||||
if !shown {
|
||||
continue
|
||||
}
|
||||
if ch.CanEdit() {
|
||||
err := ch.Edit(id, m)
|
||||
h.noteChannel(ch.Name(), err)
|
||||
if err == nil {
|
||||
h.record(Sent{At: h.Now(), Channel: ch.Name(), Key: r.Key, What: what, Outcome: "edited"})
|
||||
continue
|
||||
}
|
||||
h.Logf("[messenger] could not edit the message for %s on %s (%v); sending a new one", r.Key, ch.Name(), err)
|
||||
}
|
||||
h.sendOn(ch, r, what, m)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) foldOn(channel string, r *Record, what string) {
|
||||
h.mu.Lock()
|
||||
r.Folded = true
|
||||
list := h.folds[channel]
|
||||
replaced := false
|
||||
for i := range list {
|
||||
if list[i].Key == r.Key {
|
||||
list[i].What, list[i].Severity, replaced = what, r.Severity, true
|
||||
}
|
||||
}
|
||||
if !replaced {
|
||||
list = append(list, foldEntry{Key: r.Key, Severity: r.Severity, What: what})
|
||||
}
|
||||
h.folds[channel] = list
|
||||
first := len(list) == 1 && !replaced
|
||||
h.mu.Unlock()
|
||||
if first {
|
||||
h.Logf("[messenger] %s is at its cap of %d messages an hour: holding the rest, to be folded into one", channel, CapPerHour)
|
||||
}
|
||||
h.record(Sent{At: h.Now(), Channel: channel, Key: r.Key, What: what, Outcome: "folded"})
|
||||
}
|
||||
|
||||
func (h *Holder) flushFolds(now time.Time) {
|
||||
for _, ch := range h.channels() {
|
||||
h.mu.Lock()
|
||||
list := append([]foldEntry(nil), h.folds[ch.Name()]...)
|
||||
last := h.lastFold[ch.Name()]
|
||||
h.mu.Unlock()
|
||||
if len(list) == 0 || now.Sub(last) < FoldEvery {
|
||||
continue
|
||||
}
|
||||
urgent := false
|
||||
lines := []string{}
|
||||
for i, e := range list {
|
||||
if i == 40 {
|
||||
lines = append(lines, fmt.Sprintf("and %d more", len(list)-40))
|
||||
break
|
||||
}
|
||||
key := e.Key
|
||||
if _, ok := Check(key); !ok {
|
||||
key = "(a key withheld)"
|
||||
}
|
||||
lines = append(lines, e.Severity+" "+e.What+": "+key)
|
||||
urgent = urgent || e.Severity == Urgent
|
||||
}
|
||||
m := Message{
|
||||
Title: fmt.Sprintf("HELD BACK: %d message(s) over the cap of %d an hour", len(list), CapPerHour),
|
||||
Body: strings.Join(lines, "\n") + "\nmore: conditions (through the mesh)",
|
||||
Urgent: urgent,
|
||||
}
|
||||
if ch.Ready() != nil {
|
||||
continue
|
||||
}
|
||||
_, err := ch.Send(m)
|
||||
h.noteChannel(ch.Name(), err)
|
||||
if err != nil {
|
||||
h.record(Sent{At: now, Channel: ch.Name(), Key: "(folded)", What: "fold", Outcome: "failed: " + err.Error()})
|
||||
continue
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.folds[ch.Name()] = h.folds[ch.Name()][len(list):]
|
||||
h.lastFold[ch.Name()] = now
|
||||
h.mu.Unlock()
|
||||
h.record(Sent{At: now, Channel: ch.Name(), Key: "(folded)", What: fmt.Sprintf("fold of %d", len(list)), Outcome: "sent"})
|
||||
}
|
||||
}
|
||||
|
||||
// allow says whether a channel is under its cap: sends in the last hour, as recorded.
|
||||
func (h *Holder) allow(channel string) bool {
|
||||
return h.sentLastHour(channel) < CapPerHour
|
||||
}
|
||||
|
||||
func (h *Holder) sentLastHour(channel string) int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
since := h.Now().Add(-time.Hour)
|
||||
n := 0
|
||||
for _, s := range h.recent {
|
||||
if s.Channel == channel && s.Outcome == "sent" && s.At.After(since) && s.Key != "(folded)" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// sayOnce sends a message of the holder's own at most once per interval, on every channel ready.
|
||||
func (h *Holder) sayOnce(key string, every time.Duration, m Message) {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
if last, said := h.saidOnce[key]; said && now.Sub(last) < every {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.saidOnce[key] = now
|
||||
h.mu.Unlock()
|
||||
r := &Record{Key: key, Severity: Warning, Raised: now, Sent: map[string]string{}}
|
||||
for _, ch := range h.channels() {
|
||||
if ch.Ready() != nil {
|
||||
continue
|
||||
}
|
||||
if h.sendOn(ch, r, "notice", m) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) record(s Sent) {
|
||||
h.mu.Lock()
|
||||
h.recent = append(h.recent, s)
|
||||
if len(h.recent) > KeptSends {
|
||||
h.recent = h.recent[len(h.recent)-KeptSends:]
|
||||
}
|
||||
recent := append([]Sent(nil), h.recent...)
|
||||
h.mu.Unlock()
|
||||
if h.Store != nil {
|
||||
if err := h.Store.PutRecent(recent); err != nil {
|
||||
h.noteStore(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) persist(r *Record) {
|
||||
if h.Store == nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
c := *r
|
||||
h.mu.Unlock()
|
||||
if err := h.Store.Put(c); err != nil {
|
||||
h.noteStore(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) noteStore(err error) {
|
||||
h.mu.Lock()
|
||||
first := h.storeErr == ""
|
||||
h.storeErr = err.Error()
|
||||
h.mu.Unlock()
|
||||
if first {
|
||||
h.Logf("[messenger] cannot write its state (open messages are held in memory only until it can): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) noteChannel(name string, err error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
now := h.Now()
|
||||
if err == nil {
|
||||
if h.chanErr[name] != "" {
|
||||
h.Logf("[messenger] %s sends again", name)
|
||||
}
|
||||
h.chanErr[name] = ""
|
||||
h.chanOK[name] = now
|
||||
return
|
||||
}
|
||||
// Said in the log when it changes, and at most every ten minutes while it holds.
|
||||
if h.chanErr[name] != err.Error() || now.Sub(h.chanErrAt[name]) >= 10*time.Minute {
|
||||
h.Logf("[messenger] %s cannot send: %v", name, err)
|
||||
h.chanErrAt[name] = now
|
||||
}
|
||||
h.chanErr[name] = err.Error()
|
||||
}
|
||||
|
||||
func (h *Holder) whyNot() string {
|
||||
var parts []string
|
||||
for _, ch := range h.channels() {
|
||||
if err := ch.Ready(); err != nil {
|
||||
parts = append(parts, ch.Name()+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
h.mu.Lock()
|
||||
e := h.chanErr[ch.Name()]
|
||||
h.mu.Unlock()
|
||||
if e != "" {
|
||||
parts = append(parts, ch.Name()+": "+e)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "no channel"
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
func roughly(d time.Duration) string {
|
||||
switch {
|
||||
case d < 0:
|
||||
return "a moment"
|
||||
case d < 2*time.Minute:
|
||||
return fmt.Sprintf("%d s", int(d.Seconds()))
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d min", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%.1f h", d.Hours())
|
||||
}
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeChannel struct {
|
||||
name string
|
||||
notReady error
|
||||
fail error
|
||||
sends []Message
|
||||
edits map[string]Message
|
||||
n int
|
||||
}
|
||||
|
||||
func (f *fakeChannel) Name() string { return f.name }
|
||||
func (f *fakeChannel) Ready() error { return f.notReady }
|
||||
func (f *fakeChannel) CanEdit() bool { return true }
|
||||
func (f *fakeChannel) Send(m Message) (string, error) {
|
||||
if f.fail != nil {
|
||||
return "", f.fail
|
||||
}
|
||||
f.n++
|
||||
f.sends = append(f.sends, m)
|
||||
return fmt.Sprint(f.n), nil
|
||||
}
|
||||
func (f *fakeChannel) Edit(id string, m Message) error {
|
||||
if f.fail != nil {
|
||||
return f.fail
|
||||
}
|
||||
if f.edits == nil {
|
||||
f.edits = map[string]Message{}
|
||||
}
|
||||
f.edits[id] = m
|
||||
return nil
|
||||
}
|
||||
|
||||
type memStore struct {
|
||||
recs map[string]Record
|
||||
recent []Sent
|
||||
}
|
||||
|
||||
func (m *memStore) Put(r Record) error {
|
||||
if m.recs == nil {
|
||||
m.recs = map[string]Record{}
|
||||
}
|
||||
m.recs[r.Key] = r
|
||||
return nil
|
||||
}
|
||||
func (m *memStore) Delete(k string) error { delete(m.recs, k); return nil }
|
||||
func (m *memStore) All() ([]Record, error) {
|
||||
var out []Record
|
||||
for _, r := range m.recs {
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (m *memStore) PutRecent(s []Sent) error { m.recent = s; return nil }
|
||||
func (m *memStore) Recent() ([]Sent, error) { return m.recent, nil }
|
||||
|
||||
type clock struct{ t time.Time }
|
||||
|
||||
func (c *clock) now() time.Time { return c.t }
|
||||
func (c *clock) pass(d time.Duration) { c.t = c.t.Add(d) }
|
||||
func start() *clock { return &clock{t: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)} }
|
||||
func cond(key, sev, summary string) Condition {
|
||||
parts := strings.Split(key, ".")
|
||||
return Condition{Key: key, Scope: parts[0], ID: parts[1], Kind: parts[len(parts)-1], Severity: sev, Summary: summary}
|
||||
}
|
||||
|
||||
func newHolder(t *testing.T) (*Holder, *fakeChannel, *fakeChannel, *clock, *memStore) {
|
||||
t.Helper()
|
||||
c := start()
|
||||
tg, dt := &fakeChannel{name: "telegram"}, &fakeChannel{name: "desktop"}
|
||||
st := &memStore{}
|
||||
var emitted []string
|
||||
h := &Holder{Telegram: tg, Desktop: dt, Store: st, Now: c.now, Logf: t.Logf,
|
||||
Emit: func(e string, _ any) error { emitted = append(emitted, e); return nil }}
|
||||
h.init()
|
||||
return h, tg, dt, c, st
|
||||
}
|
||||
|
||||
func TestARaisedConditionIsSentOnceByItsKey(t *testing.T) {
|
||||
h, tg, dt, c, _ := newHolder(t)
|
||||
k := cond("machine.ace.silent", Urgent, "the home server has not been heard for 15 min")
|
||||
k.Raised = c.now()
|
||||
h.Condition(EventRaised, k)
|
||||
h.Condition(EventRaised, k) // a redelivery
|
||||
c.pass(time.Minute)
|
||||
h.Condition(EventRaised, k) // said again by the controller
|
||||
if len(tg.sends) != 1 || len(dt.sends) != 1 {
|
||||
t.Fatalf("urgent: telegram %d, desktop %d; want one each", len(tg.sends), len(dt.sends))
|
||||
}
|
||||
if !strings.Contains(tg.sends[0].Text(), "machine.ace.silent") || !strings.HasPrefix(tg.sends[0].Title, "URGENT: ") {
|
||||
t.Fatalf("message: %q", tg.sends[0].Text())
|
||||
}
|
||||
if open := h.Open(); len(open) != 1 || open[0].Count != 1 {
|
||||
t.Fatalf("open: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarningGoesToTheDesktopWhenASessionAnswersElseTelegram(t *testing.T) {
|
||||
h, tg, dt, _, _ := newHolder(t)
|
||||
h.Condition(EventRaised, cond("plan.41.stalled", Warning, "plan 41 waits on a build"))
|
||||
if len(dt.sends) != 1 || len(tg.sends) != 0 {
|
||||
t.Fatalf("desktop answered: desktop %d telegram %d", len(dt.sends), len(tg.sends))
|
||||
}
|
||||
dt.fail = errors.New("the account is not logged in")
|
||||
h.Condition(EventRaised, cond("plan.42.stalled", Warning, "plan 42 waits on a build"))
|
||||
if len(tg.sends) != 1 {
|
||||
t.Fatalf("no session: telegram %d", len(tg.sends))
|
||||
}
|
||||
dt.fail, dt.notReady = nil, errors.New("not configured")
|
||||
h.Condition(EventRaised, cond("plan.43.stalled", Warning, "plan 43 waits on a build"))
|
||||
if len(tg.sends) != 2 {
|
||||
t.Fatalf("no desktop configured: telegram %d", len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCapHoldsTheRestAndFoldsThemIntoOneMessage(t *testing.T) {
|
||||
h, tg, _, c, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
for i := 0; i < 25; i++ {
|
||||
h.Condition(EventRaised, cond(fmt.Sprintf("machine.m%d.silent", i), Urgent, "a machine is silent"))
|
||||
}
|
||||
if len(tg.sends) != CapPerHour {
|
||||
t.Fatalf("sent %d, cap %d", len(tg.sends), CapPerHour)
|
||||
}
|
||||
st := h.Status("listening")
|
||||
if st.Channels[0].Held != 5 || st.Channels[0].SentLastHour != CapPerHour {
|
||||
t.Fatalf("status: %+v", st.Channels[0])
|
||||
}
|
||||
// Said, not silent: the fold goes out at the first tick, and again only after ten minutes.
|
||||
h.Tick()
|
||||
if len(tg.sends) != CapPerHour+1 {
|
||||
t.Fatalf("no fold: %d sends", len(tg.sends))
|
||||
}
|
||||
fold := tg.sends[len(tg.sends)-1]
|
||||
if !strings.Contains(fold.Title, "HELD BACK: 5") {
|
||||
t.Fatalf("fold: %q", fold.Text())
|
||||
}
|
||||
for i := 20; i < 25; i++ {
|
||||
if !strings.Contains(fold.Body, fmt.Sprintf("machine.m%d.silent", i)) {
|
||||
t.Fatalf("fold does not name m%d: %q", i, fold.Body)
|
||||
}
|
||||
}
|
||||
h.Condition(EventRaised, cond("machine.late.silent", Urgent, "a machine is silent"))
|
||||
c.pass(time.Minute)
|
||||
h.Tick()
|
||||
if len(tg.sends) != CapPerHour+1 {
|
||||
t.Fatalf("a second fold inside ten minutes")
|
||||
}
|
||||
c.pass(FoldEvery)
|
||||
h.Tick()
|
||||
if last := tg.sends[len(tg.sends)-1]; !strings.Contains(last.Body, "machine.late.silent") {
|
||||
t.Fatalf("the second fold: %q", last.Text())
|
||||
}
|
||||
// An hour on, the window is free again.
|
||||
c.pass(time.Hour)
|
||||
h.Condition(EventRaised, cond("machine.next.silent", Urgent, "a machine is silent"))
|
||||
if last := tg.sends[len(tg.sends)-1]; !strings.Contains(last.Text(), "machine.next.silent") {
|
||||
t.Fatalf("not sent after the window: %q", last.Text())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMessageCarryingAnAddressIsRefusedAndSaidWithItsWordsWithheld(t *testing.T) {
|
||||
h, tg, _, _, _ := newHolder(t)
|
||||
var emitted []string
|
||||
h.Emit = func(e string, _ any) error { emitted = append(emitted, e); return nil }
|
||||
h.Condition(EventRaised, cond("provider.keycloak.ace.failing", Urgent, "cannot reach 192.168.1.20:8443 with token=abc"))
|
||||
if len(tg.sends) != 1 {
|
||||
t.Fatalf("sends: %d", len(tg.sends))
|
||||
}
|
||||
text := tg.sends[0].Text()
|
||||
if strings.Contains(text, "192.168") || strings.Contains(text, "token=") {
|
||||
t.Fatalf("the address left: %q", text)
|
||||
}
|
||||
if !strings.Contains(text, "channel-refused") || !strings.Contains(text, "provider.keycloak.ace.failing") {
|
||||
t.Fatalf("not said as refused, by key: %q", text)
|
||||
}
|
||||
hist := h.History(10)["refusals"].([]RefusalNote)
|
||||
if len(hist) != 1 || hist[0].Class != "address" {
|
||||
t.Fatalf("refusals: %+v", hist)
|
||||
}
|
||||
if len(emitted) != 1 || emitted[0] != "refused" {
|
||||
t.Fatalf("emitted: %v", emitted)
|
||||
}
|
||||
// Its clearing keeps the words withheld.
|
||||
h.Condition(EventCleared, Condition{Key: "provider.keycloak.ace.failing"})
|
||||
for _, m := range tg.edits {
|
||||
if strings.Contains(m.Text(), "192.168") {
|
||||
t.Fatalf("the clearing carried it: %q", m.Text())
|
||||
}
|
||||
}
|
||||
if len(tg.edits) != 1 {
|
||||
t.Fatalf("edits: %d", len(tg.edits))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyThatCarriesAnAddressIsWithheldToo(t *testing.T) {
|
||||
h, tg, _, _, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
h.Condition(EventRaised, cond("machine.10.0.0.7.silent", Urgent, "a machine is silent"))
|
||||
if len(tg.sends) != 1 || strings.Contains(tg.sends[0].Text(), "10.0.0.7") {
|
||||
t.Fatalf("sends: %+v", tg.sends)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStillOpenPastItsBoundItIsSaidOnceMore(t *testing.T) {
|
||||
h, tg, _, c, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
u := cond("bus.controller.slow-consumer", Urgent, "the controller's consumer is far behind")
|
||||
u.Raised = c.now()
|
||||
w := cond("plan.41.stalled", Warning, "plan 41 waits")
|
||||
w.Raised = c.now()
|
||||
h.Condition(EventRaised, u)
|
||||
h.Condition(EventRaised, w)
|
||||
c.pass(59 * time.Minute)
|
||||
h.Tick()
|
||||
if len(tg.sends) != 2 {
|
||||
t.Fatalf("reminded before the hour: %d", len(tg.sends))
|
||||
}
|
||||
c.pass(2 * time.Minute)
|
||||
h.Tick()
|
||||
if len(tg.sends) != 3 || !strings.HasPrefix(tg.sends[2].Title, "STILL OPEN after 61 min") {
|
||||
t.Fatalf("urgent reminder: %d %q", len(tg.sends), tg.sends[len(tg.sends)-1].Title)
|
||||
}
|
||||
c.pass(3 * time.Hour)
|
||||
h.Tick()
|
||||
if len(tg.sends) != 3 {
|
||||
t.Fatalf("reminded twice")
|
||||
}
|
||||
c.pass(9 * time.Hour) // the warning is now 13 h old
|
||||
h.Tick()
|
||||
if len(tg.sends) != 4 || !strings.Contains(tg.sends[3].Text(), "plan.41.stalled") {
|
||||
t.Fatalf("warning reminder: %d", len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearedEditsTheFirstMessageAndReopenedWithinTenMinutesIsNotNew(t *testing.T) {
|
||||
h, tg, dt, c, st := newHolder(t)
|
||||
k := cond("machine.ace.silent", Urgent, "the home server is silent")
|
||||
k.Raised = c.now()
|
||||
h.Condition(EventRaised, k)
|
||||
c.pass(14 * time.Minute)
|
||||
h.Condition(EventCleared, Condition{Key: k.Key})
|
||||
if len(tg.sends) != 1 || len(tg.edits) != 1 || len(dt.edits) != 1 {
|
||||
t.Fatalf("telegram sends %d edits %d, desktop edits %d", len(tg.sends), len(tg.edits), len(dt.edits))
|
||||
}
|
||||
if m := tg.edits["1"]; !strings.HasPrefix(m.Title, "CLEARED after 14 min") {
|
||||
t.Fatalf("edit: %q", m.Title)
|
||||
}
|
||||
if len(h.Open()) != 0 {
|
||||
t.Fatalf("still open")
|
||||
}
|
||||
c.pass(5 * time.Minute)
|
||||
h.Condition(EventRaised, k)
|
||||
if len(tg.sends) != 1 || !strings.Contains(tg.edits["1"].Title, "open again, 2 times") {
|
||||
t.Fatalf("reopened as new: sends %d, edit %q", len(tg.sends), tg.edits["1"].Title)
|
||||
}
|
||||
h.Condition(EventCleared, Condition{Key: k.Key})
|
||||
c.pass(11 * time.Minute)
|
||||
h.Tick()
|
||||
if _, kept := st.recs[k.Key]; kept {
|
||||
t.Fatalf("a cleared message kept past the reopen window")
|
||||
}
|
||||
h.Condition(EventRaised, k)
|
||||
if len(tg.sends) != 2 {
|
||||
t.Fatalf("a raise after the window is a new message: %d", len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilencedConditionSendsNothing(t *testing.T) {
|
||||
h, tg, dt, c, _ := newHolder(t)
|
||||
k := cond("machine.ace.silent", Urgent, "silent")
|
||||
k.SilencedTill = c.now().Add(2 * time.Hour)
|
||||
h.Condition(EventRaised, k)
|
||||
c.pass(90 * time.Minute)
|
||||
h.Tick()
|
||||
h.Condition(EventCleared, Condition{Key: k.Key})
|
||||
if len(tg.sends)+len(dt.sends)+len(tg.edits)+len(dt.edits) != 0 {
|
||||
t.Fatalf("a silenced condition said something")
|
||||
}
|
||||
// Silenced after it was sent: no reminder.
|
||||
k2 := cond("machine.shanks.silent", Urgent, "silent")
|
||||
k2.Raised = c.now()
|
||||
h.Condition(EventRaised, k2)
|
||||
k2.SilencedTill = c.now().Add(3 * time.Hour)
|
||||
h.Condition(EventChanged, k2)
|
||||
c.pass(2 * time.Hour)
|
||||
h.Tick()
|
||||
if len(tg.sends) != 1 {
|
||||
t.Fatalf("reminded while silenced: %d", len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscalationIsSaidOnce(t *testing.T) {
|
||||
h, tg, dt, _, _ := newHolder(t)
|
||||
k := cond("machine.novox.silent", Warning, "the anchor is silent")
|
||||
h.Condition(EventRaised, k)
|
||||
k.Severity = Urgent
|
||||
h.Condition(EventChanged, k)
|
||||
h.Condition(EventChanged, k)
|
||||
if len(dt.sends) != 2 || len(tg.sends) != 1 || !strings.HasPrefix(tg.sends[0].Title, "NOW URGENT") {
|
||||
t.Fatalf("desktop %d telegram %d", len(dt.sends), len(tg.sends))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAChannelThatCannotSendSaysSoAndIsTriedAgain(t *testing.T) {
|
||||
h, tg, _, _, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
tg.fail = errors.New("telegram sendMessage: cannot connect (dial)")
|
||||
h.Condition(EventRaised, cond("machine.ace.silent", Urgent, "silent"))
|
||||
st := h.Status("listening")
|
||||
if !strings.HasPrefix(st.Verdict, "CANNOT SEND") || len(st.Unsent) != 1 || st.Channels[0].LastError == "" {
|
||||
t.Fatalf("status: %+v", st)
|
||||
}
|
||||
tg.fail = nil
|
||||
h.Tick()
|
||||
if len(tg.sends) != 1 || h.Status("listening").Verdict != "ok" {
|
||||
t.Fatalf("not retried: %d, %s", len(tg.sends), h.Status("listening").Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingConfiguredIsSaidInTheStatus(t *testing.T) {
|
||||
h, tg, dt, _, _ := newHolder(t)
|
||||
tg.notReady = errors.New("no bot token")
|
||||
dt.notReady = errors.New("no machine")
|
||||
st := h.Status("listening")
|
||||
if !strings.HasPrefix(st.Verdict, "CANNOT SEND") || st.Channels[0].NotReady != "no bot token" {
|
||||
t.Fatalf("status: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARestartForgetsNothing(t *testing.T) {
|
||||
h, tg, _, c, st := newHolder(t)
|
||||
h.Desktop = nil
|
||||
k := cond("machine.ace.silent", Urgent, "silent")
|
||||
k.Raised = c.now()
|
||||
h.Condition(EventRaised, k)
|
||||
again := &Holder{Telegram: tg, Store: st, Now: c.now, Logf: t.Logf}
|
||||
if err := again.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again.Condition(EventRaised, k)
|
||||
if len(tg.sends) != 1 {
|
||||
t.Fatalf("a restart sent it again")
|
||||
}
|
||||
again.Condition(EventCleared, Condition{Key: k.Key})
|
||||
if len(tg.edits) != 1 {
|
||||
t.Fatalf("a restart lost the message to edit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableEventIsToldOnceAnHour(t *testing.T) {
|
||||
h, tg, _, c, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
h.Unreadable("mesh-controller.condition-raised", errors.New("no severity"))
|
||||
h.Unreadable("mesh-controller.condition-raised", errors.New("no severity"))
|
||||
if len(tg.sends) != 1 || h.Status("listening").Unreadable != 2 {
|
||||
t.Fatalf("sends %d", len(tg.sends))
|
||||
}
|
||||
c.pass(61 * time.Minute)
|
||||
h.Unreadable("mesh-controller.condition-raised", errors.New("no severity"))
|
||||
if len(tg.sends) != 2 {
|
||||
t.Fatalf("not said again after an hour")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotifyIsKeptApartFromConditions(t *testing.T) {
|
||||
h, tg, _, _, _ := newHolder(t)
|
||||
h.Desktop = nil
|
||||
out, err := h.Notify("backup.ace.failed", Warning, "last night's backup of the home server failed", "backup", false)
|
||||
if err != nil || out["key"] != "notify.backup.ace.failed" || len(tg.sends) != 1 {
|
||||
t.Fatalf("%v %v %d", out, err, len(tg.sends))
|
||||
}
|
||||
if _, err := h.Notify("x", "", "s", "", false); err == nil {
|
||||
t.Fatalf("no severity was not refused")
|
||||
}
|
||||
if _, err := h.Notify("x", Urgent, "see /var/lib/x", "", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(tg.sends[len(tg.sends)-1].Text(), "/var/lib") {
|
||||
t.Fatalf("a path left")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATestMessageRespectsTheRule(t *testing.T) {
|
||||
h, tg, dt, _, _ := newHolder(t)
|
||||
if out := h.Test("telegram", "see https://example.org"); out["refused"] == "" || len(tg.sends) != 0 {
|
||||
t.Fatalf("%v", out)
|
||||
}
|
||||
if out := h.Test("", ""); out["telegram"] != "sent" || out["desktop"] != "sent" || len(dt.sends) != 1 {
|
||||
t.Fatalf("%v", out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
// messenger: the holder of the operator-channel seat (novox/hq to-be 45 §5, ADR 0227, research 028).
|
||||
// A Go bundle the node's runtime launches. It consumes the controller's condition events and decides
|
||||
// what is said to the operator, on Telegram and on the desktop notifier of the machine the operator
|
||||
// is at. It keeps its open messages in its own state, so a restart forgets nothing. stdout is the MCP
|
||||
// channel; what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func errorf(format string, a ...any) error { return fmt.Errorf(format, a...) }
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
// Settings are the operator's values for this module, merged by the mesh into one JSON file.
|
||||
type Settings struct {
|
||||
TelegramChatID string `json:"telegram-chat-id"`
|
||||
DesktopMachines []string `json:"desktop-machines"`
|
||||
}
|
||||
|
||||
func readSettings(path string) (Settings, error) {
|
||||
var s Settings
|
||||
if path == "" {
|
||||
return s, errors.New("started without a settings file")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
// The chat id may be given as a number.
|
||||
var loose map[string]any
|
||||
if err := json.Unmarshal(raw, &loose); err != nil {
|
||||
return s, fmt.Errorf("the settings file is not JSON: %v", err)
|
||||
}
|
||||
switch v := loose["telegram-chat-id"].(type) {
|
||||
case string:
|
||||
s.TelegramChatID = strings.TrimSpace(v)
|
||||
case float64:
|
||||
s.TelegramChatID = fmt.Sprintf("%.0f", v)
|
||||
}
|
||||
if list, ok := loose["desktop-machines"].([]any); ok {
|
||||
for _, m := range list {
|
||||
if name, ok := m.(string); ok && strings.TrimSpace(name) != "" {
|
||||
s.DesktopMachines = append(s.DesktopMachines, strings.TrimSpace(name))
|
||||
}
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// stateStore keeps the open messages in the module's declared state `open`, and the recent sends
|
||||
// under one key of `sent` (ADR 0201).
|
||||
type stateStore struct{}
|
||||
|
||||
// kvKey is a condition key as a bucket key: only the characters a key may carry.
|
||||
func kvKey(key string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range key {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '.', r == '=':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteRune('_')
|
||||
}
|
||||
}
|
||||
return strings.Trim(b.String(), ".")
|
||||
}
|
||||
|
||||
func (stateStore) Put(r Record) error {
|
||||
_, err := stdio.State("open").Put(kvKey(r.Key), r)
|
||||
return err
|
||||
}
|
||||
func (stateStore) Delete(key string) error { return stdio.State("open").Delete(kvKey(key)) }
|
||||
func (stateStore) All() ([]Record, error) {
|
||||
keys, err := stdio.State("open").Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []Record
|
||||
for _, k := range keys {
|
||||
e, err := stdio.State("open").Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
var r Record
|
||||
if err := json.Unmarshal(e.Value, &r); err != nil {
|
||||
logf("[messenger] the open message kept as %s cannot be read (%v); left as it is", k, err)
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (stateStore) PutRecent(s []Sent) error {
|
||||
_, err := stdio.State("sent").Put("recent", s)
|
||||
return err
|
||||
}
|
||||
func (stateStore) Recent() ([]Sent, error) {
|
||||
e, err := stdio.State("sent").Get("recent")
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
var s []Sent
|
||||
return s, json.Unmarshal(e.Value, &s)
|
||||
}
|
||||
|
||||
// listening is whether condition events reach this holder, in words.
|
||||
type listening struct {
|
||||
mu sync.Mutex
|
||||
now string
|
||||
}
|
||||
|
||||
func (l *listening) set(s string) { l.mu.Lock(); l.now = s; l.mu.Unlock() }
|
||||
func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.now }
|
||||
|
||||
func main() {
|
||||
settingsFile := os.Getenv("MESH_MESSENGER_SETTINGS")
|
||||
var said sync.Mutex
|
||||
lastSaid := ""
|
||||
settings := func() Settings {
|
||||
s, err := readSettings(settingsFile)
|
||||
said.Lock()
|
||||
defer said.Unlock()
|
||||
if err != nil && err.Error() != lastSaid {
|
||||
logf("[messenger] settings cannot be read: %v", err)
|
||||
}
|
||||
lastSaid = ""
|
||||
if err != nil {
|
||||
lastSaid = err.Error()
|
||||
}
|
||||
return s
|
||||
}
|
||||
h := &Holder{
|
||||
Telegram: NewTelegram(TelegramConfig{
|
||||
TokenFile: os.Getenv("MESH_MESSENGER_TELEGRAM_TOKEN_FILE"),
|
||||
ChatID: func() string { return settings().TelegramChatID },
|
||||
}),
|
||||
Desktop: &Desktop{
|
||||
Machines: func() []string { return settings().DesktopMachines },
|
||||
Ask: stdio.Ask,
|
||||
},
|
||||
Store: stateStore{},
|
||||
Logf: logf,
|
||||
Emit: func(event string, body any) error { return stdio.Emit(event, body) },
|
||||
}
|
||||
h.init()
|
||||
l := &listening{now: "not yet: starting"}
|
||||
go run(h, l)
|
||||
if err := stdio.Serve("", tools(h, l)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// run reads back the state, listens for condition events, and keeps time — each retried, and each
|
||||
// failure said, never given up on quietly.
|
||||
func run(h *Holder, l *listening) {
|
||||
time.Sleep(500 * time.Millisecond) // Serve first: the state is reached through it
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := h.Load()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
logf("[messenger] cannot read back its open messages yet (%v); asking again in %s", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
for _, ch := range h.channels() {
|
||||
if err := ch.Ready(); err != nil {
|
||||
logf("[messenger] %s cannot send: %v", ch.Name(), err)
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
for range time.Tick(time.Minute) {
|
||||
h.Tick()
|
||||
}
|
||||
}()
|
||||
handle := func(e stdio.Envelope) error {
|
||||
event, ok := eventOf(e.Key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
c, err := DecodeCondition(event, e.Body)
|
||||
if err != nil {
|
||||
h.Unreadable(e.Key, err)
|
||||
return nil
|
||||
}
|
||||
h.Condition(event, c)
|
||||
return nil
|
||||
}
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := stdio.Subscribe(ControllerSeat+".*", handle)
|
||||
if err == nil {
|
||||
l.set("listening")
|
||||
logf("[messenger] listening for the controller's condition events")
|
||||
return
|
||||
}
|
||||
l.set("not yet: " + err.Error())
|
||||
logf("[messenger] not hearing condition events yet (%v); asking again in %s", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
func str(description string) map[string]any {
|
||||
return map[string]any{"type": "string", "description": description}
|
||||
}
|
||||
|
||||
func strArg(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) }
|
||||
|
||||
func limitArg(a map[string]any, def int) int {
|
||||
if v, ok := a["limit"].(float64); ok && v >= 1 {
|
||||
return min(int(v), KeptSends)
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func tools(h *Holder, l *listening) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: "operator-channel.open",
|
||||
Description: "What is open now: every message the operator was sent about something still wrong, urgent first, " +
|
||||
"oldest first — its key, severity, summary, since when, the channels it went to, whether it is silenced, " +
|
||||
"reminded, held by the cap, refused, or not sent yet.",
|
||||
Run: func(map[string]any) (any, error) { return h.Open(), nil }},
|
||||
{Name: "operator-channel.history",
|
||||
Description: "What was said to the operator lately, newest first — each send, edit, fold and failure with its " +
|
||||
"channel, key and outcome — and every message refused for carrying an address, a path or a secret.",
|
||||
Input: map[string]any{"limit": map[string]any{"type": "integer", "description": "at most this many of each (default 50)"}},
|
||||
Run: func(a map[string]any) (any, error) { return h.History(limitArg(a, 50)), nil }},
|
||||
{Name: "operator-channel.notify",
|
||||
Description: "Tell the operator something, as a module that uses the seat: a key (the same key is the same " +
|
||||
"message), urgent or warning, one line in the mesh's words, and what it is about. clear says it is over. " +
|
||||
"Roles and words only: an address, a path or a secret is refused. Deduplicated, capped and routed " +
|
||||
"like the controller's conditions.",
|
||||
Input: map[string]any{
|
||||
"key": str("what makes it the same message the next time, e.g. backup.ace.failed"),
|
||||
"severity": map[string]any{"type": "string", "enum": []string{Urgent, Warning}},
|
||||
"summary": str("one line in the mesh's words"),
|
||||
"subject": str("what it is about: a machine's role, a module, a plan"),
|
||||
"clear": map[string]any{"type": "boolean", "description": "it is over: the message is edited to say so"},
|
||||
},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
clear, _ := a["clear"].(bool)
|
||||
return h.Notify(strArg(a, "key"), strArg(a, "severity"), strArg(a, "summary"), strArg(a, "subject"), clear)
|
||||
}},
|
||||
{Name: "messenger_status",
|
||||
Description: "Whether the operator can be reached, and why not: each channel — can it send, what it lacks " +
|
||||
"(the Telegram bot token and chat id, the desktop machines), its last error, how many it sent in the " +
|
||||
"last hour against the cap, how many it holds — the open and silenced count, messages not sent yet, " +
|
||||
"refusals, unreadable events, whether condition events arrive, and the rules it applies. check asks " +
|
||||
"Telegram who the bot is, sending nothing.",
|
||||
Input: map[string]any{"check": map[string]any{"type": "boolean", "description": "also ask Telegram whether the token works"}},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
st := h.Status(l.get())
|
||||
if check, _ := a["check"].(bool); check {
|
||||
if t, ok := h.Telegram.(*Telegram); ok {
|
||||
who, err := t.Who()
|
||||
if err != nil {
|
||||
return map[string]any{"status": st, "telegram_check": "failed: " + err.Error()}, nil
|
||||
}
|
||||
return map[string]any{"status": st, "telegram_check": "the token works: the bot is @" + who}, nil
|
||||
}
|
||||
}
|
||||
return st, nil
|
||||
}},
|
||||
{Name: "messenger_recent",
|
||||
Description: "The recent sends — each message, edit, fold and failure on each channel, newest first — and the refusals.",
|
||||
Input: map[string]any{"limit": map[string]any{"type": "integer", "description": "at most this many (default 20)"}},
|
||||
Run: func(a map[string]any) (any, error) { return h.History(limitArg(a, 20)), nil }},
|
||||
{Name: "messenger_test",
|
||||
Description: "Send a test message now, to telegram, desktop or both (default both), through the content rule " +
|
||||
"and the cap: proves a channel reaches the operator. Answers per channel: sent, or why not.",
|
||||
Input: map[string]any{
|
||||
"channel": map[string]any{"type": "string", "enum": []string{"telegram", "desktop", "both"}},
|
||||
"text": str("the words (default: a line saying it is a test)"),
|
||||
},
|
||||
Run: func(a map[string]any) (any, error) { return h.Test(strArg(a, "channel"), strArg(a, "text")), nil }},
|
||||
{Name: "messenger_check",
|
||||
Description: "Would these words be allowed to leave the mesh? Runs the content rule — no address, path or " +
|
||||
"secret — and answers allowed, or the class of what it carried. Sends nothing.",
|
||||
Input: map[string]any{"text": str("the words to judge")},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if r, ok := Check(strArg(a, "text")); !ok {
|
||||
return map[string]any{"allowed": false, "class": r.Class, "carried": r.What}, nil
|
||||
}
|
||||
return map[string]any{"allowed": true}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The manifest says what the code does: the seat it declares and holds and the verbs it serves, the
|
||||
// events it consumes (the controller's three), the one it emits, the tool it calls, its state, its
|
||||
// secret, and its own tools — and names nothing of one installation.
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Seats []seat `json:"seats"`
|
||||
Claims []seat `json:"claims"`
|
||||
Consumes []string
|
||||
Emits []string
|
||||
Invokes []string
|
||||
State []string
|
||||
Own map[string]string `json:"own-secrets"`
|
||||
Tools []string
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
type seat struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T) (manifest, string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, string(raw)
|
||||
}
|
||||
|
||||
func TestItDeclaresAndHoldsTheOperatorChannel(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
want := seat{Name: "operator-channel", Scope: "mesh", Serves: []string{"open", "history", "notify"}}
|
||||
if len(m.Seats) != 1 || !reflect.DeepEqual(m.Seats[0], want) || len(m.Claims) != 1 || !reflect.DeepEqual(m.Claims[0], want) {
|
||||
t.Fatalf("seats %+v claims %+v", m.Seats, m.Claims)
|
||||
}
|
||||
if !reflect.DeepEqual(m.Consumes, []string{
|
||||
ControllerSeat + "." + EventRaised, ControllerSeat + "." + EventChanged, ControllerSeat + "." + EventCleared}) {
|
||||
t.Fatalf("consumes %v", m.Consumes)
|
||||
}
|
||||
if !reflect.DeepEqual(m.Emits, []string{"refused"}) || !reflect.DeepEqual(m.Invokes, []string{"seat:node-notifier.send"}) {
|
||||
t.Fatalf("emits %v invokes %v", m.Emits, m.Invokes)
|
||||
}
|
||||
if !reflect.DeepEqual(m.State, []string{"open", "sent"}) {
|
||||
t.Fatalf("state %v", m.State)
|
||||
}
|
||||
if m.Own["telegram-token"] == "" {
|
||||
t.Fatalf("own secrets %v", m.Own)
|
||||
}
|
||||
env, _ := m.Build.Artifacts[0]["env"].(map[string]any)
|
||||
if env["MESH_MESSENGER_TELEGRAM_TOKEN_FILE"] != m.Own["telegram-token"] {
|
||||
t.Fatalf("the bundle reads its token from %v, the secret is at %v", env["MESH_MESSENGER_TELEGRAM_TOKEN_FILE"], m.Own["telegram-token"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsAgreeWithTheManifest(t *testing.T) {
|
||||
m, raw := readManifest(t)
|
||||
var own, verbs []string
|
||||
for _, tool := range tools(&Holder{}, &listening{}) {
|
||||
if strings.TrimSpace(tool.Description) == "" {
|
||||
t.Errorf("%s has no description", tool.Name)
|
||||
}
|
||||
if seatName, verb, ok := strings.Cut(tool.Name, "."); ok {
|
||||
if seatName != "operator-channel" {
|
||||
t.Errorf("%s is a verb of a seat this does not hold", tool.Name)
|
||||
}
|
||||
verbs = append(verbs, verb)
|
||||
continue
|
||||
}
|
||||
own = append(own, tool.Name)
|
||||
}
|
||||
listed := append([]string(nil), m.Tools...)
|
||||
sort.Strings(own)
|
||||
sort.Strings(listed)
|
||||
if !reflect.DeepEqual(own, listed) {
|
||||
t.Fatalf("serves %v, lists %v", own, listed)
|
||||
}
|
||||
if !reflect.DeepEqual(verbs, m.Claims[0].Serves) {
|
||||
t.Fatalf("verbs %v, claim %v", verbs, m.Claims[0].Serves)
|
||||
}
|
||||
for _, never := range []string{"/home/", "jochen", "g14", "shanks", "novox", "zurag", "api.telegram"} {
|
||||
if strings.Contains(strings.ToLower(raw), never) {
|
||||
t.Errorf("module.json names %q", never)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ChannelStatus is one channel as the status says it.
|
||||
type ChannelStatus struct {
|
||||
Name string `json:"name"`
|
||||
CanSend bool `json:"can_send"`
|
||||
NotReady string `json:"not_ready,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
LastSent string `json:"last_sent,omitempty"`
|
||||
SentLastHour int `json:"sent_last_hour"`
|
||||
Cap int `json:"cap_per_hour"`
|
||||
Held int `json:"held_by_cap"`
|
||||
}
|
||||
|
||||
// Status is the holder's whole account of itself.
|
||||
type Status struct {
|
||||
Verdict string `json:"verdict"`
|
||||
Channels []ChannelStatus `json:"channels"`
|
||||
Open int `json:"open"`
|
||||
Silenced int `json:"silenced"`
|
||||
Unsent []string `json:"unsent,omitempty"`
|
||||
Refused int `json:"refused_since_start"`
|
||||
Unreadable int `json:"unreadable_events_since_start"`
|
||||
LastBad string `json:"last_unreadable,omitempty"`
|
||||
Heard map[string]int `json:"events_heard_since_start"`
|
||||
LastHeard string `json:"last_event_heard,omitempty"`
|
||||
Listening string `json:"listening"`
|
||||
StateProblem string `json:"state_problem,omitempty"`
|
||||
Rules []string `json:"rules"`
|
||||
}
|
||||
|
||||
// Status answers at once from what the holder keeps.
|
||||
func (h *Holder) Status(listening string) Status {
|
||||
now := h.Now()
|
||||
var st Status
|
||||
for _, ch := range h.channels() {
|
||||
cs := ChannelStatus{Name: ch.Name(), Cap: CapPerHour, SentLastHour: h.sentLastHour(ch.Name())}
|
||||
if err := ch.Ready(); err != nil {
|
||||
cs.NotReady = err.Error()
|
||||
} else {
|
||||
cs.CanSend = true
|
||||
}
|
||||
h.mu.Lock()
|
||||
cs.LastError = h.chanErr[ch.Name()]
|
||||
if t, ok := h.chanOK[ch.Name()]; ok {
|
||||
cs.LastSent = t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
cs.Held = len(h.folds[ch.Name()])
|
||||
h.mu.Unlock()
|
||||
if cs.LastError != "" {
|
||||
cs.CanSend = false
|
||||
}
|
||||
st.Channels = append(st.Channels, cs)
|
||||
}
|
||||
h.mu.Lock()
|
||||
for _, r := range h.open {
|
||||
if !r.Cleared.IsZero() {
|
||||
continue
|
||||
}
|
||||
st.Open++
|
||||
if r.silenced(now) {
|
||||
st.Silenced++
|
||||
}
|
||||
if r.Pending != "" {
|
||||
st.Unsent = append(st.Unsent, r.Key+" ("+r.Pending+")")
|
||||
}
|
||||
}
|
||||
st.Refused = len(h.refusals)
|
||||
st.Unreadable = h.unreadable
|
||||
st.LastBad = h.lastBad
|
||||
st.Heard = map[string]int{}
|
||||
for k, v := range h.heard {
|
||||
st.Heard[k] = v
|
||||
}
|
||||
if !h.lastHeard.IsZero() {
|
||||
st.LastHeard = h.lastHeard.UTC().Format(time.RFC3339)
|
||||
}
|
||||
st.StateProblem = h.storeErr
|
||||
h.mu.Unlock()
|
||||
sort.Strings(st.Unsent)
|
||||
st.Listening = listening
|
||||
st.Rules = []string{
|
||||
"urgent: Telegram and the desktop; warning: the desktop where a session answers, otherwise Telegram",
|
||||
"deduplicated by the condition's key; reminded once after 1 h (urgent) or 12 h (warning); cleared by editing the first message",
|
||||
"at most 20 messages an hour per channel; the rest folded into one message, at most every 10 min",
|
||||
"refused: anything carrying an address, a path or a secret; channel-refused is sent with the words withheld",
|
||||
"silenced conditions send nothing; silence is `conditions silence` through the mesh",
|
||||
}
|
||||
anyCan := false
|
||||
for _, c := range st.Channels {
|
||||
anyCan = anyCan || c.CanSend
|
||||
}
|
||||
switch {
|
||||
case !anyCan:
|
||||
st.Verdict = "CANNOT SEND: no channel can reach the operator — see channels"
|
||||
case len(st.Unsent) > 0:
|
||||
st.Verdict = "BEHIND: some messages could not be sent yet and are tried every minute"
|
||||
case listening != "listening":
|
||||
st.Verdict = "NOT LISTENING: condition events are not reaching this holder — " + listening
|
||||
case st.StateProblem != "":
|
||||
st.Verdict = "STATE: open messages are held in memory only — " + st.StateProblem
|
||||
default:
|
||||
st.Verdict = "ok"
|
||||
for _, c := range st.Channels {
|
||||
if !c.CanSend {
|
||||
st.Verdict = "ok on one channel: " + c.Name + " cannot send"
|
||||
}
|
||||
}
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// OpenMessage is one open message as `open` answers it.
|
||||
type OpenMessage struct {
|
||||
Key string `json:"key"`
|
||||
Severity string `json:"severity"`
|
||||
Summary string `json:"summary"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Since string `json:"since"`
|
||||
Origin string `json:"origin"`
|
||||
SentOn []string `json:"sent_on,omitempty"`
|
||||
Silenced string `json:"silenced_until,omitempty"`
|
||||
Reminded bool `json:"reminded"`
|
||||
Unsent string `json:"unsent,omitempty"`
|
||||
Held bool `json:"held_by_cap,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Count int `json:"times_opened"`
|
||||
}
|
||||
|
||||
// Open is what is unresolved now, urgent first, oldest first.
|
||||
func (h *Holder) Open() []OpenMessage {
|
||||
now := h.Now()
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
var out []OpenMessage
|
||||
for _, r := range h.open {
|
||||
if !r.Cleared.IsZero() {
|
||||
continue
|
||||
}
|
||||
o := OpenMessage{Key: r.Key, Severity: r.Severity, Summary: r.Summary, Subject: r.Subject,
|
||||
Since: r.Raised.UTC().Format(time.RFC3339), Origin: r.Origin, Reminded: r.Reminded,
|
||||
Unsent: r.Pending, Held: r.Folded, Refused: r.Refused, Count: r.Count}
|
||||
for ch := range r.Sent {
|
||||
o.SentOn = append(o.SentOn, ch)
|
||||
}
|
||||
sort.Strings(o.SentOn)
|
||||
if r.silenced(now) {
|
||||
o.Silenced = r.SilencedTill.UTC().Format(time.RFC3339)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Severity != out[j].Severity {
|
||||
return out[i].Severity == Urgent
|
||||
}
|
||||
return out[i].Since < out[j].Since
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// History is what went out, newest first, and the refusals.
|
||||
func (h *Holder) History(limit int) map[string]any {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
sends := []Sent{}
|
||||
for i := len(h.recent) - 1; i >= 0 && len(sends) < limit; i-- {
|
||||
sends = append(sends, h.recent[i])
|
||||
}
|
||||
refusals := []RefusalNote{}
|
||||
for i := len(h.refusals) - 1; i >= 0 && len(refusals) < limit; i-- {
|
||||
refusals = append(refusals, h.refusals[i])
|
||||
}
|
||||
return map[string]any{"sends": sends, "refusals": refusals}
|
||||
}
|
||||
|
||||
// Test sends a test message on one channel or both, through the content rule and the cap.
|
||||
func (h *Holder) Test(channel, text string) map[string]string {
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
if text == "" {
|
||||
text = "a test from the operator-channel's holder: this channel reaches you"
|
||||
}
|
||||
out := map[string]string{}
|
||||
if refusal, ok := Check(text); !ok {
|
||||
out["refused"] = "it carried " + refusal.String() + "; nothing was sent"
|
||||
return out
|
||||
}
|
||||
r := &Record{Key: "messenger.test", Severity: Warning, Raised: h.Now(), Sent: map[string]string{}}
|
||||
m := Message{Title: "TEST: " + text, Body: "nothing is wrong; this was asked for"}
|
||||
for _, ch := range h.channels() {
|
||||
if channel != "" && channel != "both" && channel != ch.Name() {
|
||||
continue
|
||||
}
|
||||
if err := ch.Ready(); err != nil {
|
||||
out[ch.Name()] = "not sent: " + err.Error()
|
||||
continue
|
||||
}
|
||||
if !h.allow(ch.Name()) {
|
||||
out[ch.Name()] = "not sent: the channel is at its cap of 20 an hour"
|
||||
continue
|
||||
}
|
||||
if h.sendOn(ch, r, "test", m) {
|
||||
out[ch.Name()] = "sent"
|
||||
} else {
|
||||
h.mu.Lock()
|
||||
out[ch.Name()] = "failed: " + h.chanErr[ch.Name()]
|
||||
h.mu.Unlock()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Notify takes a message from a module that uses the seat (research 028 Q5, modules as sources):
|
||||
// the same shape, rule, cap and deduplication as a condition. Its key is put under `notify.` so it
|
||||
// can never be taken for one of the controller's conditions.
|
||||
func (h *Holder) Notify(key, severity, summary, subject string, clear bool) (map[string]any, error) {
|
||||
if key == "" {
|
||||
return nil, errorf("key is required: the same key is the same message")
|
||||
}
|
||||
if len(key) < 7 || key[:7] != "notify." {
|
||||
key = "notify." + key
|
||||
}
|
||||
h.work.Lock()
|
||||
defer h.work.Unlock()
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
h.mu.Unlock()
|
||||
if clear {
|
||||
h.cleared(key)
|
||||
return map[string]any{"key": key, "cleared": true}, nil
|
||||
}
|
||||
switch severity {
|
||||
case Urgent, Warning:
|
||||
case "":
|
||||
return nil, errorf("severity is required: urgent or warning")
|
||||
default:
|
||||
return nil, errorf("severity %q is neither urgent nor warning", severity)
|
||||
}
|
||||
if summary == "" {
|
||||
return nil, errorf("summary is required: one line in the mesh's words")
|
||||
}
|
||||
h.raised(Record{Key: key, Kind: "notice", Subject: subject, Severity: severity, Summary: summary,
|
||||
Origin: "notify", More: "operator-channel.open"})
|
||||
h.mu.Lock()
|
||||
r := h.open[key]
|
||||
var sent []string
|
||||
refused, pending := "", ""
|
||||
if r != nil {
|
||||
for ch := range r.Sent {
|
||||
sent = append(sent, ch)
|
||||
}
|
||||
refused, pending = r.Refused, r.Pending
|
||||
}
|
||||
h.mu.Unlock()
|
||||
sort.Strings(sent)
|
||||
return map[string]any{"key": key, "sent_on": sent, "refused": refused, "unsent": pending}, nil
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package main
|
||||
|
||||
// The Telegram channel: a bot to the operator's chat (novox/hq to-be 45 §5, research 028/02). Sending
|
||||
// needs only outbound HTTPS. The bot token is this module's own secret, accepted from the operator; the
|
||||
// chat id is a setting. Neither is ever said: an error from the HTTP client carries the URL, and the
|
||||
// URL carries the token, so every error is rebuilt here from its kind before it leaves this file.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TelegramAPI is where the bot API answers; a test points it elsewhere.
|
||||
var TelegramAPI = "https://api.telegram.org"
|
||||
|
||||
var (
|
||||
tokenShape = regexp.MustCompile(`^\d{5,}:[A-Za-z0-9_-]{30,}$`)
|
||||
chatIDShape = regexp.MustCompile(`^(-?\d{1,20}|@[A-Za-z][A-Za-z0-9_]{4,})$`)
|
||||
)
|
||||
|
||||
// TelegramConfig is where the token is and what the chat is; read each time it is used, so a secret
|
||||
// accepted or a setting changed takes effect at the next message without a restart.
|
||||
type TelegramConfig struct {
|
||||
TokenFile string
|
||||
ChatID func() string
|
||||
}
|
||||
|
||||
// Telegram sends to one chat.
|
||||
type Telegram struct {
|
||||
Config TelegramConfig
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func NewTelegram(cfg TelegramConfig) *Telegram {
|
||||
return &Telegram{Config: cfg, Client: &http.Client{Timeout: 20 * time.Second}}
|
||||
}
|
||||
|
||||
func (t *Telegram) Name() string { return "telegram" }
|
||||
|
||||
// Ready says whether the channel can send, in words.
|
||||
func (t *Telegram) Ready() error {
|
||||
_, _, err := t.ready()
|
||||
return err
|
||||
}
|
||||
|
||||
// ready says whether the channel can send, and when not, what the operator must give. The words name
|
||||
// the setting and the secret, never a value.
|
||||
func (t *Telegram) ready() (string, string, error) {
|
||||
token, err := t.token()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
chat := strings.TrimSpace(t.Config.ChatID())
|
||||
if chat == "" {
|
||||
return "", "", errors.New("no chat to send to: the setting telegram-chat-id is not given " +
|
||||
"(`settings` for messenger with {\"telegram-chat-id\": \"<the chat's id>\"})")
|
||||
}
|
||||
if !chatIDShape.MatchString(chat) {
|
||||
return "", "", errors.New("the setting telegram-chat-id is not a chat id (a number, or @name of a channel)")
|
||||
}
|
||||
return token, chat, nil
|
||||
}
|
||||
|
||||
func (t *Telegram) token() (string, error) {
|
||||
if t.Config.TokenFile == "" {
|
||||
return "", errors.New("no bot token: this module was started without a token file")
|
||||
}
|
||||
raw, err := os.ReadFile(t.Config.TokenFile)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", errors.New("no bot token: the own secret telegram-token is not on this machine yet " +
|
||||
"(`secret accept <machine> messenger telegram-token`, then push the machine)")
|
||||
}
|
||||
if err != nil {
|
||||
return "", errors.New("the own secret telegram-token cannot be read: " + plainError(err))
|
||||
}
|
||||
token := strings.TrimSpace(string(raw))
|
||||
if token == "" {
|
||||
return "", errors.New("no bot token: the own secret telegram-token is empty")
|
||||
}
|
||||
if !tokenShape.MatchString(token) {
|
||||
// The mesh mints a random value for an own secret nobody accepted; that is not a bot token.
|
||||
return "", errors.New("the own secret telegram-token is not a bot token (digits, a colon, then the key " +
|
||||
"BotFather gave) — most likely the mesh made it because none was accepted: " +
|
||||
"`secret accept <machine> messenger telegram-token`, then push the machine")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// Send posts a message and answers its message id.
|
||||
func (t *Telegram) Send(m Message) (string, error) {
|
||||
text := m.Text()
|
||||
token, chat, err := t.ready()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var out struct {
|
||||
MessageID int64 `json:"message_id"`
|
||||
}
|
||||
if err := t.call(token, "sendMessage", map[string]any{
|
||||
"chat_id": chat, "text": text, "disable_web_page_preview": true,
|
||||
}, &out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprint(out.MessageID), nil
|
||||
}
|
||||
|
||||
// Edit replaces the text of a message sent before: how a cleared condition is said (to-be 45 §5).
|
||||
func (t *Telegram) Edit(id string, m Message) error {
|
||||
text := m.Text()
|
||||
token, chat, err := t.ready()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return t.call(token, "editMessageText", map[string]any{
|
||||
"chat_id": chat, "message_id": json.Number(id), "text": text, "disable_web_page_preview": true,
|
||||
}, nil)
|
||||
}
|
||||
|
||||
// CanEdit: Telegram edits a message in place.
|
||||
func (t *Telegram) CanEdit() bool { return true }
|
||||
|
||||
// Who asks the bot API who it is: the check that the token works, with no message sent.
|
||||
func (t *Telegram) Who() (string, error) {
|
||||
token, _, err := t.ready()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var me struct {
|
||||
Username string `json:"username"`
|
||||
}
|
||||
if err := t.call(token, "getMe", map[string]any{}, &me); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return me.Username, nil
|
||||
}
|
||||
|
||||
func (t *Telegram) call(token, method string, body map[string]any, into any) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
req, err := http.NewRequest(http.MethodPost, TelegramAPI+"/bot"+token+"/"+method, bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return errors.New("telegram " + method + ": the request could not be made")
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("telegram %s: %s", method, plainError(err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var answer struct {
|
||||
OK bool `json:"ok"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
Description string `json:"description"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&answer); err != nil {
|
||||
return fmt.Errorf("telegram %s: HTTP %d with an answer that is not the bot API's", method, resp.StatusCode)
|
||||
}
|
||||
if !answer.OK {
|
||||
d := strings.ReplaceAll(answer.Description, token, "<token>")
|
||||
return fmt.Errorf("telegram %s refused: %d %s", method, answer.ErrorCode, d)
|
||||
}
|
||||
if into != nil && len(answer.Result) > 0 {
|
||||
_ = json.Unmarshal(answer.Result, into)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// plainError is an error in words, without the URL a transport error carries.
|
||||
func plainError(err error) string {
|
||||
var ue *url.Error
|
||||
if errors.As(err, &ue) {
|
||||
err = ue.Err
|
||||
}
|
||||
var ne net.Error
|
||||
switch {
|
||||
case errors.As(err, &ne) && ne.Timeout():
|
||||
return "no answer in time"
|
||||
case errors.Is(err, os.ErrPermission):
|
||||
return "permission denied"
|
||||
}
|
||||
var dns *net.DNSError
|
||||
if errors.As(err, &dns) {
|
||||
return "the bot API's name does not resolve"
|
||||
}
|
||||
var op *net.OpError
|
||||
if errors.As(err, &op) {
|
||||
return "cannot connect (" + op.Op + ")"
|
||||
}
|
||||
s := err.Error()
|
||||
if strings.Contains(s, "/bot") || strings.Contains(s, "://") {
|
||||
return "the request failed"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module messenger
|
||||
|
||||
go 1.22
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.7
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
Executable
BIN
Binary file not shown.
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"module": "messenger",
|
||||
"version": "1",
|
||||
"slug": "msgr",
|
||||
"seats": [
|
||||
{
|
||||
"name": "operator-channel",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"open",
|
||||
"history",
|
||||
"notify"
|
||||
]
|
||||
}
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "operator-channel",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"open",
|
||||
"history",
|
||||
"notify"
|
||||
]
|
||||
}
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-controller.condition-raised",
|
||||
"mesh-controller.condition-changed",
|
||||
"mesh-controller.condition-cleared"
|
||||
],
|
||||
"emits": [
|
||||
"refused"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:node-notifier.send"
|
||||
],
|
||||
"state": [
|
||||
"open",
|
||||
"sent"
|
||||
],
|
||||
"own-secrets": {
|
||||
"telegram-token": "${dir:state}/telegram-token"
|
||||
},
|
||||
"tools": [
|
||||
"messenger_status",
|
||||
"messenger_recent",
|
||||
"messenger_test",
|
||||
"messenger_check"
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": "."
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"merge": "json",
|
||||
"content": "{\n \"telegram-chat-id\": \"\",\n \"desktop-machines\": []\n}\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/messenger",
|
||||
"binary": "messenger",
|
||||
"loads": [
|
||||
"messenger"
|
||||
],
|
||||
"env": {
|
||||
"MESH_MESSENGER_SETTINGS": "${dir:state}/settings.json",
|
||||
"MESH_MESSENGER_TELEGRAM_TOKEN_FILE": "${dir:state}/telegram-token"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user