348 lines
17 KiB
Go
348 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
_ "time/tzdata"
|
|
)
|
|
|
|
// The notifications the operator could not read on 2026-10-08, said again from the events a controller
|
|
// with plain words sends (novox/hq ADR 0253). BEFORE is the popup as it appeared; AFTER is what the desktop
|
|
// shows now: a title in plain words, an explanation, since when in the operator's time — no key, id,
|
|
// commit, verb or markup — and a clearance in one line.
|
|
|
|
type example struct {
|
|
name, before string
|
|
key, kind, severity, summary string
|
|
subject map[string]string
|
|
headline, explanation string
|
|
needs, resolved string
|
|
actions []Action
|
|
popup bool // shown at all: a warning that needs nothing is not
|
|
title, body string // what the desktop shows
|
|
offers string // the answers it offers, by label
|
|
choose string // the answer chosen in the test
|
|
asked string // the seat verb that answer calls, and its arguments
|
|
cleared string // the clearance's one line
|
|
openFor time.Duration
|
|
}
|
|
|
|
const waitingSummary = "the walk of novox/mesh-catalog a6385479 has waited 56m0s for mesh-delivery's word to start: " +
|
|
"`mesh-delivery.show` for the delivery that landed as a6385479 says why; `plans go " +
|
|
"plan-1791454185265004861 --why …` starts it by hand"
|
|
|
|
const waitingBefore = "CLEARED after 26 min: the walk of novox/mesh-catalog a6385479 has waited 56m0s for " +
|
|
"mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why; `plans go " +
|
|
"plan-1791454185265004861 --why ...` starts it by hand\nabout: plan plan-1791454185265004861 (waiting)\n" +
|
|
"since: 2026-10-08 10:40 UTC\nkey: plan.plan-1791454185265004861.waiting\nmore: conditions show " +
|
|
"plan.plan-1791454... 65004861.waiting"
|
|
|
|
var examples = []example{{
|
|
name: "a delivery waiting, under its bound", before: waitingBefore,
|
|
key: "plan.plan-1791454185265004861.waiting", kind: "waiting", severity: Warning,
|
|
subject: map[string]string{"scope": "plan", "id": "plan-1791454185265004861"}, summary: waitingSummary,
|
|
headline: "openrazer delivery waiting to start",
|
|
explanation: "Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for " +
|
|
"mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question for " +
|
|
"you if it still waits after 4 hours.",
|
|
resolved: "openrazer delivery no longer waiting", popup: false, openFor: 26 * time.Minute,
|
|
}, {
|
|
name: "a delivery waiting, past its bound", before: waitingBefore,
|
|
key: "plan.plan-1791454185265004861.waiting", kind: "waiting", severity: Urgent,
|
|
subject: map[string]string{"scope": "plan", "id": "plan-1791454185265004861"}, summary: waitingSummary,
|
|
headline: "openrazer delivery waiting to start", needs: "start it, or stop it.",
|
|
explanation: "Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery " +
|
|
"(the module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may be stuck.",
|
|
resolved: "openrazer delivery no longer waiting", popup: true,
|
|
title: "Urgent: openrazer delivery waiting to start",
|
|
body: "Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the module " +
|
|
"that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may be stuck.\nSince 12:40.",
|
|
offers: "Details", choose: "Details",
|
|
|
|
cleared: "Resolved: openrazer delivery no longer waiting, after 5.0 h", openFor: 5 * time.Hour,
|
|
}, {
|
|
name: "a module unhealthy",
|
|
before: "WARNING: openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the account's own " +
|
|
"service manager (exit-code)\nabout: module openrazer.g14 on g14 (module-unhealthy)\nsince: 2026-10-08 " +
|
|
"10:40 UTC\nkey: module.openrazer.g14.unhealthy\nmore: conditions show module.openrazer.g14.unhealthy",
|
|
key: "module.openrazer.g14.unhealthy", kind: "module-unhealthy", severity: Warning,
|
|
subject: map[string]string{"scope": "module", "id": "openrazer.g14", "machine": "g14"},
|
|
summary: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the account's own " +
|
|
"service manager (exit-code)",
|
|
headline: "openrazer not working on g14",
|
|
needs: "restart its service openrazer-daemon on g14; if it fails again, the details say why.",
|
|
explanation: "Needs you: restart its service openrazer-daemon on g14; if it fails again, the details say why. " +
|
|
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon as " +
|
|
"it runs again.",
|
|
resolved: "openrazer works again on g14", popup: true,
|
|
title: "openrazer not working on g14",
|
|
body: "Needs you: restart its service openrazer-daemon on g14; if it fails again, the details say why. openrazer " +
|
|
"on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon as it runs " +
|
|
"again.\nSince 12:40.",
|
|
offers: "Details", choose: "Details",
|
|
cleared: "Resolved: openrazer works again on g14, after 2.0 h", openFor: 2 * time.Hour,
|
|
}, {
|
|
name: "failed units on a machine",
|
|
before: "WARNING: shanks's service manager is degraded: 3 failed unit(s) no module places — mnt-recalbox.mount, " +
|
|
"storage-media.mount, greenclip.service (the account's own manager). Each is the machine's own: mend or " +
|
|
"remove it there, or have a module place it\nabout: machine shanks on shanks (machine-units)\nsince: " +
|
|
"2026-10-08 10:40 UTC\nkey: machine.shanks.units\nmore: conditions show machine.shanks.units",
|
|
key: "machine.shanks.units", kind: "machine-units", severity: Warning,
|
|
subject: map[string]string{"scope": "machine", "id": "shanks", "machine": "shanks"},
|
|
summary: "shanks's service manager is degraded: 3 failed unit(s) no module places — mnt-recalbox.mount, " +
|
|
"storage-media.mount, greenclip.service (the account's own manager). Each is the machine's own: mend or " +
|
|
"remove it there, or have a module place it",
|
|
headline: "3 failed services on shanks", needs: "mend or remove them on shanks, or silence this if they do not matter.",
|
|
explanation: "Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, " +
|
|
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the mesh " +
|
|
"does not repair them.",
|
|
actions: []Action{{Label: "Silence for a week", Verb: "mesh-controller.conditions",
|
|
Arguments: map[string]string{"silence": "machine.shanks.units", "for": "7d", "why": "", "cause": "operator-answer"}}},
|
|
resolved: "No failed services on shanks any more", popup: true,
|
|
title: "3 failed services on shanks",
|
|
body: "Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, mnt-recalbox " +
|
|
"(a mount), storage-media (a mount) and greenclip failed. No module manages them, so the mesh does not repair " +
|
|
"them.\nSince 12:40.",
|
|
offers: "Details", choose: "Details",
|
|
cleared: "Resolved: no failed services on shanks any more, after 45 min", openFor: 45 * time.Minute,
|
|
}, {
|
|
name: "a healer wanted",
|
|
before: "WARNING: \"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through the " +
|
|
"mesh-controller seat: a healer is wanted for it\nabout: mesh hand-acts.push (healer-wanted)\nsince: " +
|
|
"2026-10-08 10:40 UTC\nkey: mesh.hand-acts.push.healer-wanted\nmore: conditions show " +
|
|
"mesh.hand-acts.push.healer-wanted",
|
|
key: "mesh.hand-acts.push.healer-wanted", kind: "healer-wanted", severity: Warning,
|
|
subject: map[string]string{"scope": "mesh", "id": "hand-acts.push"},
|
|
summary: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through the " +
|
|
"mesh-controller seat: a healer is wanted for it",
|
|
headline: "Push keeps being fixed by hand",
|
|
explanation: "Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair " +
|
|
"is wanted for it. Nothing is broken now.",
|
|
resolved: "Resolved: no more hand repairs of push", popup: false, openFor: 3 * 24 * time.Hour,
|
|
}, {
|
|
name: "a delivery held",
|
|
before: "WARNING: the delivery novox/hq@055550802096 has been held for 36h2m6s, past its bound of 24h0m0s (it " +
|
|
"waits for the operator): healer H2 may none: the state is the operator's — `mesh-delivery.show " +
|
|
"novox/hq@055550802096`\nabout: delivery novox/hq@055550802096 (stalled)\nsince: 2026-10-07 00:38 UTC\n" +
|
|
"key: delivery.novox/hq_055550802096.stalled\nmore: conditions show delivery.novox/hq_055550802096.stalled",
|
|
key: "delivery.novox/hq_055550802096.stalled", kind: "stalled", severity: Warning,
|
|
subject: map[string]string{"scope": "delivery", "id": "novox/hq@055550802096"},
|
|
summary: "the delivery novox/hq@055550802096 has been held for 36h2m6s, past its bound of 24h0m0s (it waits for " +
|
|
"the operator): healer H2 may none: the state is the operator's — `mesh-delivery.show novox/hq@055550802096`",
|
|
headline: "Delivery of hq held for 36 hours", needs: "release it, or stop it.",
|
|
explanation: "Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
|
|
resolved: "Delivery of hq is no longer held", popup: true,
|
|
title: "Delivery of hq held for 36 hours",
|
|
body: "Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.\nSince 12:40.",
|
|
offers: "Details", choose: "Details",
|
|
|
|
cleared: "Resolved: delivery of hq is no longer held, after 3 min", openFor: 3 * time.Minute,
|
|
}}
|
|
|
|
// event is the body a controller with plain words sends for a condition.
|
|
func (e example) event(at time.Time, cleared bool) []byte {
|
|
body := map[string]any{"key": e.key, "kind": e.kind, "subject": e.subject, "severity": e.severity,
|
|
"summary": e.summary, "headline": e.headline, "explanation": e.explanation, "resolved": e.resolved,
|
|
"needs": e.needs, "actions": e.actions,
|
|
"raised": "2026-10-08T10:40:00Z", "at": at.Format(time.RFC3339), "silenced": nil}
|
|
if cleared {
|
|
body["cleared"] = at.Format(time.RFC3339)
|
|
}
|
|
raw, _ := json.Marshal(body)
|
|
return raw
|
|
}
|
|
|
|
func TestRealNotificationsReadInOneGlanceAndAreAnsweredFromTheNotification(t *testing.T) {
|
|
brussels, err := time.LoadLocation("Europe/Brussels")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, e := range examples {
|
|
t.Run(e.name, func(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
h.Zone = func() *time.Location { return brussels }
|
|
var asked []string
|
|
h.Ask = func(address string, body any) (json.RawMessage, error) {
|
|
raw, _ := json.Marshal(body)
|
|
asked = append(asked, address+" "+string(raw))
|
|
return json.RawMessage(`{"ok":true}`), nil
|
|
}
|
|
c.t = time.Date(2026, 10, 8, 10, 40, 0, 0, time.UTC)
|
|
raised, err := DecodeCondition(EventRaised, e.event(c.now(), false))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.Condition(EventRaised, raised)
|
|
settle(h, c)
|
|
if !e.popup {
|
|
if len(dt.sends)+len(tg.sends) != 0 {
|
|
t.Fatalf("a warning that needs nothing was shown: %+v %+v", dt.sends, tg.sends)
|
|
}
|
|
hist := h.History(5)["sends"].([]Sent)
|
|
if len(hist) == 0 || !strings.HasPrefix(hist[0].Outcome, "quiet") {
|
|
t.Errorf("not kept as quiet in the history: %+v", hist)
|
|
}
|
|
t.Logf("\nBEFORE:\n%s\n\nAFTER: no popup. Kept in the controller's conditions and the channel's history:\n%s\n%s",
|
|
e.before, e.headline, e.explanation)
|
|
return
|
|
}
|
|
if len(dt.sends) != 1 {
|
|
t.Fatalf("desktop sends %d", len(dt.sends))
|
|
}
|
|
m := dt.sends[0]
|
|
keyless(t, m)
|
|
var labels []string
|
|
for _, a := range m.Actions {
|
|
labels = append(labels, a.Label)
|
|
}
|
|
if m.Title != e.title || m.Body != e.body || strings.Join(labels, ", ") != e.offers {
|
|
t.Errorf("shows\n %q\n %q\n %v\nwant\n %q\n %q\n %s", m.Title, m.Body, labels, e.title, e.body, e.offers)
|
|
}
|
|
for _, w := range []string{"`", "UTC", "key:", "more:", "about:", "plan-", "a6385479", "agent", "by hand"} {
|
|
if strings.Contains(m.Text(), w) {
|
|
t.Errorf("it still says %q: %q", w, m.Text())
|
|
}
|
|
}
|
|
// The operator answers from the notification.
|
|
var tok string
|
|
for _, a := range m.Actions {
|
|
if a.Label == e.choose {
|
|
tok = a.Token
|
|
}
|
|
}
|
|
// The router performs nothing: Details is shown where it was chosen, and the condition's own actions
|
|
// are never called from here (novox/hq ADR 0259) — the controller asks for them.
|
|
outcome, err := h.Chosen("desktop", choiceOf(tok, "g14", c.now()))
|
|
done := dt.sends[len(dt.sends)-1]
|
|
if err != nil || outcome != "shown" || len(asked) != 0 || done.Title != "Details: "+e.headline {
|
|
t.Fatalf("details: %q %v %v %q", outcome, err, asked, done.Title)
|
|
}
|
|
c.pass(e.openFor - BurstWindow)
|
|
cleared, err := DecodeCondition(EventCleared, e.event(c.now(), true))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.Condition(EventCleared, cleared)
|
|
end := dt.edits["g14=1"]
|
|
if end.Title != e.cleared || end.Body != "" || len(end.Actions) != 0 {
|
|
t.Errorf("clearance %q / %q / %v, want one line %q", end.Title, end.Body, end.Actions, e.cleared)
|
|
}
|
|
t.Logf("\nBEFORE:\n%s\n\nAFTER:\n%s\n%s\n[%s]\n\nCHOSEN %s:\n%s\n%s\n\nCLEARED:\n%s", e.before, m.Title,
|
|
m.Body, strings.Join(labels, "] ["), e.choose, done.Title, done.Body, end.Title)
|
|
})
|
|
}
|
|
}
|
|
|
|
// **A warning kept quiet is said once it needs the operator**: at its bound it becomes urgent, and then
|
|
// its words say "Needs you:".
|
|
func TestAQuietWarningIsSaidWhenItBecomesUrgent(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
e := examples[0]
|
|
raised, _ := DecodeCondition(EventRaised, e.event(c.now(), false))
|
|
h.Condition(EventRaised, raised)
|
|
settle(h, c)
|
|
if len(dt.sends)+len(tg.sends) != 0 {
|
|
t.Fatal("said while it needed nothing")
|
|
}
|
|
past := examples[1]
|
|
c.pass(4 * time.Hour)
|
|
changed, _ := DecodeCondition(EventChanged, past.event(c.now(), false))
|
|
h.Condition(EventChanged, changed)
|
|
settle(h, c)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 || !strings.HasPrefix(dt.sends[0].Body, "Needs you: start it, or stop it.") {
|
|
t.Fatalf("desktop %+v telegram %d", dt.sends, len(tg.sends))
|
|
}
|
|
}
|
|
|
|
// **Details shows the summary and the key, where it was asked**, and an answer for a message that ended
|
|
// is refused in words.
|
|
func TestDetailsAreShownOnTheMachineThatAsked(t *testing.T) {
|
|
h, _, dt, c, _ := newHolder(t)
|
|
h.Zone = func() *time.Location { return time.UTC }
|
|
e := examples[5]
|
|
raised, _ := DecodeCondition(EventRaised, e.event(c.now(), false))
|
|
h.Condition(EventRaised, raised)
|
|
settle(h, c)
|
|
m := dt.sends[0]
|
|
details := m.Actions[len(m.Actions)-1]
|
|
if details.Label != DetailsLabel {
|
|
t.Fatalf("%+v", m.Actions)
|
|
}
|
|
if _, err := h.Chosen("desktop", choiceOf(details.Token, "g14", c.now())); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
shown := dt.sends[len(dt.sends)-1]
|
|
if shown.Title != "Details: "+e.headline || !strings.HasPrefix(shown.Body, e.summary) ||
|
|
!strings.Contains(shown.Body, "Since: 8 Oct 10:40") {
|
|
t.Errorf("details: %q / %q", shown.Title, shown.Body)
|
|
}
|
|
if _, err := h.Chosen("desktop", choiceOf("not-a-token", "g14", c.now())); err == nil {
|
|
t.Error("an unknown answer was taken")
|
|
}
|
|
}
|
|
|
|
// **A condition from a controller without plain words is said from its summary**, its markup taken out.
|
|
func TestAConditionWithoutPlainWordsIsSaidFromItsSummary(t *testing.T) {
|
|
h, tg, _, c, _ := newHolder(t)
|
|
h.Zone = func() *time.Location { return time.UTC }
|
|
k := Condition{Key: "machine.ace.silent", Scope: "machine", ID: "ace", Kind: "silent", Severity: Urgent,
|
|
Summary: "ace has not been heard from: `push ace` sends it again", Raised: c.now()}
|
|
h.Condition(EventRaised, k)
|
|
m := tg.sends[0]
|
|
if m.Title != "Urgent: Silent on ace" || m.Body != "Ace has not been heard from: a command sends it again\nSince 12:00." {
|
|
t.Errorf("%q / %q", m.Title, m.Body)
|
|
}
|
|
keyless(t, m)
|
|
}
|
|
|
|
// **Times are the operator's**: today a clock time, this week a weekday, before that a date.
|
|
func TestTimesAreSaidInTheOperatorsZone(t *testing.T) {
|
|
brussels, _ := time.LoadLocation("Europe/Brussels")
|
|
now := time.Date(2026, 10, 8, 10, 40, 0, 0, time.UTC)
|
|
for at, want := range map[time.Time]string{
|
|
now.Add(-56 * time.Minute): "11:44",
|
|
now.Add(-26 * time.Hour): "Wed 10:40",
|
|
now.Add(-10 * 24 * time.Hour): "28 Sep 12:40",
|
|
} {
|
|
if got := when(at, now, brussels); got != want {
|
|
t.Errorf("%s: %q, want %q", at, got, want)
|
|
}
|
|
}
|
|
if loc, err := zoneOf("Nowhere/Atlantis"); err == nil || loc != time.Local {
|
|
t.Errorf("a zone that does not exist was not said: %v", err)
|
|
}
|
|
}
|
|
|
|
// A warning kept quiet is said once after a day, and not again (novox/hq ADR 0258 §6).
|
|
func TestAQuietWarningIsSaidOnceAfterADay(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
c.t = time.Date(2026, 10, 8, 10, 40, 0, 0, time.UTC)
|
|
e := examples[0]
|
|
raised, _ := DecodeCondition(EventRaised, e.event(c.now(), false))
|
|
h.Condition(EventRaised, raised)
|
|
settle(h, c)
|
|
if len(dt.sends)+len(tg.sends) != 0 {
|
|
t.Fatal("said while it needed nothing")
|
|
}
|
|
c.pass(QuietRemindAfter - time.Hour)
|
|
h.Tick()
|
|
settle(h, c)
|
|
if len(dt.sends)+len(tg.sends) != 0 {
|
|
t.Fatal("said before a day")
|
|
}
|
|
c.pass(time.Hour)
|
|
h.Tick()
|
|
settle(h, c)
|
|
if len(dt.sends) != 1 || !strings.HasPrefix(dt.sends[0].Title, "Still open after a day: ") {
|
|
t.Fatalf("not said once after a day: %+v", dt.sends)
|
|
}
|
|
c.pass(QuietRemindAfter)
|
|
h.Tick()
|
|
settle(h, c)
|
|
if len(dt.sends) != 1 {
|
|
t.Errorf("said again: %d", len(dt.sends))
|
|
}
|
|
}
|