jochen
63dba87587
mesh/delivery-group group feat/400-alert-names-its-issue checking: 0 of 2 member(s) ready
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate pass: builds mesh-issues, messenger → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
The operator could not tell from a notification whether its cause was known, tracked and being fixed, or whether they had to act. The issue tracker gains the verb tracking, answering for every condition key an issue links which issues link it and where each stands; the operator channel's router asks it outside the work and says one line from it in every condition's message, per hq ADR 0285.
353 lines
17 KiB
Go
353 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
_ "time/tzdata"
|
|
)
|
|
|
|
// The notifications the operator could not read on 2026-10-08, said again from the events a controller
|
|
// with plain words sends (novox/hq ADR 0253). BEFORE is the popup as it appeared; AFTER is what the desktop
|
|
// shows now: a title in plain words, an explanation, since when in the operator's time — no key, id,
|
|
// commit, verb or markup — and a clearance in one line.
|
|
|
|
type example struct {
|
|
name, before string
|
|
key, kind, severity, summary string
|
|
subject map[string]string
|
|
headline, explanation string
|
|
needs, resolved string
|
|
actions []Action
|
|
popup bool // shown at all: a warning that needs nothing is not
|
|
title, body string // what the desktop shows
|
|
offers string // the answers it offers, by label
|
|
choose string // the answer chosen in the test
|
|
asked string // the seat verb that answer calls, and its arguments
|
|
cleared string // the clearance's one line
|
|
openFor time.Duration
|
|
}
|
|
|
|
const waitingSummary = "the walk of novox/mesh-catalog a6385479 has waited 56m0s for mesh-delivery's word to start: " +
|
|
"`mesh-delivery.show` for the delivery that landed as a6385479 says why; `plans go " +
|
|
"plan-1791454185265004861 --why …` starts it by hand"
|
|
|
|
const waitingBefore = "CLEARED after 26 min: the walk of novox/mesh-catalog a6385479 has waited 56m0s for " +
|
|
"mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why; `plans go " +
|
|
"plan-1791454185265004861 --why ...` starts it by hand\nabout: plan plan-1791454185265004861 (waiting)\n" +
|
|
"since: 2026-10-08 10:40 UTC\nkey: plan.plan-1791454185265004861.waiting\nmore: conditions show " +
|
|
"plan.plan-1791454... 65004861.waiting"
|
|
|
|
var examples = []example{{
|
|
name: "a delivery waiting, under its bound", before: waitingBefore,
|
|
key: "plan.plan-1791454185265004861.waiting", kind: "waiting", severity: Warning,
|
|
subject: map[string]string{"scope": "plan", "id": "plan-1791454185265004861"}, summary: waitingSummary,
|
|
headline: "openrazer delivery waiting to start",
|
|
explanation: "Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for " +
|
|
"mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question for " +
|
|
"you if it still waits after 4 hours.",
|
|
resolved: "openrazer delivery no longer waiting", popup: false, openFor: 26 * time.Minute,
|
|
}, {
|
|
name: "a delivery waiting, past its bound", before: waitingBefore,
|
|
key: "plan.plan-1791454185265004861.waiting", kind: "waiting", severity: Urgent,
|
|
subject: map[string]string{"scope": "plan", "id": "plan-1791454185265004861"}, summary: waitingSummary,
|
|
headline: "openrazer delivery waiting to start", needs: "start it, or stop it.",
|
|
explanation: "Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery " +
|
|
"(the module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may be stuck.",
|
|
resolved: "openrazer delivery no longer waiting", popup: true,
|
|
title: "Urgent: openrazer delivery waiting to start",
|
|
body: "Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the module " +
|
|
"that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may be stuck.\nNo issue tracks this yet.\nSince 12:40.",
|
|
offers: "Details", choose: "Details",
|
|
|
|
cleared: "Resolved: openrazer delivery no longer waiting, after 5.0 h", openFor: 5 * time.Hour,
|
|
}, {
|
|
name: "a module unhealthy",
|
|
before: "WARNING: openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the account's own " +
|
|
"service manager (exit-code)\nabout: module openrazer.g14 on g14 (module-unhealthy)\nsince: 2026-10-08 " +
|
|
"10:40 UTC\nkey: module.openrazer.g14.unhealthy\nmore: conditions show module.openrazer.g14.unhealthy",
|
|
key: "module.openrazer.g14.unhealthy", kind: "module-unhealthy", severity: Warning,
|
|
subject: map[string]string{"scope": "module", "id": "openrazer.g14", "machine": "g14"},
|
|
summary: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the account's own " +
|
|
"service manager (exit-code)",
|
|
headline: "openrazer not working on g14",
|
|
needs: "restart its service openrazer-daemon on g14; if it fails again, the details say why.",
|
|
explanation: "Needs you: restart its service openrazer-daemon on g14; if it fails again, the details say why. " +
|
|
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon as " +
|
|
"it runs again.",
|
|
resolved: "openrazer works again on g14", popup: true,
|
|
title: "openrazer not working on g14",
|
|
body: "Needs you: restart its service openrazer-daemon on g14; if it fails again, the details say why. openrazer " +
|
|
"on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon as it runs " +
|
|
"again.\nNo issue tracks this yet.\nSince 12:40.",
|
|
offers: "Details", choose: "Details",
|
|
cleared: "Resolved: openrazer works again on g14, after 2.0 h", openFor: 2 * time.Hour,
|
|
}, {
|
|
name: "failed units on a machine",
|
|
before: "WARNING: shanks's service manager is degraded: 3 failed unit(s) no module places — mnt-recalbox.mount, " +
|
|
"storage-media.mount, greenclip.service (the account's own manager). Each is the machine's own: mend or " +
|
|
"remove it there, or have a module place it\nabout: machine shanks on shanks (machine-units)\nsince: " +
|
|
"2026-10-08 10:40 UTC\nkey: machine.shanks.units\nmore: conditions show machine.shanks.units",
|
|
key: "machine.shanks.units", kind: "machine-units", severity: Warning,
|
|
subject: map[string]string{"scope": "machine", "id": "shanks", "machine": "shanks"},
|
|
summary: "shanks's service manager is degraded: 3 failed unit(s) no module places — mnt-recalbox.mount, " +
|
|
"storage-media.mount, greenclip.service (the account's own manager). Each is the machine's own: mend or " +
|
|
"remove it there, or have a module place it",
|
|
headline: "3 failed services on shanks", needs: "mend or remove them on shanks, or silence this if they do not matter.",
|
|
explanation: "Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, " +
|
|
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the mesh " +
|
|
"does not repair them.",
|
|
actions: []Action{{Label: "Silence for a week", Verb: "mesh-controller.conditions",
|
|
Arguments: map[string]string{"silence": "machine.shanks.units", "for": "7d", "why": "", "cause": "operator-answer"}}},
|
|
resolved: "No failed services on shanks any more", popup: true,
|
|
title: "3 failed services on shanks",
|
|
body: "Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, mnt-recalbox " +
|
|
"(a mount), storage-media (a mount) and greenclip failed. No module manages them, so the mesh does not repair " +
|
|
"them.\nNo issue tracks this yet.\nSince 12:40.",
|
|
offers: "Details", choose: "Details",
|
|
cleared: "Resolved: no failed services on shanks any more, after 45 min", openFor: 45 * time.Minute,
|
|
}, {
|
|
name: "a healer wanted",
|
|
before: "WARNING: \"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through the " +
|
|
"mesh-controller seat: a healer is wanted for it\nabout: mesh hand-acts.push (healer-wanted)\nsince: " +
|
|
"2026-10-08 10:40 UTC\nkey: mesh.hand-acts.push.healer-wanted\nmore: conditions show " +
|
|
"mesh.hand-acts.push.healer-wanted",
|
|
key: "mesh.hand-acts.push.healer-wanted", kind: "healer-wanted", severity: Warning,
|
|
subject: map[string]string{"scope": "mesh", "id": "hand-acts.push"},
|
|
summary: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through the " +
|
|
"mesh-controller seat: a healer is wanted for it",
|
|
headline: "Push keeps being fixed by hand",
|
|
explanation: "Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair " +
|
|
"is wanted for it. Nothing is broken now.",
|
|
resolved: "Resolved: no more hand repairs of push", popup: false, openFor: 3 * 24 * time.Hour,
|
|
}, {
|
|
name: "a delivery held",
|
|
before: "WARNING: the delivery novox/hq@055550802096 has been held for 36h2m6s, past its bound of 24h0m0s (it " +
|
|
"waits for the operator): healer H2 may none: the state is the operator's — `mesh-delivery.show " +
|
|
"novox/hq@055550802096`\nabout: delivery novox/hq@055550802096 (stalled)\nsince: 2026-10-07 00:38 UTC\n" +
|
|
"key: delivery.novox/hq_055550802096.stalled\nmore: conditions show delivery.novox/hq_055550802096.stalled",
|
|
key: "delivery.novox/hq_055550802096.stalled", kind: "stalled", severity: Warning,
|
|
subject: map[string]string{"scope": "delivery", "id": "novox/hq@055550802096"},
|
|
summary: "the delivery novox/hq@055550802096 has been held for 36h2m6s, past its bound of 24h0m0s (it waits for " +
|
|
"the operator): healer H2 may none: the state is the operator's — `mesh-delivery.show novox/hq@055550802096`",
|
|
headline: "Delivery of hq held for 36 hours", needs: "release it, or stop it.",
|
|
explanation: "Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
|
|
resolved: "Delivery of hq is no longer held", popup: true,
|
|
title: "Delivery of hq held for 36 hours",
|
|
body: "Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.\nNo issue tracks this yet.\nSince 12:40.",
|
|
offers: "Details", choose: "Details",
|
|
|
|
cleared: "Resolved: delivery of hq is no longer held, after 3 min", openFor: 3 * time.Minute,
|
|
}}
|
|
|
|
// event is the body a controller with plain words sends for a condition.
|
|
func (e example) event(at time.Time, cleared bool) []byte {
|
|
body := map[string]any{"key": e.key, "kind": e.kind, "subject": e.subject, "severity": e.severity,
|
|
"summary": e.summary, "headline": e.headline, "explanation": e.explanation, "resolved": e.resolved,
|
|
"needs": e.needs, "actions": e.actions,
|
|
"raised": "2026-10-08T10:40:00Z", "at": at.Format(time.RFC3339), "silenced": nil}
|
|
if cleared {
|
|
body["cleared"] = at.Format(time.RFC3339)
|
|
}
|
|
raw, _ := json.Marshal(body)
|
|
return raw
|
|
}
|
|
|
|
func TestRealNotificationsReadInOneGlanceAndAreAnsweredFromTheNotification(t *testing.T) {
|
|
brussels, err := time.LoadLocation("Europe/Brussels")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, e := range examples {
|
|
t.Run(e.name, func(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
h.Zone = func() *time.Location { return brussels }
|
|
var asked []string
|
|
h.Ask = func(address string, body any) (json.RawMessage, error) {
|
|
if address == TrackingVerb {
|
|
return json.RawMessage(`{"conditions":{}}`), nil // no issue tracks these
|
|
}
|
|
raw, _ := json.Marshal(body)
|
|
asked = append(asked, address+" "+string(raw))
|
|
return json.RawMessage(`{"ok":true}`), nil
|
|
}
|
|
c.t = time.Date(2026, 10, 8, 10, 40, 0, 0, time.UTC)
|
|
raised, err := DecodeCondition(EventRaised, e.event(c.now(), false))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.Condition(EventRaised, raised)
|
|
settle(h, c)
|
|
if !e.popup {
|
|
if len(dt.sends)+len(tg.sends) != 0 {
|
|
t.Fatalf("a warning that needs nothing was shown: %+v %+v", dt.sends, tg.sends)
|
|
}
|
|
hist := h.History(5)["sends"].([]Sent)
|
|
if len(hist) == 0 || !strings.HasPrefix(hist[0].Outcome, "quiet") {
|
|
t.Errorf("not kept as quiet in the history: %+v", hist)
|
|
}
|
|
t.Logf("\nBEFORE:\n%s\n\nAFTER: no popup. Kept in the controller's conditions and the channel's history:\n%s\n%s",
|
|
e.before, e.headline, e.explanation)
|
|
return
|
|
}
|
|
if len(dt.sends) != 1 {
|
|
t.Fatalf("desktop sends %d", len(dt.sends))
|
|
}
|
|
m := dt.sends[0]
|
|
keyless(t, m)
|
|
var labels []string
|
|
for _, a := range m.Actions {
|
|
labels = append(labels, a.Label)
|
|
}
|
|
if m.Title != e.title || m.Body != e.body || strings.Join(labels, ", ") != e.offers {
|
|
t.Errorf("shows\n %q\n %q\n %v\nwant\n %q\n %q\n %s", m.Title, m.Body, labels, e.title, e.body, e.offers)
|
|
}
|
|
for _, w := range []string{"`", "UTC", "key:", "more:", "about:", "plan-", "a6385479", "agent", "by hand"} {
|
|
if strings.Contains(m.Text(), w) {
|
|
t.Errorf("it still says %q: %q", w, m.Text())
|
|
}
|
|
}
|
|
// The operator answers from the notification.
|
|
var tok string
|
|
for _, a := range m.Actions {
|
|
if a.Label == e.choose {
|
|
tok = a.Token
|
|
}
|
|
}
|
|
// The router performs nothing: Details is shown where it was chosen, and the condition's own actions
|
|
// are never called from here (novox/hq ADR 0259) — the controller asks for them.
|
|
outcome, err := h.Chosen("desktop", choiceOf(tok, "g14", c.now()))
|
|
done := dt.sends[len(dt.sends)-1]
|
|
if err != nil || outcome != "shown" || len(asked) != 0 || done.Title != "Details: "+e.headline {
|
|
t.Fatalf("details: %q %v %v %q", outcome, err, asked, done.Title)
|
|
}
|
|
c.pass(e.openFor - BurstWindow)
|
|
cleared, err := DecodeCondition(EventCleared, e.event(c.now(), true))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.Condition(EventCleared, cleared)
|
|
c.pass(ClearSettle)
|
|
h.Tick()
|
|
end := dt.edits["g14=1"]
|
|
if end.Title != e.cleared || end.Body != "" || len(end.Actions) != 0 {
|
|
t.Errorf("clearance %q / %q / %v, want one line %q", end.Title, end.Body, end.Actions, e.cleared)
|
|
}
|
|
t.Logf("\nBEFORE:\n%s\n\nAFTER:\n%s\n%s\n[%s]\n\nCHOSEN %s:\n%s\n%s\n\nCLEARED:\n%s", e.before, m.Title,
|
|
m.Body, strings.Join(labels, "] ["), e.choose, done.Title, done.Body, end.Title)
|
|
})
|
|
}
|
|
}
|
|
|
|
// **A warning kept quiet is said once it needs the operator**: at its bound it becomes urgent, and then
|
|
// its words say "Needs you:".
|
|
func TestAQuietWarningIsSaidWhenItBecomesUrgent(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
e := examples[0]
|
|
raised, _ := DecodeCondition(EventRaised, e.event(c.now(), false))
|
|
h.Condition(EventRaised, raised)
|
|
settle(h, c)
|
|
if len(dt.sends)+len(tg.sends) != 0 {
|
|
t.Fatal("said while it needed nothing")
|
|
}
|
|
past := examples[1]
|
|
c.pass(4 * time.Hour)
|
|
changed, _ := DecodeCondition(EventChanged, past.event(c.now(), false))
|
|
h.Condition(EventChanged, changed)
|
|
settle(h, c)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 || !strings.HasPrefix(dt.sends[0].Body, "Needs you: start it, or stop it.") {
|
|
t.Fatalf("desktop %+v telegram %d", dt.sends, len(tg.sends))
|
|
}
|
|
}
|
|
|
|
// **Details shows the summary and the key, where it was asked**, and an answer for a message that ended
|
|
// is refused in words.
|
|
func TestDetailsAreShownOnTheMachineThatAsked(t *testing.T) {
|
|
h, _, dt, c, _ := newHolder(t)
|
|
h.Zone = func() *time.Location { return time.UTC }
|
|
e := examples[5]
|
|
raised, _ := DecodeCondition(EventRaised, e.event(c.now(), false))
|
|
h.Condition(EventRaised, raised)
|
|
settle(h, c)
|
|
m := dt.sends[0]
|
|
details := m.Actions[len(m.Actions)-1]
|
|
if details.Label != DetailsLabel {
|
|
t.Fatalf("%+v", m.Actions)
|
|
}
|
|
if _, err := h.Chosen("desktop", choiceOf(details.Token, "g14", c.now())); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
shown := dt.sends[len(dt.sends)-1]
|
|
if shown.Title != "Details: "+e.headline || !strings.HasPrefix(shown.Body, e.summary) ||
|
|
!strings.Contains(shown.Body, "Since: 8 Oct 10:40") {
|
|
t.Errorf("details: %q / %q", shown.Title, shown.Body)
|
|
}
|
|
if _, err := h.Chosen("desktop", choiceOf("not-a-token", "g14", c.now())); err == nil {
|
|
t.Error("an unknown answer was taken")
|
|
}
|
|
}
|
|
|
|
// **A condition from a controller without plain words is said from its summary**, its markup taken out.
|
|
func TestAConditionWithoutPlainWordsIsSaidFromItsSummary(t *testing.T) {
|
|
h, tg, _, c, _ := newHolder(t)
|
|
h.Zone = func() *time.Location { return time.UTC }
|
|
k := Condition{Key: "machine.ace.silent", Scope: "machine", ID: "ace", Kind: "silent", Severity: Urgent,
|
|
Summary: "ace has not been heard from: `push ace` sends it again", Raised: c.now()}
|
|
h.Condition(EventRaised, k)
|
|
m := tg.sends[0]
|
|
if m.Title != "Urgent: Silent on ace" || m.Body != "Ace has not been heard from: a command sends it again\nSince 12:00." {
|
|
t.Errorf("%q / %q", m.Title, m.Body)
|
|
}
|
|
keyless(t, m)
|
|
}
|
|
|
|
// **Times are the operator's**: today a clock time, this week a weekday, before that a date.
|
|
func TestTimesAreSaidInTheOperatorsZone(t *testing.T) {
|
|
brussels, _ := time.LoadLocation("Europe/Brussels")
|
|
now := time.Date(2026, 10, 8, 10, 40, 0, 0, time.UTC)
|
|
for at, want := range map[time.Time]string{
|
|
now.Add(-56 * time.Minute): "11:44",
|
|
now.Add(-26 * time.Hour): "Wed 10:40",
|
|
now.Add(-10 * 24 * time.Hour): "28 Sep 12:40",
|
|
} {
|
|
if got := when(at, now, brussels); got != want {
|
|
t.Errorf("%s: %q, want %q", at, got, want)
|
|
}
|
|
}
|
|
if loc, err := zoneOf("Nowhere/Atlantis"); err == nil || loc != time.Local {
|
|
t.Errorf("a zone that does not exist was not said: %v", err)
|
|
}
|
|
}
|
|
|
|
// A warning kept quiet is said once after a day, and not again (novox/hq ADR 0258 §6).
|
|
func TestAQuietWarningIsSaidOnceAfterADay(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
c.t = time.Date(2026, 10, 8, 10, 40, 0, 0, time.UTC)
|
|
e := examples[0]
|
|
raised, _ := DecodeCondition(EventRaised, e.event(c.now(), false))
|
|
h.Condition(EventRaised, raised)
|
|
settle(h, c)
|
|
if len(dt.sends)+len(tg.sends) != 0 {
|
|
t.Fatal("said while it needed nothing")
|
|
}
|
|
c.pass(QuietRemindAfter - time.Hour)
|
|
h.Tick()
|
|
settle(h, c)
|
|
if len(dt.sends)+len(tg.sends) != 0 {
|
|
t.Fatal("said before a day")
|
|
}
|
|
c.pass(time.Hour)
|
|
h.Tick()
|
|
settle(h, c)
|
|
if len(dt.sends) != 1 || !strings.HasPrefix(dt.sends[0].Title, "Still open after a day: ") {
|
|
t.Fatalf("not said once after a day: %+v", dt.sends)
|
|
}
|
|
c.pass(QuietRemindAfter)
|
|
h.Tick()
|
|
settle(h, c)
|
|
if len(dt.sends) != 1 {
|
|
t.Errorf("said again: %d", len(dt.sends))
|
|
}
|
|
}
|