350 lines
18 KiB
Go
350 lines
18 KiB
Go
package main
|
|
|
|
import (
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The conditions the operator read on 2026-10-08, raised again from the same facts (novox/hq ADR 0253):
|
|
// each keeps its summary for whoever looks closer, and now carries a headline, an explanation and a
|
|
// resolved line the operator reads at a glance — no plan id, commit, key, verb or clock time in them.
|
|
|
|
// plainExample checks one finding's words as the keeper will keep them: plain, within their bounds,
|
|
// opened by the verdict, and saying what the operator needs; and its actions by label.
|
|
func plainExample(t *testing.T, o conditions.Observation, headline, explanation string, actions ...string) {
|
|
t.Helper()
|
|
machines := append([]string{o.Machine}, o.Also...)
|
|
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs,
|
|
Actions: o.Actions}
|
|
if why, ok := conditions.PlainWords(w, machines...); !ok {
|
|
t.Errorf("%s: not plain: %s\n %+v", o.Key(), why, w)
|
|
}
|
|
said := conditions.Verdict(o.Needs, o.Explanation)
|
|
if o.Headline != headline || said != explanation {
|
|
t.Errorf("%s: says\n %q\n %q\nwant\n %q\n %q", o.Key(), o.Headline, said, headline, explanation)
|
|
}
|
|
var labels []string
|
|
for _, a := range o.Actions {
|
|
labels = append(labels, a.Label)
|
|
}
|
|
if strings.Join(labels, ",") != strings.Join(actions, ",") {
|
|
t.Errorf("%s: actions %v, want %v", o.Key(), labels, actions)
|
|
}
|
|
quiet := ""
|
|
if o.Needs == "" && o.Severity == conditions.Warning {
|
|
quiet = " (needs nothing and is a warning: no popup; it stays in conditions and the history)"
|
|
}
|
|
t.Logf("\nBEFORE: %s\nAFTER: %s%s\n %s\n actions %v\nCLEARS: %s", o.Summary, o.Headline, quiet,
|
|
said, labels, o.Resolved)
|
|
}
|
|
|
|
// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for
|
|
// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why;
|
|
// `plans go plan-1791454185265004861 --why …` starts it by hand". Under four hours it needs nothing — no
|
|
// popup; past them, the operator starts or stops it from the notification.
|
|
func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *testing.T) {
|
|
now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee",
|
|
awaits: "mesh-delivery", since: now.Add(-56 * time.Minute), modules: []string{"openrazer"}}}
|
|
got := watchWaits(f)
|
|
if len(got) != 1 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
plainExample(t, got[0], "openrazer delivery waiting to start",
|
|
"Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for "+
|
|
"mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question "+
|
|
"for you if it still waits after 4 hours.")
|
|
if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") {
|
|
t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary)
|
|
}
|
|
|
|
// Past four hours it is urgent, and asks the operator to start or stop it (novox/hq ADR 0259): the plan's
|
|
// own verbs, approved, which the controller performs on the warrant. The router says where to answer.
|
|
f.waits[0].since = now.Add(-5 * time.Hour)
|
|
got = watchWaits(f)
|
|
plainExample(t, got[0], "openrazer delivery waiting to start",
|
|
"Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+
|
|
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
|
|
"be stuck.", "Start", "Stop")
|
|
for i, want := range []string{"go", "stop"} {
|
|
a := got[0].Actions[i]
|
|
if a.Verb != "mesh-controller.plans" || a.Arguments[want] != "plan-1791454185265004861" ||
|
|
a.Level != conditions.LevelApprove || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
|
|
t.Errorf("%s: %+v", a.Label, a)
|
|
}
|
|
}
|
|
|
|
// Many modules are counted, not listed in the headline.
|
|
f.waits[0].modules = []string{"a", "b", "c", "d"}
|
|
got = watchWaits(f)
|
|
if got[0].Headline != "mesh-catalog delivery waiting to start" || !strings.Contains(got[0].Explanation, "4 modules of mesh-catalog") {
|
|
t.Errorf("%+v", got[0])
|
|
}
|
|
}
|
|
|
|
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
|
|
// account's own service manager (exit-code)". Restarting is not an acknowledgement: it is asked of the
|
|
// operator at the approve level (novox/hq ADR 0259), so a desk click never performs it (ADR 0258).
|
|
func TestAModuleUnhealthyAsksForARestartInWords(t *testing.T) {
|
|
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
|
|
Resource: "openrazer-daemon", Target: "openrazer-daemon.service", Account: "jochen",
|
|
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
|
|
plainExample(t, o, "openrazer not working on g14",
|
|
"Needs you: restart its service openrazer-daemon on g14. "+
|
|
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon "+
|
|
"as it runs again.", "Restart")
|
|
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" || a.Level != conditions.LevelApprove ||
|
|
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
|
|
t.Errorf("restart: %+v", a)
|
|
}
|
|
// An account waiting for a new login (ADR 0252) asks for the login, held to the plain rule.
|
|
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
|
|
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
|
|
if o.Needs != "log out of g14 completely and log in again, or restart it." || len(o.Actions) != 0 {
|
|
t.Errorf("relogin: %q %+v", o.Needs, o.Actions)
|
|
}
|
|
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
|
if why, ok := conditions.PlainWords(w, "g14"); !ok {
|
|
t.Errorf("the relogin words are not plain: %s", why)
|
|
}
|
|
// And the kind issue 318 raises for it (ADR 0254).
|
|
rw := plainWordings["relogin-needed"](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14",
|
|
Machine: "g14", Kind: "relogin-needed", Severity: conditions.Warning})
|
|
if why, ok := conditions.PlainWords(rw, "g14"); !ok || rw.Needs == "" || len(rw.Actions) != 0 {
|
|
t.Errorf("relogin-needed: %s %+v", why, rw)
|
|
}
|
|
if strings.Contains(o.Summary, "session") || !strings.Contains(o.Summary, "waits for a new login of jochen") {
|
|
t.Errorf("relogin summary: %q", o.Summary)
|
|
}
|
|
}
|
|
|
|
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
|
|
// mnt-recalbox.mount, storage-media.mount, greenclip.service (the account's own manager). …"
|
|
func TestFailedUnitsOnAMachineAreNamedWithoutTheirSuffixes(t *testing.T) {
|
|
o, raise := machineUnitsObservation("shanks", &link.UnitsHealth{State: "degraded", Failed: []link.FailedUnit{
|
|
{Unit: "mnt-recalbox.mount", Result: "exit-code"}, {Unit: "storage-media.mount", Result: "exit-code"},
|
|
{Unit: "greenclip.service", Scope: "user", Result: "exit-code"}}})
|
|
if !raise {
|
|
t.Fatal("not raised")
|
|
}
|
|
plainExample(t, o, "3 failed services on shanks",
|
|
"Needs you: mend or remove them on shanks, or silence this if they do not matter. On shanks, "+
|
|
"mnt-recalbox (a mount), storage-media (a mount) and greenclip failed. No module manages them, so the "+
|
|
"mesh does not repair them.", "Silence for a week")
|
|
}
|
|
|
|
// **A healer wanted**: "\"push\" was repaired by hand 35 times in 14 days, the last by g14.node-tools, through
|
|
// the mesh-controller seat: a healer is wanted for it". Nothing is broken: no popup.
|
|
func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) {
|
|
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
for i := 0; i < 35; i++ {
|
|
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), "push")
|
|
a.Verb, a.By = "push", "g14.node-tools, through the mesh-controller seat"
|
|
f.handActs = append(f.handActs, a)
|
|
}
|
|
got := watchHandActs(f)
|
|
if len(got) != 1 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
plainExample(t, got[0], "Push keeps being fixed by hand",
|
|
"Nothing for you to do. A person repaired push by hand 35 times in 14 days, so an automatic repair is "+
|
|
"wanted for it. Nothing is broken now.")
|
|
}
|
|
|
|
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
|
|
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
|
|
func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) {
|
|
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
|
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
|
|
plainExample(t, got[0], "Delivery of hq held for 36 hours",
|
|
"Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
|
|
"Release", "Stop")
|
|
for i, verb := range []string{"mesh-delivery.release", "mesh-delivery.stop"} {
|
|
if a := got[0].Actions[i]; a.Verb != verb || a.Arguments["id"] != "novox/hq@055550802096" || a.Level != conditions.LevelApprove {
|
|
t.Errorf("%+v", a)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
|
|
// shape it is raised with.
|
|
func TestEveryWordingIsPlain(t *testing.T) {
|
|
subjects := []conditions.Observation{
|
|
{Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"},
|
|
{Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"},
|
|
{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"},
|
|
{Scope: conditions.ScopeProvider, ID: "postgres.anchor.app_db", Machine: "anchor"},
|
|
{Scope: conditions.ScopePlan, ID: "plan-1791454185265004861"},
|
|
{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"},
|
|
{Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"},
|
|
{Scope: conditions.ScopeBus, ID: "mesh"},
|
|
{Scope: conditions.ScopeSeat, ID: "node-backup.ace", Machine: "ace"},
|
|
{Scope: conditions.ScopeMesh, ID: "hand-acts.disk-load"},
|
|
{Scope: conditions.ScopeBuild, ID: "openrazer.g14", Machine: "g14"},
|
|
}
|
|
for kind, fn := range plainWordings {
|
|
if !conditions.Worded(kind) {
|
|
t.Errorf("%s is not registered", kind)
|
|
}
|
|
for _, s := range subjects {
|
|
s.Kind = kind
|
|
w := fn(s)
|
|
w.Explanation = conditions.Verdict(w.Needs, w.Explanation)
|
|
if why, ok := conditions.PlainWords(w, s.Machine); !ok {
|
|
t.Errorf("%s about %s %s: %s\n %+v", kind, s.Scope, s.ID, why, w)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// **An answer on a notification is no repair** (novox/hq ADR 0258): silencing chosen by the operator on the
|
|
// desk does not count toward a healer wanted; the same silence by hand for another cause still does.
|
|
func TestAnOperatorsAnswerIsNoHandRepair(t *testing.T) {
|
|
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
for i := 0; i < 3; i++ {
|
|
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), conditions.CauseOperatorAnswer)
|
|
a.Verb = "conditions silence"
|
|
f.handActs = append(f.handActs, a)
|
|
}
|
|
if got := watchHandActs(f); len(got) != 0 {
|
|
t.Fatalf("an answer counted as a repair: %+v", got)
|
|
}
|
|
for i := range f.handActs {
|
|
f.handActs[i].Cause = "machine-units"
|
|
}
|
|
if got := watchHandActs(f); len(got) != 1 {
|
|
t.Fatalf("a silence by hand stopped counting: %+v", got)
|
|
}
|
|
}
|
|
|
|
// **Data loss is never silenced from a notification** (ADR 0258): data missing or shrunk offers no answer,
|
|
// and says where it is silenced.
|
|
func TestDataLossOffersNoSilence(t *testing.T) {
|
|
for _, kind := range []string{kindDataMissing, kindDataShrank} {
|
|
w := plainWordings[kind](conditions.Observation{Scope: conditions.ScopeMachine, ID: "ace.immich.library",
|
|
Machine: "ace", Kind: kind, Severity: conditions.Urgent})
|
|
if len(w.Actions) != 0 || !strings.HasSuffix(w.Needs, FromMeshMCPServer) {
|
|
t.Errorf("%s: %+v", kind, w)
|
|
}
|
|
if why, ok := conditions.PlainWords(w, "ace"); !ok {
|
|
t.Errorf("%s: %s", kind, why)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **Updates held after a failed release** (2026-10-08): the popup read "Needs you: release them, or leave
|
|
// them held." — naming neither what waits nor where it is released. It names the modules and machines, and
|
|
// the mesh MCP server.
|
|
func TestUpdatesHeldNameWhatWaitsAndWhereItIsReleased(t *testing.T) {
|
|
saved := backlogNow
|
|
t.Cleanup(func() { backlogNow = saved })
|
|
backlogNow.held = "release-1791457717307061152 failed (failed its gate on g14); what waits is released again by a person"
|
|
backlogNow.waiting = map[string][]inventory.CarriedMove{
|
|
"shanks": {{Module: "openrazer"}},
|
|
"g14": {{Module: "openrazer"}, {Module: "sensors"}},
|
|
}
|
|
got := backlogObservation()
|
|
if len(got) != 1 {
|
|
t.Fatalf("raised %d", len(got))
|
|
}
|
|
plainExample(t, got[0], "Updates of openrazer and sensors wait for your release",
|
|
"Needs you: release them from the mesh MCP server; this notification cannot do it. Updates of openrazer and "+
|
|
"sensors on g14 and shanks wait for a person to release them, because the last walk failed. "+
|
|
"They are not delivered until then, and stay held if you leave them.")
|
|
}
|
|
|
|
// **What held them is said in the glossary's words** (2026-10-08 review): the backlog is held after any
|
|
// walk failed, not a release, and a "check" is a pull request's status. So the words say the walk failed,
|
|
// and never that a release failed or a check did — with the modules and machines named or not.
|
|
func TestUpdatesHeldSayTheWalkFailed(t *testing.T) {
|
|
for _, w := range []words{
|
|
releaseHeldWords([]string{"openrazer"}, []string{"g14"}),
|
|
releaseHeldWords(nil, nil),
|
|
plainWordings["release-held"](conditions.Observation{Scope: conditions.ScopeMesh, ID: "release"}),
|
|
} {
|
|
if !strings.Contains(w.Explanation, "because the last walk failed.") {
|
|
t.Errorf("does not say the walk failed: %q", w.Explanation)
|
|
}
|
|
for _, wrong := range []string{"release failed", "check"} {
|
|
if strings.Contains(w.Explanation, wrong) {
|
|
t.Errorf("says %q: %q", wrong, w.Explanation)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// mcpVerb is a need that opens with a verb only the mesh MCP server performs (ADR 0258 §1): release, stop,
|
|
// start, restart, and a restore.
|
|
var mcpVerb = regexp.MustCompile(`^(release|stop|start|restart|restore)\b`)
|
|
|
|
// **A need no notification can answer says where it is answered** (ADR 0258 §1): every wording whose need
|
|
// opens with a verb the mesh MCP server performs, and offers no action, ends with FromMeshMCPServer — the
|
|
// kinds worded here for every subject shape, and those worded where they are raised. A new kind that misses
|
|
// it fails here; release-held did (2026-10-08).
|
|
func TestANeedNoNotificationAnswersNamesTheMeshMCPServer(t *testing.T) {
|
|
check := func(what string, w words) {
|
|
t.Helper()
|
|
if w.Needs == "" || len(w.Actions) > 0 || !mcpVerb.MatchString(w.Needs) {
|
|
return
|
|
}
|
|
if !strings.HasSuffix(w.Needs, FromMeshMCPServer) {
|
|
t.Errorf("%s needs %q without %q", what, w.Needs, FromMeshMCPServer)
|
|
}
|
|
}
|
|
subjects := []conditions.Observation{
|
|
{Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"},
|
|
{Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"},
|
|
{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"},
|
|
{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"},
|
|
{Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"},
|
|
{Scope: conditions.ScopeMesh, ID: "release", Also: []string{"g14"}},
|
|
}
|
|
for kind, fn := range plainWordings {
|
|
for _, s := range subjects {
|
|
for _, sev := range []conditions.Severity{conditions.Warning, conditions.Urgent} {
|
|
s.Kind, s.Severity, s.Resolver = kind, sev, conditions.ResolverOperator
|
|
check(kind+" about "+s.ID, fn(s))
|
|
}
|
|
}
|
|
}
|
|
check("a walk waiting", words{Needs: waitingNeeds(conditions.Urgent)})
|
|
check("a module's failed service", words{Needs: moduleNeeds("g14",
|
|
[]inventory.ResourceHealth{{Kind: link.KindUnit, Target: "openrazer-daemon.service"}})})
|
|
for _, state := range []string{"held", "ready", "failing"} {
|
|
_, _, _, needs, actions := stalledWords(stalledLine{ID: "novox/hq@055550802096", State: state, For: "36h"},
|
|
conditions.Observation{Resolver: conditions.ResolverOperator})
|
|
check("a delivery "+state, words{Needs: needs, Actions: actions})
|
|
}
|
|
check("updates held", releaseHeldWords([]string{"openrazer"}, []string{"g14"}))
|
|
}
|
|
|
|
// **A pull request the forge never announced says what to do about it** (novox/hq issue 347): mesh-delivery says one
|
|
// as a stalled line in state `unannounced` — no delivery exists, so there is nothing to stop, release or close —
|
|
// and the operator's one act is a new commit on its branch, which the forge announces.
|
|
func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) {
|
|
l := stalledLine{ID: "novox/hq@bfe82315f42c", Number: 243, State: "unannounced", For: "11m0s", Bound: "10m0s",
|
|
H2: "none: the forge never announced it, so there is no delivery to close — the operator's",
|
|
Says: "novox/hq#243 is open on main, which requires the merge check, and its head has had no merge check"}
|
|
obs := stalledObservations([]stalledLine{l})
|
|
if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator {
|
|
t.Fatalf("an unannounced pull request is not the operator's: %+v", obs)
|
|
}
|
|
o := obs[0]
|
|
if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") {
|
|
t.Fatalf("it says to %q", o.Needs)
|
|
}
|
|
if !strings.Contains(o.Headline, "no merge check") || !strings.Contains(o.Headline, "#243") ||
|
|
!strings.Contains(o.Explanation, "never") {
|
|
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
|
|
}
|
|
}
|