package main import ( "context" "encoding/json" "errors" "fmt" "strings" "sync" "testing" "time" "git.novox.be/novox/mesh-sdk/go/asks" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" ) // novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs // the one chosen on the router's warrant — once, for its own ask, the option offered, at its level. type memAskedStore map[string]asked // memAskedMu guards every memAskedStore: Change is a compare-and-set as the bus's is. var memAskedMu sync.Mutex func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) { memAskedMu.Lock() defer memAskedMu.Unlock() r, ok := m[id] if !ok { return nil, nil } return &r, nil } func (m memAskedStore) Create(_ context.Context, r asked) error { memAskedMu.Lock() defer memAskedMu.Unlock() if _, kept := m[r.ID]; kept { return errors.New("an ask is kept under that id") } m[r.ID] = r return nil } func (m memAskedStore) Change(_ context.Context, id string, change func(*asked) bool) (bool, error) { memAskedMu.Lock() defer memAskedMu.Unlock() r, ok := m[id] if !ok || !change(&r) { return false, nil } m[id] = r return true, nil } func (m memAskedStore) All(context.Context) ([]asked, error) { memAskedMu.Lock() defer memAskedMu.Unlock() var out []asked for _, r := range m { out = append(out, r) } return out, nil } type published struct { subject, id string body []byte } type askerRig struct { a *asker open []conditions.Condition store memAskedStore sent []published called []string silenced []string acts []link.HandAct now time.Time } func newAskerRig(t *testing.T) *askerRig { r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)} r.a = &asker{ open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil }, silence: func(_ context.Context, key string, d time.Duration, by, why string) error { r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why) // As the controller's conditions do (the confirmation review of 2026-10-09, M1): the condition is // silenced from now on, so what is asked next sees it silenced. for i := range r.open { if r.open[i].Key == key { r.open[i].Silenced = &conditions.Silence{Until: r.now.Add(d), By: by, Why: why, Since: r.now} } } return nil }, store: r.store, publish: func(_ context.Context, subject string, body []byte, id string) error { r.sent = append(r.sent, published{subject, id, body}) return nil }, call: func(_ context.Context, a conditions.Action, args map[string]string) error { raw, _ := json.Marshal(args) r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw)) return nil }, record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil }, now: func() time.Time { return r.now }, logf: t.Logf, } return r } func heldCondition() conditions.Condition { o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0] return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline, Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} } func unitsCondition() conditions.Condition { key := "machine.shanks.units" return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning, Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.", Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}} } func (r *askerRig) asksSent(t *testing.T) []asks.Ask { t.Helper() var out []asks.Ask for _, p := range r.sent { if p.subject != asks.AskSubject("mesh-controller") { continue } var q asks.Ask if err := json.Unmarshal(p.body, &q); err != nil { t.Fatal(err) } out = append(out, q) } return out } func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) { r := newAskerRig(t) quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"} r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet} if err := r.a.reconcile(context.Background()); err != nil { t.Fatal(err) } sent := r.asksSent(t) if len(sent) != 2 { t.Fatalf("asked %d times: %+v", len(sent), sent) } byAbout := map[string]asks.Ask{} for _, q := range sent { byAbout[q.About] = q if err := q.Check(r.now); err != nil { t.Errorf("%s: %v", q.About, err) } } held := byAbout[heldCondition().Key] if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve || held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator || held.OnExpiry == "" { t.Errorf("the held delivery is asked %+v", held) } units := byAbout["machine.shanks.units"] if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) { t.Errorf("the failed units are asked %+v", units) } // No second ask while one is open. r.now = r.now.Add(time.Minute) _ = r.a.reconcile(context.Background()) if n := len(r.asksSent(t)); n != 2 { t.Errorf("asked again while open: %d", n) } } func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{heldCondition(), unitsCondition()} _ = r.a.reconcile(context.Background()) // The units are silenced, the held delivery lasts past its ask's day. units := unitsCondition() units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)} r.open = []conditions.Condition{heldCondition(), units} r.now = r.now.Add(askApproveFor) if err := r.a.reconcile(context.Background()); err != nil { t.Fatal(err) } var cancels int for _, p := range r.sent { if p.subject == asks.CancelSubject("mesh-controller") { cancels++ } } if cancels != 1 { t.Errorf("cancels %d, want the silenced one's", cancels) } if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key { t.Errorf("the expired ask was not asked again: %+v", sent) } } // warrantFor is the router's warrant for the open ask about a condition, choosing an option by label. func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant { t.Helper() for _, a := range r.store { if a.Condition != condition || a.State != askOpen { continue } for _, o := range a.Ask.Options { if o.Label == label { return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now, AskDigest: a.Ask.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} } } } t.Fatalf("no open ask about %s offers %s", condition, label) return asks.Warrant{} } func answerWith(t *testing.T, r *askerRig, w asks.Warrant) { t.Helper() body, _ := json.Marshal(w) if err := r.a.Decided(context.Background(), body); err != nil { t.Fatal(err) } } func TestAWarrantIsActedOnOnce(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) w := r.warrantFor(t, heldCondition().Key, "Release") answerWith(t, r, w) answerWith(t, r, w) // heard again if len(r.called) != 1 { t.Fatalf("called %v", r.called) } want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}` if r.called[0] != want { t.Errorf("called\n %s\nwant\n %s", r.called[0], want) } if len(r.acts) != 1 { t.Fatalf("hand-acts %+v", r.acts) } act := r.acts[0] if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" || act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" || act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" { t.Errorf("the hand-act %+v", act) } if !personsDecision(act) { t.Error("an act on a warrant counts as a repair") } if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" { t.Errorf("kept %+v", got) } } func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) { for name, change := range map[string]func(*asks.Warrant){ "another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" }, "an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" }, "an option not offered": func(w *asks.Warrant) { w.Option = "delete" }, "another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge }, "no person": func(w *asks.Warrant) { w.By = nil }, "another ask's digest": func(w *asks.Warrant) { w.AskDigest = "sha256:0000" }, "no ask's digest": func(w *asks.Warrant) { w.AskDigest = "" }, } { t.Run(name, func(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) w := r.warrantFor(t, heldCondition().Key, "Stop") change(&w) answerWith(t, r, w) if len(r.called)+len(r.acts)+len(r.silenced) != 0 { t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced) } }) } } func TestEachAnswerCallsExactlyItsVerb(t *testing.T) { plan := "plan-1791454185265004861" waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent, Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.", Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)} module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning, Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.", Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart", Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove, Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}} for _, tc := range []struct { c conditions.Condition label string want string }{ {waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`}, {waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`}, {module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`}, } { r := newAskerRig(t) r.open = []conditions.Condition{tc.c} _ = r.a.reconcile(context.Background()) answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label)) if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) { t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want) } } } func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{unitsCondition()} _ = r.a.reconcile(context.Background()) w := r.warrantFor(t, "machine.shanks.units", "Silence for a week") w.Level, w.Proofs = asks.Acknowledge, nil w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14", Verified: "a desk click: whoever was at the operator's session on g14"} w.Channel = "desk-channel" answerWith(t, r, w) if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") { t.Fatalf("silenced %v, called %v", r.silenced, r.called) } if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 { t.Errorf("%+v", r.acts) } } func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) w := r.warrantFor(t, heldCondition().Key, "Release") w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time" answerWith(t, r, w) if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) || !strings.HasPrefix(r.store[w.Ask].Acted, "nothing") { t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask]) } // And a choice for a condition that ended meanwhile does nothing either. r2 := newAskerRig(t) r2.open = []conditions.Condition{heldCondition()} _ = r2.a.reconcile(context.Background()) w2 := r2.warrantFor(t, heldCondition().Key, "Release") r2.open = nil answerWith(t, r2, w2) if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" { t.Errorf("%v %+v", r2.called, r2.store[w2.Ask]) } } func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) w := r.warrantFor(t, heldCondition().Key, "Stop") r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) { if id != w.Ask { return nil, errors.New("another ask") } return &w, nil } r.now = r.now.Add(askCatchUpAfter) if err := r.a.reconcile(context.Background()); err != nil { t.Fatal(err) } if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") { t.Errorf("called %v", r.called) } if n := len(r.asksSent(t)); n != 1 { t.Errorf("asked again after the answer: %d", n) } } // After review (2026-10-08): a refused ask is not asked again at once; issue 369: nor is the refusal believed // for ever — it is asked again after a wait that doubles with each refusal in a row, and at once when the // channels change. func TestAnAskTheRouterRefusedIsAskedAgainAfterAGrowingWait(t *testing.T) { r := newAskerRig(t) channels := "channel/telegram=telegram@anchor[choice]own:true" r.a.channels = func(context.Context) string { return channels } r.open = []conditions.Condition{heldCondition()} refuse := func() { t.Helper() sent := r.asksSent(t) refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused, Words: "no channel can carry any of its answers now", At: r.now}) if err := r.a.Decided(context.Background(), refusal); err != nil { t.Fatal(err) } } _ = r.a.reconcile(context.Background()) first := r.asksSent(t)[0] refuse() if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") { t.Fatalf("the refusal was kept as %+v", got) } // Each wait: nothing asked before it ends, asked once when it has. for i, wait := range []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute, 16 * time.Minute, 30 * time.Minute, 30 * time.Minute} { before := len(r.asksSent(t)) r.now = r.now.Add(wait - 10*time.Second) _ = r.a.reconcile(context.Background()) if n := len(r.asksSent(t)); n != before { t.Fatalf("refusal %d: asked again before %s", i+1, wait) } r.now = r.now.Add(10 * time.Second) _ = r.a.reconcile(context.Background()) if n := len(r.asksSent(t)); n != before+1 { t.Fatalf("refusal %d: not asked again after %s (%d asks)", i+1, wait, n) } refuse() } before := len(r.asksSent(t)) channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true" _ = r.a.reconcile(context.Background()) if n := len(r.asksSent(t)); n != before+1 { t.Errorf("not asked again once the channels changed: %d", n-before) } } // After review: at most three asks open at once, the most urgent first, then the oldest. func TestAtMostThreeAsksAreOpenTheMostUrgentFirst(t *testing.T) { r := newAskerRig(t) var open []conditions.Condition for i := 0; i < 4; i++ { c := unitsCondition() c.Key = "machine.m" + string(rune('a'+i)) + ".units" c.Actions = []conditions.Action{conditions.SilenceAction(c.Key)} c.Raised = r.now.Add(-time.Duration(10-i) * time.Hour) open = append(open, c) } urgent := heldCondition() urgent.Severity, urgent.Raised = conditions.Urgent, r.now.Add(-time.Minute) r.open = append(open, urgent) _ = r.a.reconcile(context.Background()) sent := r.asksSent(t) if len(sent) != askMostOpen || sent[0].About != urgent.Key || sent[1].About != "machine.ma.units" || sent[2].About != "machine.mb.units" { var about []string for _, q := range sent { about = append(about, q.About) } t.Fatalf("asked %v", about) } } // After review: nothing is asked while no router takes asks under the controller's name, and that is said once. func TestNothingIsAskedWithoutARouter(t *testing.T) { r := newAskerRig(t) var said []string r.a.logf = func(f string, a ...any) { said = append(said, f) } r.a.routerHere = func(context.Context) (bool, error) { return false, nil } r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) _ = r.a.reconcile(context.Background()) if len(r.asksSent(t)) != 0 { t.Error("asked with no router") } n := 0 for _, s := range said { if strings.Contains(s, "no router takes asks") { n++ } } if n != 1 { t.Errorf("said %d times", n) } } // After review: the condition's words keep where an answer is given without a channel; the ask's text does not. func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) { c := heldCondition() if !strings.Contains(c.Explanation, FromMeshMCPServer) { t.Fatalf("the condition lost where it is answered: %q", c.Explanation) } q, _ := askOf("x", c, partsOf(c)[0], time.Now()) if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") { t.Errorf("the ask says %q", q.Explanation) } if askApproveFor >= 24*time.Hour { t.Errorf("an approving ask lasts %s, which the SDK may refuse at its bound", askApproveFor) } } // After review (security finding 9): a warrant is acted on only for an ask open in the controller's own record, // once the claim stands, and never when given after the ask expired. func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) w := r.warrantFor(t, heldCondition().Key, "Release") late := w late.At = r.store[w.Ask].Ask.Expires.Add(time.Minute) body, _ := json.Marshal(late) _ = r.a.Decided(context.Background(), body) if len(r.called) != 0 { t.Fatalf("acted on a warrant given after the ask expired: %v", r.called) } // Claimed already by another delivery: nothing done here. kept := r.store[w.Ask] kept.Acted = "acting" r.store[w.Ask] = kept body, _ = json.Marshal(w) _ = r.a.Decided(context.Background(), body) if len(r.called) != 0 { t.Fatalf("acted though the claim was another's: %v", r.called) } // Cancelled in its own record: refused. kept.Acted, kept.State = "", askCancelled r.store[w.Ask] = kept _ = r.a.Decided(context.Background(), body) if len(r.called) != 0 { t.Errorf("acted on a cancelled ask: %v", r.called) } } // novox/hq ADR 0259 §6: a warrant authorises the act its option bound when the controller asked, and no // other. A record of the act changed after the ask — another delivery, another machine, another argument — // is refused and nothing is performed. func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) { for name, change := range map[string]func(*conditions.Action){ "another argument": func(a *conditions.Action) { a.Arguments = map[string]string{"id": "novox/mesh-controller@000000000000"} }, "another verb": func(a *conditions.Action) { a.Verb = "mesh-delivery.stop" }, "another machine": func(a *conditions.Action) { a.Machine = "anchor" }, } { t.Run(name, func(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) w := r.warrantFor(t, heldCondition().Key, "Release") kept := r.store[w.Ask] acts := append([]conditions.Action(nil), kept.Actions...) i := kept.Options[w.Option] change(&acts[i]) kept.Actions = acts r.store[w.Ask] = kept answerWith(t, r, w) if len(r.called)+len(r.acts) != 0 { t.Errorf("performed an act the option did not bind: %v %v", r.called, r.acts) } }) } // Every option of an ask binds its act. q, _ := askOf("x", heldCondition(), partsOf(heldCondition())[0], time.Now()) for _, o := range q.Options { if o.Binds == "" { t.Errorf("the option %s binds nothing", o.ID) } } } // Failure is loud (novox/hq ADR 0259, the self-review of 2026-10-09): a condition that needs the operator and // could not be asked on any channel — no router, or the router refused the ask — is a condition of its own, // cleared once it can be asked again. func TestAnAskThatCannotBeDeliveredIsSaid(t *testing.T) { r := newAskerRig(t) var raised [][]conditions.Observation r.a.raise = func(_ context.Context, obs []conditions.Observation) error { raised = append(raised, obs) return nil } last := func() []conditions.Observation { return raised[len(raised)-1] } routerHere := false r.a.routerHere = func(context.Context) (bool, error) { return routerHere, nil } channels := "channel/telegram=telegram@anchor[choice]own:true" r.a.channels = func(context.Context) string { return channels } r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) if got := last(); len(got) != 1 || got[0].Kind != "asks-undelivered" || !strings.Contains(got[0].Summary, heldCondition().Key) || !strings.Contains(got[0].Summary, "no router") { t.Fatalf("no router, said as %+v", got) } routerHere = true _ = r.a.reconcile(context.Background()) if got := last(); len(got) != 0 { t.Fatalf("asked, and still said undelivered: %+v", got) } first := r.asksSent(t)[0] refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused, Words: "no channel can carry any of its answers now", At: r.now}) if err := r.a.Decided(context.Background(), refusal); err != nil { t.Fatal(err) } _ = r.a.reconcile(context.Background()) if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no channel can carry") { t.Fatalf("the router's refusal, said as %+v", got) } if why, ok := conditions.PlainWords(conditions.Words{Headline: last()[0].Headline, Explanation: last()[0].Explanation, Needs: last()[0].Needs, Resolved: last()[0].Resolved}, ""); !ok { t.Errorf("not plain: %s", why) } r.open = nil _ = r.a.reconcile(context.Background()) if got := last(); len(got) != 0 { t.Errorf("nothing needs asking, and still said: %+v", got) } } // The review of 2026-10-09 (M1): an acknowledging answer never shares an ask with an authorising one. A // condition offering Restart and Silence is asked twice — Restart alone, about the condition, and Silence // alone, apart — so Silence chosen on a channel that only acknowledges leaves the Restart ask open. func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) { r := newAskerRig(t) key := "module.shanks.plex.down" c := conditions.Condition{Key: key, Kind: "module-down", Severity: conditions.Urgent, Headline: "Plex down on shanks", Explanation: "Needs you: restart it, or silence this.", Needs: "restart it, or silence this.", Actions: []conditions.Action{ {Label: "Restart", Verb: "node-service-manager.restart", Machine: "shanks", Level: conditions.LevelApprove, Arguments: map[string]string{"unit": "plex"}}, conditions.SilenceAction(key)}} r.open = []conditions.Condition{c} if err := r.a.reconcile(context.Background()); err != nil { t.Fatal(err) } sent := r.asksSent(t) if len(sent) != 2 { t.Fatalf("asked %d time(s): %+v", len(sent), sent) } for _, q := range sent { if err := q.Check(r.now); err != nil { t.Errorf("%s: %v", q.About, err) } levels := map[asks.Level]bool{} for _, o := range q.Options { levels[o.Level] = true } if len(levels) != 1 { t.Errorf("the ask about %s mixes levels: %+v", q.About, q.Options) } } byAbout := map[string]asks.Ask{} for _, q := range sent { byAbout[q.About] = q } if q := byAbout[key]; len(q.Options) != 1 || q.Options[0].Label != "Restart" { t.Errorf("the condition's own ask: %+v", q) } if q := byAbout[key+".acknowledge"]; len(q.Options) != 1 || q.Options[0].Level != asks.Acknowledge { t.Errorf("the acknowledging ask: %+v", q) } // Silence chosen: performed, and the Restart ask stays open, never asked twice. answerWith(t, r, r.warrantFor(t, key, "Silence for a week")) if len(r.silenced) != 1 || len(r.called) != 0 { t.Fatalf("silenced %v called %v", r.silenced, r.called) } _ = r.a.reconcile(context.Background()) open := 0 for _, a := range r.store { if a.State == askOpen && a.Condition == key { open++ if a.Part != "" || a.Ask.Options[0].Label != "Restart" { t.Errorf("the open ask is %+v", a) } } } if open != 1 || len(r.asksSent(t)) != 2 { t.Errorf("after the silence: %d open, %d asked", open, len(r.asksSent(t))) } // And the approval still answers: Restart chosen on a channel that proves who answered is performed. answerWith(t, r, r.warrantFor(t, key, "Restart")) if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "node-service-manager.restart@shanks") { t.Errorf("the approval kept through a silence was not performed: %v", r.called) } } // novox/hq issue 353: the controller's grant to ask is composed from the router's assignment and reaches the // bus only when its machine is pushed. Between the two, the bus refuses every ask (measured 2026-10-09, 17:54 to // 17:56 local: seven refusals of mesh.seat.operator-channel.accept.ask.mesh-controller). So nothing is asked // while the bus's user list is behind, it is said once, the conditions that need the operator are raised as // undelivered with what to do, and the asks go out once the bus holds the grant. func TestNothingIsAskedWhileTheBusLacksTheControllersGrant(t *testing.T) { r := newAskerRig(t) var said []string r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) } var raised [][]conditions.Observation r.a.raise = func(_ context.Context, obs []conditions.Observation) error { raised = append(raised, obs) return nil } held := false r.a.grantHeld = func(context.Context) (bool, string, error) { return held, "the bus's user list on anchor is behind what the mesh composes; `push anchor` carries it", nil } r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) _ = r.a.reconcile(context.Background()) if len(r.asksSent(t)) != 0 { t.Error("asked while the bus lacks the grant") } n := 0 for _, s := range said { if strings.Contains(s, "does not hold the controller's grant") { n++ } } if n != 1 { t.Errorf("said %d times: %q", n, said) } if len(raised) == 0 || len(raised[len(raised)-1]) != 1 || !strings.Contains(raised[len(raised)-1][0].Summary, "`push anchor` carries it") || raised[len(raised)-1][0].Kind != "asks-undelivered" { t.Fatalf("not said as a condition with what to do: %+v", raised) } held = true _ = r.a.reconcile(context.Background()) if sent := r.asksSent(t); len(sent) != 1 || sent[0].About != heldCondition().Key { t.Errorf("not asked once the bus holds the grant: %+v", sent) } if last := raised[len(raised)-1]; len(last) != 0 { t.Errorf("the undelivered condition was not cleared: %+v", last) } } // novox/hq issue 369: the router refused while its channels had not yet said they could send; they could twenty // minutes later, and "Questions for you not delivered" repeated that refusal for eleven hours, escalating on it. // The ask is made again; while the router's word on it is awaited the condition stands unchanged (neither // cleared nor raised again); once the router took it, the condition clears; a new refusal is said in its own // words. func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testing.T) { r := newAskerRig(t) var raised [][]conditions.Observation r.a.raise = func(_ context.Context, obs []conditions.Observation) error { raised = append(raised, obs) return nil } last := func() []conditions.Observation { return raised[len(raised)-1] } r.a.channels = func(context.Context) string { return "channel/telegram=telegram@anchor[choice]own:true" } r.open = []conditions.Condition{unitsCondition()} refuse := func(words string) { t.Helper() sent := r.asksSent(t) refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused, Words: words, At: r.now}) if err := r.a.Decided(context.Background(), refusal); err != nil { t.Fatal(err) } } _ = r.a.reconcile(context.Background()) refuse("no channel can carry any of its answers now: telegram: telegram has not said whether it can send") _ = r.a.reconcile(context.Background()) if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "telegram has not said") { t.Fatalf("the refusal, said as %+v", got) } // The wait ends: asked again; until the router's word on it is in, the condition stands as it was. r.now = r.now.Add(askEvery) _ = r.a.reconcile(context.Background()) if n := len(r.asksSent(t)); n != 2 { t.Fatalf("not asked again: %d asks", n) } if got := last(); len(got) != 1 { t.Fatalf("cleared while the router's word on the new ask was awaited: the condition would flap") } r.now = r.now.Add(askVerdictWait / 2) _ = r.a.reconcile(context.Background()) if got := last(); len(got) != 1 { t.Fatalf("cleared inside the verdict wait: the condition would flap") } // Refused again, now for a reason of today: said in those words. refuse("no channel can carry any of its answers now: telegram: no account is linked") _ = r.a.reconcile(context.Background()) if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no account is linked") || strings.Contains(got[0].Summary, "has not said") { t.Fatalf("an old refusal's words repeated: %+v", got) } // Asked again after the doubled wait, and taken: no refusal comes, and the condition clears. r.now = r.now.Add(2 * askEvery) _ = r.a.reconcile(context.Background()) if n := len(r.asksSent(t)); n != 3 { t.Fatalf("not asked again after the doubled wait: %d asks", n) } r.now = r.now.Add(askVerdictWait) _ = r.a.reconcile(context.Background()) if got := last(); len(got) != 0 { t.Fatalf("taken, and still said undelivered: %+v", got) } if n := len(r.asksSent(t)); n != 3 { t.Fatalf("an ask taken was asked again: %d asks", n) } // The operator answers it; the condition stays open (its fix takes a while): the old refusal is not said again. taken := r.asksSent(t)[2] r.store.Change(context.Background(), taken.ID, func(x *asked) bool { x.State, x.Ended = string(asks.OutcomeChosen), r.now return true }) for i := 0; i < 5; i++ { r.now = r.now.Add(askEvery) _ = r.a.reconcile(context.Background()) if got := last(); len(got) != 0 { t.Fatalf("an answered ask brought its old refusal back: %+v", got) } } // Its explanation opens with one verdict, not two. r.open = append(r.open, heldCondition()) _ = r.a.reconcile(context.Background()) refuse("no channel can carry any of its answers now") _ = r.a.reconcile(context.Background()) if e := conditions.Verdict(last()[0].Needs, last()[0].Explanation); strings.Count(e, "Needs you") != 1 { t.Errorf("the explanation says its verdict twice: %q", e) } } // hq issue 353, the review: the bus holds the controller's grant to ask only when both facts hold — the mesh // composes the controller's own grant, and the bus's module says it is in force on the running server — and // each fact not known is not held, said with why. func TestTheGrantIsHeldOnlyWhenComposedAndInForceOnTheBus(t *testing.T) { yes := func(context.Context) (bool, string, error) { return true, "", nil } no := func(why string) grantJudge { return func(context.Context) (bool, string, error) { return false, why, nil } } broken := func(context.Context) (bool, string, error) { return false, "", errors.New("the store is away") } for _, c := range []struct { name string composed, inForce grantJudge held bool says string }{ {"composed and in force", yes, yes, true, ""}, {"not composed", no("no router is assigned"), yes, false, "no router is assigned"}, {"composed, written, not reloaded", yes, no("the server has not reloaded"), false, "has not reloaded"}, {"the composition unknown", broken, yes, false, "could not be worked out: the store is away"}, {"the bus unknown", yes, broken, false, "could not be worked out: the store is away"}, {"nothing judges the bus", yes, nil, false, "not judged"}, } { held, why, err := grantHeldBy(c.composed, c.inForce, time.Now)(context.Background()) if err != nil || held != c.held || !strings.Contains(why, c.says) { t.Errorf("%s: held %v (%q, %v)", c.name, held, why, err) } } // Remembered for grantLookEvery, then judged again. now := time.Date(2026, 10, 9, 18, 0, 0, 0, time.UTC) inForce := false judge := grantHeldBy(yes, func(context.Context) (bool, string, error) { return inForce, "not yet", nil }, func() time.Time { return now }) if held, _, _ := judge(context.Background()); held { t.Fatal("held before the bus enforces it") } inForce = true if held, _, _ := judge(context.Background()); held { t.Error("judged again inside grantLookEvery") } now = now.Add(grantLookEvery) if held, _, _ := judge(context.Background()); !held { t.Error("not judged again after grantLookEvery") } } // The controller's own grant, as composed: present while a router that takes asks is assigned, absent // otherwise — whatever else in the user list changes. func TestTheControllersOwnGrantToAskIsReadFromTheComposition(t *testing.T) { router := broker.Declared{Module: "messenger", Holds: []broker.Seat{{Name: broker.AsksSeat, Scope: "mesh", Accepts: []string{"ask", "cancel"}, Emits: []string{"decided"}, ByCaller: []string{"ask", "cancel", "decided"}}}} other := broker.Declared{Module: "dunst"} with := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {router, other}}} without := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {other}}} if may, err := controllerMayAsk(with); err != nil || !may { t.Errorf("with a router assigned: %v %v", may, err) } if may, err := controllerMayAsk(without); err != nil || may { t.Errorf("with no router assigned: %v %v", may, err) } } // The bus module's answer, with the record's word on the list sent as the check only where the answer does // not settle it (reviews 22 and 31 of PR 188); each way it is not held names its own remedy. func TestTheBusModulesAnswerNamesTheRemedyForEachCase(t *testing.T) { const ( reloaded = `{"allowed":true,"in_force":true,"file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:56:05Z"}` notLoaded = `{"allowed":true,"in_force":false,"why":"the user list on the machine allows it, and the server has not reloaded since it was written","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}` noVarz = `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}` noStat = `{"allowed":true,"in_force":false,"why":"when the bus's user list was written could not be read inside mesh-broker-nats: exit 1"}` notAllowed = `{"allowed":false,"in_force":false,"why":"the user list on the machine does not allow it"}` ) current := busListState{Machine: "anchor"} behind := busListState{Machine: "anchor", Behind: true} stepped := busListState{Machine: "anchor", Behind: true, BusStep: "sending anchor would replace the bus; `bus upgrade --why …`"} uncomposed := busListState{Machine: "anchor", Behind: true, Uncomposed: errors.New("module foo requires a provision nobody provides")} for _, c := range []struct { name, answer string askErr error list busListState listErr error held bool says, not string }{ {"in force", reloaded, nil, behind, nil, true, "", "push"}, {"written, not reloaded", notLoaded, nil, current, nil, false, "within about 5 s", "push"}, {"in force not known: varz", noVarz, nil, current, nil, false, "is not known: the server's last configuration load", "wait"}, {"in force not known: stat", noStat, nil, current, nil, false, "is not known: when the bus's user list was written", "wait"}, {"not allowed, list behind", notAllowed, nil, behind, nil, false, "`push anchor` carries it", "5 s"}, {"not allowed, bus step pending", notAllowed, nil, stepped, nil, false, "`bus upgrade", "`push anchor`"}, {"not allowed, list uncomposed", notAllowed, nil, uncomposed, nil, false, "`plan anchor`", "`push anchor`"}, {"older module, allowed, list current", `{"allowed":true}`, nil, current, nil, true, "not confirmed", "push"}, {"older module, allowed, list behind", `{"allowed":true}`, nil, behind, nil, true, "not confirmed", "push"}, {"older module, allowed, list unknown", `{"allowed":true}`, nil, current, errors.New("the store is away"), true, "`bus upgrade`", "push"}, {"older module, not allowed, list behind", `{"allowed":false}`, nil, behind, nil, false, "`push anchor` carries it", "5 s"}, {"module cannot answer, list current", "", errors.New("no responders"), current, nil, true, "could not answer", "push"}, {"module cannot answer, list behind", "", errors.New("no responders"), behind, nil, false, "`push anchor` carries it", ""}, {"module cannot answer, list unknown", "", errors.New("no responders"), current, errors.New("the store is away"), false, "could not be worked out", ""}, {"answer unreadable, list behind", `not json`, nil, behind, nil, false, "could not be read", ""}, } { judge := busGrantBy( func(context.Context) (string, json.RawMessage, error) { return "anchor", json.RawMessage(c.answer), c.askErr }, func(context.Context) (busListState, error) { return c.list, c.listErr }) held, why, _ := judgeGrantOnce(judge) if held != c.held || !strings.Contains(why, c.says) || (c.not != "" && strings.Contains(why, c.not)) { t.Errorf("%s: held %v (%q)", c.name, held, why) } } none := busGrantBy(func(context.Context) (string, json.RawMessage, error) { return "", nil, nil }, nil) if held, why, _ := judgeGrantOnce(none); held || !strings.Contains(why, "no module holds the bus") { t.Errorf("no bus: held %v (%q)", held, why) } } // judgeGrantOnce is the whole judgement with the composed grant held, as the asker makes it. func judgeGrantOnce(bus grantJudge) (bool, string, error) { yes := func(context.Context) (bool, string, error) { return true, "", nil } return grantHeldBy(yes, bus, time.Now)(context.Background()) } // The orders reviews 22 and 31 of PR 188 found: this controller live before the bus module that answers // in_force, with the bus's machine behind on its list — and, after mesh-catalog #166 merged, unpushable until a // person's bus upgrade. An older module that allows the ask asks (once, with the caveat said); a new module that // has not reloaded names the wait, not a push; one that cannot say whether it reloaded names what is not known. func TestAnOlderBusModuleNeitherSilencesTheAsksNorNamesTheWrongRemedy(t *testing.T) { for _, c := range []struct { name, answer string list busListState asks bool says, not string }{ {"older module, list current", `{"allowed":true}`, busListState{Machine: "anchor"}, true, "not confirmed", ""}, {"older module, list behind, bus step pending", `{"allowed":true}`, busListState{Machine: "anchor", Behind: true, BusStep: "a planned step"}, true, "not confirmed", ""}, {"new module, not reloaded", `{"allowed":true,"in_force":false,"why":"not reloaded","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`, busListState{Machine: "anchor"}, false, "within about 5 s", "push"}, {"new module, reload not known", `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`, busListState{Machine: "anchor"}, false, "is not known", "wait"}, } { t.Run(c.name, func(t *testing.T) { r := newAskerRig(t) var said []string r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) } var raised [][]conditions.Observation r.a.raise = func(_ context.Context, obs []conditions.Observation) error { raised = append(raised, obs) return nil } yes := func(context.Context) (bool, string, error) { return true, "", nil } r.a.grantHeld = grantHeldBy(yes, busGrantBy( func(context.Context) (string, json.RawMessage, error) { return "anchor", json.RawMessage(c.answer), nil }, func(context.Context) (busListState, error) { return c.list, nil }), func() time.Time { return r.now }) r.open = []conditions.Condition{heldCondition()} _ = r.a.reconcile(context.Background()) _ = r.a.reconcile(context.Background()) sent := r.asksSent(t) if c.asks { if len(sent) != 1 || sent[0].About != heldCondition().Key { t.Fatalf("not asked: %+v", sent) } n := 0 for _, s := range said { if strings.Contains(s, c.says) { n++ } } if n != 1 { t.Errorf("the caveat said %d times: %q", n, said) } return } if len(sent) != 0 { t.Fatalf("asked while the bus does not hold the grant: %+v", sent) } if len(raised) == 0 || len(raised[len(raised)-1]) != 1 || raised[len(raised)-1][0].Kind != "asks-undelivered" { t.Fatalf("not said as undelivered: %+v", raised) } summary := raised[len(raised)-1][0].Summary if !strings.Contains(summary, c.says) || (c.not != "" && strings.Contains(summary, c.not)) { t.Errorf("the condition names the wrong remedy: %q", summary) } }) } } // The serving asker is wired to the real judgement: with records and a bus it cannot read, the grant is not // held, and so nothing is asked (a stub saying yes, or no judgement at all, would ask). func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) { a := servingAsker(&stores{}, nil, nil, nil) if a.grantHeld == nil { t.Fatal("the serving asker does not judge the bus's grant") } held, why, err := a.grantHeld(context.Background()) if err != nil || held || !strings.Contains(why, "could not be worked out") { t.Errorf("held %v (%q, %v) with nothing to read", held, why, err) } }