Files
mesh-controller/cmd/mesh-controller/asker_test.go
T
jschoubben a66c13719d Bind each asked option to the exact act, and perform only that act on its warrant (hq ADR 0259 §6)
Every option the controller asks with carries the digest of its verb, machine, arguments and level
(the SDK's Option.Binds). A warrant must name the digest of the ask the controller keeps, and before
acting the controller checks that the act it is about to perform is the one the option bound: a
record changed after the ask is refused, never performed. mesh-sdk moves to d4077b4.
2026-10-09 11:03:53 +02:00

517 lines
19 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs
// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level.
type memAskedStore map[string]asked
func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) {
r, ok := m[id]
if !ok {
return nil, nil
}
return &r, nil
}
func (m memAskedStore) Put(_ context.Context, r asked) error { m[r.ID] = r; return nil }
func (m memAskedStore) Claim(_ context.Context, id string, w asks.Warrant) (bool, error) {
r, ok := m[id]
if !ok || r.State != askOpen || r.Acted != "" {
return false, nil
}
r.State, r.Warrant, r.Acted = string(asks.OutcomeChosen), &w, "acting"
m[id] = r
return true, nil
}
func (m memAskedStore) All(context.Context) ([]asked, error) {
var out []asked
for _, r := range m {
out = append(out, r)
}
return out, nil
}
type published struct {
subject, id string
body []byte
}
type askerRig struct {
a *asker
open []conditions.Condition
store memAskedStore
sent []published
called []string
silenced []string
acts []link.HandAct
now time.Time
}
func newAskerRig(t *testing.T) *askerRig {
r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)}
r.a = &asker{
open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil },
silence: func(_ context.Context, key string, d time.Duration, by, why string) error {
r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why)
return nil
},
store: r.store,
publish: func(_ context.Context, subject string, body []byte, id string) error {
r.sent = append(r.sent, published{subject, id, body})
return nil
},
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
raw, _ := json.Marshal(args)
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
return nil
},
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
now: func() time.Time { return r.now },
logf: t.Logf,
}
return r
}
func heldCondition() conditions.Condition {
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
}
func unitsCondition() conditions.Condition {
key := "machine.shanks.units"
return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning,
Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.",
Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}}
}
func (r *askerRig) asksSent(t *testing.T) []asks.Ask {
t.Helper()
var out []asks.Ask
for _, p := range r.sent {
if p.subject != asks.AskSubject("mesh-controller") {
continue
}
var q asks.Ask
if err := json.Unmarshal(p.body, &q); err != nil {
t.Fatal(err)
}
out = append(out, q)
}
return out
}
func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) {
r := newAskerRig(t)
quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"}
r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet}
if err := r.a.reconcile(context.Background()); err != nil {
t.Fatal(err)
}
sent := r.asksSent(t)
if len(sent) != 2 {
t.Fatalf("asked %d times: %+v", len(sent), sent)
}
byAbout := map[string]asks.Ask{}
for _, q := range sent {
byAbout[q.About] = q
if err := q.Check(r.now); err != nil {
t.Errorf("%s: %v", q.About, err)
}
}
held := byAbout[heldCondition().Key]
if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve ||
held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator ||
held.OnExpiry == "" {
t.Errorf("the held delivery is asked %+v", held)
}
units := byAbout["machine.shanks.units"]
if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) {
t.Errorf("the failed units are asked %+v", units)
}
// No second ask while one is open.
r.now = r.now.Add(time.Minute)
_ = r.a.reconcile(context.Background())
if n := len(r.asksSent(t)); n != 2 {
t.Errorf("asked again while open: %d", n)
}
}
func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) {
r := newAskerRig(t)
r.open = []conditions.Condition{heldCondition(), unitsCondition()}
_ = r.a.reconcile(context.Background())
// The units are silenced, the held delivery lasts past its ask's day.
units := unitsCondition()
units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)}
r.open = []conditions.Condition{heldCondition(), units}
r.now = r.now.Add(askApproveFor)
if err := r.a.reconcile(context.Background()); err != nil {
t.Fatal(err)
}
var cancels int
for _, p := range r.sent {
if p.subject == asks.CancelSubject("mesh-controller") {
cancels++
}
}
if cancels != 1 {
t.Errorf("cancels %d, want the silenced one's", cancels)
}
if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key {
t.Errorf("the expired ask was not asked again: %+v", sent)
}
}
// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label.
func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant {
t.Helper()
for _, a := range r.store {
if a.Condition != condition || a.State != askOpen {
continue
}
for _, o := range a.Ask.Options {
if o.Label == label {
return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen,
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now,
AskDigest: a.Ask.Digest(),
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
}
}
}
t.Fatalf("no open ask about %s offers %s", condition, label)
return asks.Warrant{}
}
func answerWith(t *testing.T, r *askerRig, w asks.Warrant) {
t.Helper()
body, _ := json.Marshal(w)
if err := r.a.Decided(context.Background(), body); err != nil {
t.Fatal(err)
}
}
func TestAWarrantIsActedOnOnce(t *testing.T) {
r := newAskerRig(t)
r.open = []conditions.Condition{heldCondition()}
_ = r.a.reconcile(context.Background())
w := r.warrantFor(t, heldCondition().Key, "Release")
answerWith(t, r, w)
answerWith(t, r, w) // heard again
if len(r.called) != 1 {
t.Fatalf("called %v", r.called)
}
want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}`
if r.called[0] != want {
t.Errorf("called\n %s\nwant\n %s", r.called[0], want)
}
if len(r.acts) != 1 {
t.Fatalf("hand-acts %+v", r.acts)
}
act := r.acts[0]
if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" ||
act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" ||
act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" {
t.Errorf("the hand-act %+v", act)
}
if !personsDecision(act) {
t.Error("an act on a warrant counts as a repair")
}
if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
t.Errorf("kept %+v", got)
}
}
func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
for name, change := range map[string]func(*asks.Warrant){
"another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" },
"an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" },
"an option not offered": func(w *asks.Warrant) { w.Option = "delete" },
"another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge },
"no person": func(w *asks.Warrant) { w.By = nil },
"another ask's digest": func(w *asks.Warrant) { w.AskDigest = "sha256:0000" },
"no ask's digest": func(w *asks.Warrant) { w.AskDigest = "" },
} {
t.Run(name, func(t *testing.T) {
r := newAskerRig(t)
r.open = []conditions.Condition{heldCondition()}
_ = r.a.reconcile(context.Background())
w := r.warrantFor(t, heldCondition().Key, "Stop")
change(&w)
answerWith(t, r, w)
if len(r.called)+len(r.acts)+len(r.silenced) != 0 {
t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced)
}
})
}
}
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
plan := "plan-1791454185265004861"
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
for _, tc := range []struct {
c conditions.Condition
label string
want string
}{
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
} {
r := newAskerRig(t)
r.open = []conditions.Condition{tc.c}
_ = r.a.reconcile(context.Background())
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
}
}
}
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
r := newAskerRig(t)
r.open = []conditions.Condition{unitsCondition()}
_ = r.a.reconcile(context.Background())
w := r.warrantFor(t, "machine.shanks.units", "Silence for a week")
w.Level, w.Proofs = asks.Acknowledge, nil
w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14",
Verified: "a desk click: whoever was at the operator's session on g14"}
w.Channel = "desk-channel"
answerWith(t, r, w)
if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") {
t.Fatalf("silenced %v, called %v", r.silenced, r.called)
}
if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 {
t.Errorf("%+v", r.acts)
}
}
func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) {
r := newAskerRig(t)
r.open = []conditions.Condition{heldCondition()}
_ = r.a.reconcile(context.Background())
w := r.warrantFor(t, heldCondition().Key, "Release")
w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time"
answerWith(t, r, w)
if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) ||
!strings.HasPrefix(r.store[w.Ask].Acted, "nothing") {
t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask])
}
// And a choice for a condition that ended meanwhile does nothing either.
r2 := newAskerRig(t)
r2.open = []conditions.Condition{heldCondition()}
_ = r2.a.reconcile(context.Background())
w2 := r2.warrantFor(t, heldCondition().Key, "Release")
r2.open = nil
answerWith(t, r2, w2)
if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" {
t.Errorf("%v %+v", r2.called, r2.store[w2.Ask])
}
}
func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) {
r := newAskerRig(t)
r.open = []conditions.Condition{heldCondition()}
_ = r.a.reconcile(context.Background())
w := r.warrantFor(t, heldCondition().Key, "Stop")
r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) {
if id != w.Ask {
return nil, errors.New("another ask")
}
return &w, nil
}
r.now = r.now.Add(askCatchUpAfter)
if err := r.a.reconcile(context.Background()); err != nil {
t.Fatal(err)
}
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") {
t.Errorf("called %v", r.called)
}
if n := len(r.asksSent(t)); n != 1 {
t.Errorf("asked again after the answer: %d", n)
}
}
// After review (2026-10-08): a refused ask is not asked again until its answers or the channels change.
func TestAnAskTheRouterRefusedWaitsUntilSomethingChanges(t *testing.T) {
r := newAskerRig(t)
channels := "channel/telegram=telegram@anchor[choice]own:true"
r.a.channels = func(context.Context) string { return channels }
r.open = []conditions.Condition{heldCondition()}
_ = r.a.reconcile(context.Background())
first := r.asksSent(t)[0]
refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused,
Words: "no channel can carry any of its answers now", At: r.now})
if err := r.a.Decided(context.Background(), refusal); err != nil {
t.Fatal(err)
}
if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") {
t.Fatalf("the refusal was kept as %+v", got)
}
for i := 0; i < 3; i++ {
r.now = r.now.Add(askEvery)
_ = r.a.reconcile(context.Background())
}
if n := len(r.asksSent(t)); n != 1 {
t.Fatalf("asked again %d time(s) though nothing changed", n-1)
}
channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true"
_ = r.a.reconcile(context.Background())
if n := len(r.asksSent(t)); n != 2 {
t.Errorf("not asked again once the channels changed: %d", n)
}
}
// After review: at most three asks open at once, the most urgent first, then the oldest.
func TestAtMostThreeAsksAreOpenTheMostUrgentFirst(t *testing.T) {
r := newAskerRig(t)
var open []conditions.Condition
for i := 0; i < 4; i++ {
c := unitsCondition()
c.Key = "machine.m" + string(rune('a'+i)) + ".units"
c.Actions = []conditions.Action{conditions.SilenceAction(c.Key)}
c.Raised = r.now.Add(-time.Duration(10-i) * time.Hour)
open = append(open, c)
}
urgent := heldCondition()
urgent.Severity, urgent.Raised = conditions.Urgent, r.now.Add(-time.Minute)
r.open = append(open, urgent)
_ = r.a.reconcile(context.Background())
sent := r.asksSent(t)
if len(sent) != askMostOpen || sent[0].About != urgent.Key || sent[1].About != "machine.ma.units" || sent[2].About != "machine.mb.units" {
var about []string
for _, q := range sent {
about = append(about, q.About)
}
t.Fatalf("asked %v", about)
}
}
// After review: nothing is asked while no router takes asks under the controller's name, and that is said once.
func TestNothingIsAskedWithoutARouter(t *testing.T) {
r := newAskerRig(t)
var said []string
r.a.logf = func(f string, a ...any) { said = append(said, f) }
r.a.routerHere = func(context.Context) (bool, error) { return false, nil }
r.open = []conditions.Condition{heldCondition()}
_ = r.a.reconcile(context.Background())
_ = r.a.reconcile(context.Background())
if len(r.asksSent(t)) != 0 {
t.Error("asked with no router")
}
n := 0
for _, s := range said {
if strings.Contains(s, "no router takes asks") {
n++
}
}
if n != 1 {
t.Errorf("said %d times", n)
}
}
// After review: the condition's words keep where an answer is given without a channel; the ask's text does not.
func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) {
c := heldCondition()
if !strings.Contains(c.Explanation, FromMeshMCPServer) {
t.Fatalf("the condition lost where it is answered: %q", c.Explanation)
}
q, _ := askOf("x", c, time.Now())
if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") {
t.Errorf("the ask says %q", q.Explanation)
}
if askApproveFor >= 24*time.Hour {
t.Errorf("an approving ask lasts %s, which the SDK may refuse at its bound", askApproveFor)
}
}
// After review (security finding 9): a warrant is acted on only for an ask open in the controller's own record,
// once the claim stands, and never when given after the ask expired.
func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) {
r := newAskerRig(t)
r.open = []conditions.Condition{heldCondition()}
_ = r.a.reconcile(context.Background())
w := r.warrantFor(t, heldCondition().Key, "Release")
late := w
late.At = r.store[w.Ask].Ask.Expires.Add(time.Minute)
body, _ := json.Marshal(late)
_ = r.a.Decided(context.Background(), body)
if len(r.called) != 0 {
t.Fatalf("acted on a warrant given after the ask expired: %v", r.called)
}
// Claimed already by another delivery: nothing done here.
kept := r.store[w.Ask]
kept.Acted = "acting"
r.store[w.Ask] = kept
body, _ = json.Marshal(w)
_ = r.a.Decided(context.Background(), body)
if len(r.called) != 0 {
t.Fatalf("acted though the claim was another's: %v", r.called)
}
// Cancelled in its own record: refused.
kept.Acted, kept.State = "", askCancelled
r.store[w.Ask] = kept
_ = r.a.Decided(context.Background(), body)
if len(r.called) != 0 {
t.Errorf("acted on a cancelled ask: %v", r.called)
}
}
// novox/hq ADR 0259 §6: a warrant authorises the act its option bound when the controller asked, and no
// other. A record of the act changed after the ask — another delivery, another machine, another argument —
// is refused and nothing is performed.
func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) {
for name, change := range map[string]func(*conditions.Action){
"another argument": func(a *conditions.Action) {
a.Arguments = map[string]string{"id": "novox/mesh-controller@000000000000"}
},
"another verb": func(a *conditions.Action) { a.Verb = "mesh-delivery.stop" },
"another machine": func(a *conditions.Action) { a.Machine = "anchor" },
} {
t.Run(name, func(t *testing.T) {
r := newAskerRig(t)
r.open = []conditions.Condition{heldCondition()}
_ = r.a.reconcile(context.Background())
w := r.warrantFor(t, heldCondition().Key, "Release")
kept := r.store[w.Ask]
acts := append([]conditions.Action(nil), kept.Actions...)
i := kept.Options[w.Option]
change(&acts[i])
kept.Actions = acts
r.store[w.Ask] = kept
answerWith(t, r, w)
if len(r.called)+len(r.acts) != 0 {
t.Errorf("performed an act the option did not bind: %v %v", r.called, r.acts)
}
})
}
// Every option of an ask binds its act.
q, _ := askOf("x", heldCondition(), time.Now())
for _, o := range q.Options {
if o.Binds == "" {
t.Errorf("the option %s binds nothing", o.ID)
}
}
}