1241 lines
52 KiB
Go
1241 lines
52 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs
|
|
// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level.
|
|
|
|
type memAskedStore map[string]asked
|
|
|
|
// memAskedMu guards every memAskedStore: Change is a compare-and-set as the bus's is.
|
|
var memAskedMu sync.Mutex
|
|
|
|
func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
r, ok := m[id]
|
|
if !ok {
|
|
return nil, nil
|
|
}
|
|
return &r, nil
|
|
}
|
|
func (m memAskedStore) Create(_ context.Context, r asked) error {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
if _, kept := m[r.ID]; kept {
|
|
return errors.New("an ask is kept under that id")
|
|
}
|
|
m[r.ID] = r
|
|
return nil
|
|
}
|
|
func (m memAskedStore) Change(_ context.Context, id string, change func(*asked) bool) (bool, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
r, ok := m[id]
|
|
if !ok || !change(&r) {
|
|
return false, nil
|
|
}
|
|
m[id] = r
|
|
return true, nil
|
|
}
|
|
func (m memAskedStore) All(context.Context) ([]asked, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
var out []asked
|
|
for _, r := range m {
|
|
out = append(out, r)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
type published struct {
|
|
subject, id string
|
|
body []byte
|
|
}
|
|
|
|
type askerRig struct {
|
|
a *asker
|
|
open []conditions.Condition
|
|
store memAskedStore
|
|
sent []published
|
|
called []string
|
|
silenced []string
|
|
acts []link.HandAct
|
|
now time.Time
|
|
// refuses, when set, is what every verb the asker calls answers; told is every notice the operator was told.
|
|
refuses error
|
|
told []told
|
|
}
|
|
|
|
// told is one plain notice the asker told the operator.
|
|
type told struct {
|
|
headline string
|
|
lines []string
|
|
}
|
|
|
|
func newAskerRig(t *testing.T) *askerRig {
|
|
r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)}
|
|
r.a = &asker{
|
|
open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil },
|
|
silence: func(_ context.Context, key string, d time.Duration, by, why string) error {
|
|
r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why)
|
|
// As the controller's conditions do (the confirmation review of 2026-10-09, M1): the condition is
|
|
// silenced from now on, so what is asked next sees it silenced.
|
|
for i := range r.open {
|
|
if r.open[i].Key == key {
|
|
r.open[i].Silenced = &conditions.Silence{Until: r.now.Add(d), By: by, Why: why, Since: r.now}
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
store: r.store,
|
|
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
|
r.sent = append(r.sent, published{subject, id, body})
|
|
return nil
|
|
},
|
|
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
|
raw, _ := json.Marshal(args)
|
|
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
|
return r.refuses
|
|
},
|
|
tell: func(_ context.Context, headline string, lines []string) error {
|
|
r.told = append(r.told, told{headline, lines})
|
|
return nil
|
|
},
|
|
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
|
now: func() time.Time { return r.now },
|
|
logf: t.Logf,
|
|
}
|
|
return r
|
|
}
|
|
|
|
func heldCondition() conditions.Condition {
|
|
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
|
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
|
|
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
|
}
|
|
|
|
func unitsCondition() conditions.Condition {
|
|
key := "machine.shanks.units"
|
|
return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning,
|
|
Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.",
|
|
Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}}
|
|
}
|
|
|
|
func (r *askerRig) asksSent(t *testing.T) []asks.Ask {
|
|
t.Helper()
|
|
var out []asks.Ask
|
|
for _, p := range r.sent {
|
|
if p.subject != asks.AskSubject("mesh-controller") {
|
|
continue
|
|
}
|
|
var q asks.Ask
|
|
if err := json.Unmarshal(p.body, &q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out = append(out, q)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"}
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 2 {
|
|
t.Fatalf("asked %d times: %+v", len(sent), sent)
|
|
}
|
|
byAbout := map[string]asks.Ask{}
|
|
for _, q := range sent {
|
|
byAbout[q.About] = q
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Errorf("%s: %v", q.About, err)
|
|
}
|
|
}
|
|
held := byAbout[heldCondition().Key]
|
|
if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve ||
|
|
held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator ||
|
|
held.OnExpiry == "" {
|
|
t.Errorf("the held delivery is asked %+v", held)
|
|
}
|
|
units := byAbout["machine.shanks.units"]
|
|
if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) {
|
|
t.Errorf("the failed units are asked %+v", units)
|
|
}
|
|
// No second ask while one is open.
|
|
r.now = r.now.Add(time.Minute)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Errorf("asked again while open: %d", n)
|
|
}
|
|
}
|
|
|
|
func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
// The units are silenced, the held delivery lasts past its ask's day.
|
|
units := unitsCondition()
|
|
units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)}
|
|
r.open = []conditions.Condition{heldCondition(), units}
|
|
r.now = r.now.Add(askApproveFor)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cancels int
|
|
for _, p := range r.sent {
|
|
if p.subject == asks.CancelSubject("mesh-controller") {
|
|
cancels++
|
|
}
|
|
}
|
|
if cancels != 1 {
|
|
t.Errorf("cancels %d, want the silenced one's", cancels)
|
|
}
|
|
if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key {
|
|
t.Errorf("the expired ask was not asked again: %+v", sent)
|
|
}
|
|
}
|
|
|
|
// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label.
|
|
func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant {
|
|
t.Helper()
|
|
for _, a := range r.store {
|
|
if a.Condition != condition || a.State != askOpen {
|
|
continue
|
|
}
|
|
for _, o := range a.Ask.Options {
|
|
if o.Label == label {
|
|
return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen,
|
|
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now,
|
|
AskDigest: a.Ask.Digest(),
|
|
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("no open ask about %s offers %s", condition, label)
|
|
return asks.Warrant{}
|
|
}
|
|
|
|
func answerWith(t *testing.T, r *askerRig, w asks.Warrant) {
|
|
t.Helper()
|
|
body, _ := json.Marshal(w)
|
|
if err := r.a.Decided(context.Background(), body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantIsActedOnOnce(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
answerWith(t, r, w)
|
|
answerWith(t, r, w) // heard again
|
|
if len(r.called) != 1 {
|
|
t.Fatalf("called %v", r.called)
|
|
}
|
|
want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}`
|
|
if r.called[0] != want {
|
|
t.Errorf("called\n %s\nwant\n %s", r.called[0], want)
|
|
}
|
|
if len(r.acts) != 1 {
|
|
t.Fatalf("hand-acts %+v", r.acts)
|
|
}
|
|
act := r.acts[0]
|
|
if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" ||
|
|
act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" ||
|
|
act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" {
|
|
t.Errorf("the hand-act %+v", act)
|
|
}
|
|
if !personsDecision(act) {
|
|
t.Error("an act on a warrant counts as a repair")
|
|
}
|
|
if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
|
t.Errorf("kept %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
|
|
for name, change := range map[string]func(*asks.Warrant){
|
|
"another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" },
|
|
"an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" },
|
|
"an option not offered": func(w *asks.Warrant) { w.Option = "delete" },
|
|
"another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge },
|
|
"no person": func(w *asks.Warrant) { w.By = nil },
|
|
"another ask's digest": func(w *asks.Warrant) { w.AskDigest = "sha256:0000" },
|
|
"no ask's digest": func(w *asks.Warrant) { w.AskDigest = "" },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
change(&w)
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts)+len(r.silenced) != 0 {
|
|
t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
|
|
plan := "plan-1791454185265004861"
|
|
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
|
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
|
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
|
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
|
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
|
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
|
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
|
for _, tc := range []struct {
|
|
c conditions.Condition
|
|
label string
|
|
want string
|
|
}{
|
|
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
|
|
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
|
|
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
|
|
} {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{tc.c}
|
|
_ = r.a.reconcile(context.Background())
|
|
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
|
|
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, "machine.shanks.units", "Silence for a week")
|
|
w.Level, w.Proofs = asks.Acknowledge, nil
|
|
w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14",
|
|
Verified: "a desk click: whoever was at the operator's session on g14"}
|
|
w.Channel = "desk-channel"
|
|
answerWith(t, r, w)
|
|
if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") {
|
|
t.Fatalf("silenced %v, called %v", r.silenced, r.called)
|
|
}
|
|
if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 {
|
|
t.Errorf("%+v", r.acts)
|
|
}
|
|
}
|
|
|
|
func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time"
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) ||
|
|
!strings.HasPrefix(r.store[w.Ask].Acted, "nothing") {
|
|
t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask])
|
|
}
|
|
// And a choice for a condition that ended meanwhile does nothing either.
|
|
r2 := newAskerRig(t)
|
|
r2.open = []conditions.Condition{heldCondition()}
|
|
_ = r2.a.reconcile(context.Background())
|
|
w2 := r2.warrantFor(t, heldCondition().Key, "Release")
|
|
r2.open = nil
|
|
answerWith(t, r2, w2)
|
|
if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" {
|
|
t.Errorf("%v %+v", r2.called, r2.store[w2.Ask])
|
|
}
|
|
}
|
|
|
|
func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) {
|
|
if id != w.Ask {
|
|
return nil, errors.New("another ask")
|
|
}
|
|
return &w, nil
|
|
}
|
|
r.now = r.now.Add(askCatchUpAfter)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") {
|
|
t.Errorf("called %v", r.called)
|
|
}
|
|
if n := len(r.asksSent(t)); n != 1 {
|
|
t.Errorf("asked again after the answer: %d", n)
|
|
}
|
|
}
|
|
|
|
// After review (2026-10-08): a refused ask is not asked again at once; issue 369: nor is the refusal believed
|
|
// for ever — it is asked again after a wait that doubles with each refusal in a row, and at once when the
|
|
// channels change.
|
|
func TestAnAskTheRouterRefusedIsAskedAgainAfterAGrowingWait(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
|
r.a.channels = func(context.Context) string { return channels }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
refuse := func() {
|
|
t.Helper()
|
|
sent := r.asksSent(t)
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
|
Outcome: asks.OutcomeRefused, Words: "no channel can carry any of its answers now", At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
first := r.asksSent(t)[0]
|
|
refuse()
|
|
if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") {
|
|
t.Fatalf("the refusal was kept as %+v", got)
|
|
}
|
|
// Each wait: nothing asked before it ends, asked once when it has.
|
|
for i, wait := range []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute,
|
|
16 * time.Minute, 30 * time.Minute, 30 * time.Minute} {
|
|
before := len(r.asksSent(t))
|
|
r.now = r.now.Add(wait - 10*time.Second)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != before {
|
|
t.Fatalf("refusal %d: asked again before %s", i+1, wait)
|
|
}
|
|
r.now = r.now.Add(10 * time.Second)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != before+1 {
|
|
t.Fatalf("refusal %d: not asked again after %s (%d asks)", i+1, wait, n)
|
|
}
|
|
refuse()
|
|
}
|
|
before := len(r.asksSent(t))
|
|
channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true"
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != before+1 {
|
|
t.Errorf("not asked again once the channels changed: %d", n-before)
|
|
}
|
|
}
|
|
|
|
// After review: at most three asks open at once, the most urgent first, then the oldest.
|
|
func TestAtMostThreeAsksAreOpenTheMostUrgentFirst(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var open []conditions.Condition
|
|
for i := 0; i < 4; i++ {
|
|
c := unitsCondition()
|
|
c.Key = "machine.m" + string(rune('a'+i)) + ".units"
|
|
c.Actions = []conditions.Action{conditions.SilenceAction(c.Key)}
|
|
c.Raised = r.now.Add(-time.Duration(10-i) * time.Hour)
|
|
open = append(open, c)
|
|
}
|
|
urgent := heldCondition()
|
|
urgent.Severity, urgent.Raised = conditions.Urgent, r.now.Add(-time.Minute)
|
|
r.open = append(open, urgent)
|
|
_ = r.a.reconcile(context.Background())
|
|
sent := r.asksSent(t)
|
|
if len(sent) != askMostOpen || sent[0].About != urgent.Key || sent[1].About != "machine.ma.units" || sent[2].About != "machine.mb.units" {
|
|
var about []string
|
|
for _, q := range sent {
|
|
about = append(about, q.About)
|
|
}
|
|
t.Fatalf("asked %v", about)
|
|
}
|
|
}
|
|
|
|
// After review: nothing is asked while no router takes asks under the controller's name, and that is said once.
|
|
func TestNothingIsAskedWithoutARouter(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var said []string
|
|
r.a.logf = func(f string, a ...any) { said = append(said, f) }
|
|
r.a.routerHere = func(context.Context) (bool, error) { return false, nil }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
_ = r.a.reconcile(context.Background())
|
|
if len(r.asksSent(t)) != 0 {
|
|
t.Error("asked with no router")
|
|
}
|
|
n := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, "no router takes asks") {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("said %d times", n)
|
|
}
|
|
}
|
|
|
|
// After review: the condition's words keep where an answer is given without a channel; the ask's text does not.
|
|
func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) {
|
|
c := heldCondition()
|
|
if !strings.Contains(c.Explanation, FromMeshMCPServer) {
|
|
t.Fatalf("the condition lost where it is answered: %q", c.Explanation)
|
|
}
|
|
q, _ := askOf("x", c, partsOf(c)[0], time.Now())
|
|
if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") {
|
|
t.Errorf("the ask says %q", q.Explanation)
|
|
}
|
|
if askApproveFor >= 24*time.Hour {
|
|
t.Errorf("an approving ask lasts %s, which the SDK may refuse at its bound", askApproveFor)
|
|
}
|
|
}
|
|
|
|
// After review (security finding 9): a warrant is acted on only for an ask open in the controller's own record,
|
|
// once the claim stands, and never when given after the ask expired.
|
|
func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
late := w
|
|
late.At = r.store[w.Ask].Ask.Expires.Add(time.Minute)
|
|
body, _ := json.Marshal(late)
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Fatalf("acted on a warrant given after the ask expired: %v", r.called)
|
|
}
|
|
// Claimed already by another delivery: nothing done here.
|
|
kept := r.store[w.Ask]
|
|
kept.Acted = "acting"
|
|
r.store[w.Ask] = kept
|
|
body, _ = json.Marshal(w)
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Fatalf("acted though the claim was another's: %v", r.called)
|
|
}
|
|
// Cancelled in its own record: refused.
|
|
kept.Acted, kept.State = "", askCancelled
|
|
r.store[w.Ask] = kept
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Errorf("acted on a cancelled ask: %v", r.called)
|
|
}
|
|
}
|
|
|
|
// novox/hq ADR 0259 §6: a warrant authorises the act its option bound when the controller asked, and no
|
|
// other. A record of the act changed after the ask — another delivery, another machine, another argument —
|
|
// is refused and nothing is performed.
|
|
func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) {
|
|
for name, change := range map[string]func(*conditions.Action){
|
|
"another argument": func(a *conditions.Action) {
|
|
a.Arguments = map[string]string{"id": "novox/mesh-controller@000000000000"}
|
|
},
|
|
"another verb": func(a *conditions.Action) { a.Verb = "mesh-delivery.stop" },
|
|
"another machine": func(a *conditions.Action) { a.Machine = "anchor" },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
kept := r.store[w.Ask]
|
|
acts := append([]conditions.Action(nil), kept.Actions...)
|
|
i := kept.Options[w.Option]
|
|
change(&acts[i])
|
|
kept.Actions = acts
|
|
r.store[w.Ask] = kept
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts) != 0 {
|
|
t.Errorf("performed an act the option did not bind: %v %v", r.called, r.acts)
|
|
}
|
|
})
|
|
}
|
|
// Every option of an ask binds its act.
|
|
q, _ := askOf("x", heldCondition(), partsOf(heldCondition())[0], time.Now())
|
|
for _, o := range q.Options {
|
|
if o.Binds == "" {
|
|
t.Errorf("the option %s binds nothing", o.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Failure is loud (novox/hq ADR 0259, the self-review of 2026-10-09): a condition that needs the operator and
|
|
// could not be asked on any channel — no router, or the router refused the ask — is a condition of its own,
|
|
// cleared once it can be asked again.
|
|
func TestAnAskThatCannotBeDeliveredIsSaid(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
|
routerHere := false
|
|
r.a.routerHere = func(context.Context) (bool, error) { return routerHere, nil }
|
|
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
|
r.a.channels = func(context.Context) string { return channels }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || got[0].Kind != "asks-undelivered" ||
|
|
!strings.Contains(got[0].Summary, heldCondition().Key) || !strings.Contains(got[0].Summary, "no router") {
|
|
t.Fatalf("no router, said as %+v", got)
|
|
}
|
|
|
|
routerHere = true
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("asked, and still said undelivered: %+v", got)
|
|
}
|
|
first := r.asksSent(t)[0]
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused,
|
|
Words: "no channel can carry any of its answers now", At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no channel can carry") {
|
|
t.Fatalf("the router's refusal, said as %+v", got)
|
|
}
|
|
if why, ok := conditions.PlainWords(conditions.Words{Headline: last()[0].Headline, Explanation: last()[0].Explanation,
|
|
Needs: last()[0].Needs, Resolved: last()[0].Resolved}, ""); !ok {
|
|
t.Errorf("not plain: %s", why)
|
|
}
|
|
r.open = nil
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Errorf("nothing needs asking, and still said: %+v", got)
|
|
}
|
|
}
|
|
|
|
// The review of 2026-10-09 (M1): an acknowledging answer never shares an ask with an authorising one. A
|
|
// condition offering Restart and Silence is asked twice — Restart alone, about the condition, and Silence
|
|
// alone, apart — so Silence chosen on a channel that only acknowledges leaves the Restart ask open.
|
|
func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
key := "module.shanks.plex.down"
|
|
c := conditions.Condition{Key: key, Kind: "module-down", Severity: conditions.Urgent, Headline: "Plex down on shanks",
|
|
Explanation: "Needs you: restart it, or silence this.", Needs: "restart it, or silence this.",
|
|
Actions: []conditions.Action{
|
|
{Label: "Restart", Verb: "node-service-manager.restart", Machine: "shanks", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "plex"}},
|
|
conditions.SilenceAction(key)}}
|
|
r.open = []conditions.Condition{c}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 2 {
|
|
t.Fatalf("asked %d time(s): %+v", len(sent), sent)
|
|
}
|
|
for _, q := range sent {
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Errorf("%s: %v", q.About, err)
|
|
}
|
|
levels := map[asks.Level]bool{}
|
|
for _, o := range q.Options {
|
|
levels[o.Level] = true
|
|
}
|
|
if len(levels) != 1 {
|
|
t.Errorf("the ask about %s mixes levels: %+v", q.About, q.Options)
|
|
}
|
|
}
|
|
byAbout := map[string]asks.Ask{}
|
|
for _, q := range sent {
|
|
byAbout[q.About] = q
|
|
}
|
|
if q := byAbout[key]; len(q.Options) != 1 || q.Options[0].Label != "Restart" {
|
|
t.Errorf("the condition's own ask: %+v", q)
|
|
}
|
|
if q := byAbout[key+".acknowledge"]; len(q.Options) != 1 || q.Options[0].Level != asks.Acknowledge {
|
|
t.Errorf("the acknowledging ask: %+v", q)
|
|
}
|
|
// Silence chosen: performed, and the Restart ask stays open, never asked twice.
|
|
answerWith(t, r, r.warrantFor(t, key, "Silence for a week"))
|
|
if len(r.silenced) != 1 || len(r.called) != 0 {
|
|
t.Fatalf("silenced %v called %v", r.silenced, r.called)
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
open := 0
|
|
for _, a := range r.store {
|
|
if a.State == askOpen && a.Condition == key {
|
|
open++
|
|
if a.Part != "" || a.Ask.Options[0].Label != "Restart" {
|
|
t.Errorf("the open ask is %+v", a)
|
|
}
|
|
}
|
|
}
|
|
if open != 1 || len(r.asksSent(t)) != 2 {
|
|
t.Errorf("after the silence: %d open, %d asked", open, len(r.asksSent(t)))
|
|
}
|
|
// And the approval still answers: Restart chosen on a channel that proves who answered is performed.
|
|
answerWith(t, r, r.warrantFor(t, key, "Restart"))
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "node-service-manager.restart@shanks") {
|
|
t.Errorf("the approval kept through a silence was not performed: %v", r.called)
|
|
}
|
|
}
|
|
|
|
// novox/hq issue 353: the controller's grant to ask is composed from the router's assignment and reaches the
|
|
// bus only when its machine is pushed. Between the two, the bus refuses every ask (measured 2026-10-09, 17:54 to
|
|
// 17:56 local: seven refusals of mesh.seat.operator-channel.accept.ask.mesh-controller). So nothing is asked
|
|
// while the bus's user list is behind, it is said once, the conditions that need the operator are raised as
|
|
// undelivered with what to do, and the asks go out once the bus holds the grant.
|
|
func TestNothingIsAskedWhileTheBusLacksTheControllersGrant(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var said []string
|
|
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
held := false
|
|
r.a.grantHeld = func(context.Context) (bool, string, error) {
|
|
return held, "the bus's user list on anchor is behind what the mesh composes; `push anchor` carries it", nil
|
|
}
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
_ = r.a.reconcile(context.Background())
|
|
if len(r.asksSent(t)) != 0 {
|
|
t.Error("asked while the bus lacks the grant")
|
|
}
|
|
n := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, "does not hold the controller's grant") {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("said %d times: %q", n, said)
|
|
}
|
|
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
|
!strings.Contains(raised[len(raised)-1][0].Summary, "`push anchor` carries it") ||
|
|
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
|
t.Fatalf("not said as a condition with what to do: %+v", raised)
|
|
}
|
|
held = true
|
|
_ = r.a.reconcile(context.Background())
|
|
if sent := r.asksSent(t); len(sent) != 1 || sent[0].About != heldCondition().Key {
|
|
t.Errorf("not asked once the bus holds the grant: %+v", sent)
|
|
}
|
|
if last := raised[len(raised)-1]; len(last) != 0 {
|
|
t.Errorf("the undelivered condition was not cleared: %+v", last)
|
|
}
|
|
}
|
|
|
|
// novox/hq issue 369: the router refused while its channels had not yet said they could send; they could twenty
|
|
// minutes later, and "Questions for you not delivered" repeated that refusal for eleven hours, escalating on it.
|
|
// The ask is made again; while the router's word on it is awaited the condition stands unchanged (neither
|
|
// cleared nor raised again); once the router took it, the condition clears; a new refusal is said in its own
|
|
// words.
|
|
func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
|
r.a.channels = func(context.Context) string { return "channel/telegram=telegram@anchor[choice]own:true" }
|
|
r.open = []conditions.Condition{unitsCondition()}
|
|
refuse := func(words string) {
|
|
t.Helper()
|
|
sent := r.asksSent(t)
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
|
Outcome: asks.OutcomeRefused, Words: words, At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
refuse("no channel can carry any of its answers now: telegram: telegram has not said whether it can send")
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "telegram has not said") {
|
|
t.Fatalf("the refusal, said as %+v", got)
|
|
}
|
|
// The wait ends: asked again; until the router's word on it is in, the condition stands as it was.
|
|
r.now = r.now.Add(askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Fatalf("not asked again: %d asks", n)
|
|
}
|
|
if got := last(); len(got) != 1 {
|
|
t.Fatalf("cleared while the router's word on the new ask was awaited: the condition would flap")
|
|
}
|
|
r.now = r.now.Add(askVerdictWait / 2)
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 {
|
|
t.Fatalf("cleared inside the verdict wait: the condition would flap")
|
|
}
|
|
// Refused again, now for a reason of today: said in those words.
|
|
refuse("no channel can carry any of its answers now: telegram: no account is linked")
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no account is linked") ||
|
|
strings.Contains(got[0].Summary, "has not said") {
|
|
t.Fatalf("an old refusal's words repeated: %+v", got)
|
|
}
|
|
// Asked again after the doubled wait, and taken: no refusal comes, and the condition clears.
|
|
r.now = r.now.Add(2 * askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 3 {
|
|
t.Fatalf("not asked again after the doubled wait: %d asks", n)
|
|
}
|
|
r.now = r.now.Add(askVerdictWait)
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("taken, and still said undelivered: %+v", got)
|
|
}
|
|
if n := len(r.asksSent(t)); n != 3 {
|
|
t.Fatalf("an ask taken was asked again: %d asks", n)
|
|
}
|
|
// The operator answers it; the condition stays open (its fix takes a while): the old refusal is not said again.
|
|
taken := r.asksSent(t)[2]
|
|
r.store.Change(context.Background(), taken.ID, func(x *asked) bool {
|
|
x.State, x.Ended = string(asks.OutcomeChosen), r.now
|
|
return true
|
|
})
|
|
for i := 0; i < 5; i++ {
|
|
r.now = r.now.Add(askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("an answered ask brought its old refusal back: %+v", got)
|
|
}
|
|
}
|
|
// Its explanation opens with one verdict, not two.
|
|
r.open = append(r.open, heldCondition())
|
|
_ = r.a.reconcile(context.Background())
|
|
refuse("no channel can carry any of its answers now")
|
|
_ = r.a.reconcile(context.Background())
|
|
if e := conditions.Verdict(last()[0].Needs, last()[0].Explanation); strings.Count(e, "Needs you") != 1 {
|
|
t.Errorf("the explanation says its verdict twice: %q", e)
|
|
}
|
|
}
|
|
|
|
// hq issue 353, the review: the bus holds the controller's grant to ask only when both facts hold — the mesh
|
|
// composes the controller's own grant, and the bus's module says it is in force on the running server — and
|
|
// each fact not known is not held, said with why.
|
|
func TestTheGrantIsHeldOnlyWhenComposedAndInForceOnTheBus(t *testing.T) {
|
|
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
|
no := func(why string) grantJudge {
|
|
return func(context.Context) (bool, string, error) { return false, why, nil }
|
|
}
|
|
broken := func(context.Context) (bool, string, error) { return false, "", errors.New("the store is away") }
|
|
for _, c := range []struct {
|
|
name string
|
|
composed, inForce grantJudge
|
|
held bool
|
|
says string
|
|
}{
|
|
{"composed and in force", yes, yes, true, ""},
|
|
{"not composed", no("no router is assigned"), yes, false, "no router is assigned"},
|
|
{"composed, written, not reloaded", yes, no("the server has not reloaded"), false, "has not reloaded"},
|
|
{"the composition unknown", broken, yes, false, "could not be worked out: the store is away"},
|
|
{"the bus unknown", yes, broken, false, "could not be worked out: the store is away"},
|
|
{"nothing judges the bus", yes, nil, false, "not judged"},
|
|
} {
|
|
held, why, err := grantHeldBy(c.composed, c.inForce, time.Now)(context.Background())
|
|
if err != nil || held != c.held || !strings.Contains(why, c.says) {
|
|
t.Errorf("%s: held %v (%q, %v)", c.name, held, why, err)
|
|
}
|
|
}
|
|
// Remembered for grantLookEvery, then judged again.
|
|
now := time.Date(2026, 10, 9, 18, 0, 0, 0, time.UTC)
|
|
inForce := false
|
|
judge := grantHeldBy(yes, func(context.Context) (bool, string, error) { return inForce, "not yet", nil },
|
|
func() time.Time { return now })
|
|
if held, _, _ := judge(context.Background()); held {
|
|
t.Fatal("held before the bus enforces it")
|
|
}
|
|
inForce = true
|
|
if held, _, _ := judge(context.Background()); held {
|
|
t.Error("judged again inside grantLookEvery")
|
|
}
|
|
now = now.Add(grantLookEvery)
|
|
if held, _, _ := judge(context.Background()); !held {
|
|
t.Error("not judged again after grantLookEvery")
|
|
}
|
|
}
|
|
|
|
// The controller's own grant, as composed: present while a router that takes asks is assigned, absent
|
|
// otherwise — whatever else in the user list changes.
|
|
func TestTheControllersOwnGrantToAskIsReadFromTheComposition(t *testing.T) {
|
|
router := broker.Declared{Module: "messenger", Holds: []broker.Seat{{Name: broker.AsksSeat, Scope: "mesh",
|
|
Accepts: []string{"ask", "cancel"}, Emits: []string{"decided"}, ByCaller: []string{"ask", "cancel", "decided"}}}}
|
|
other := broker.Declared{Module: "dunst"}
|
|
with := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {router, other}}}
|
|
without := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {other}}}
|
|
if may, err := controllerMayAsk(with); err != nil || !may {
|
|
t.Errorf("with a router assigned: %v %v", may, err)
|
|
}
|
|
if may, err := controllerMayAsk(without); err != nil || may {
|
|
t.Errorf("with no router assigned: %v %v", may, err)
|
|
}
|
|
}
|
|
|
|
// The bus module's answer, with the record's word on the list sent as the check only where the answer does
|
|
// not settle it (reviews 22 and 31 of PR 188); each way it is not held names its own remedy.
|
|
func TestTheBusModulesAnswerNamesTheRemedyForEachCase(t *testing.T) {
|
|
const (
|
|
reloaded = `{"allowed":true,"in_force":true,"file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:56:05Z"}`
|
|
notLoaded = `{"allowed":true,"in_force":false,"why":"the user list on the machine allows it, and the server has not reloaded since it was written","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`
|
|
noVarz = `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`
|
|
noStat = `{"allowed":true,"in_force":false,"why":"when the bus's user list was written could not be read inside mesh-broker-nats: exit 1"}`
|
|
notAllowed = `{"allowed":false,"in_force":false,"why":"the user list on the machine does not allow it"}`
|
|
)
|
|
current := busListState{Machine: "anchor"}
|
|
behind := busListState{Machine: "anchor", Behind: true}
|
|
stepped := busListState{Machine: "anchor", Behind: true, BusStep: "sending anchor would replace the bus; `bus upgrade --why …`"}
|
|
uncomposed := busListState{Machine: "anchor", Behind: true, Uncomposed: errors.New("module foo requires a provision nobody provides")}
|
|
for _, c := range []struct {
|
|
name, answer string
|
|
askErr error
|
|
list busListState
|
|
listErr error
|
|
held bool
|
|
says, not string
|
|
}{
|
|
{"in force", reloaded, nil, behind, nil, true, "", "push"},
|
|
{"written, not reloaded", notLoaded, nil, current, nil, false, "within about 5 s", "push"},
|
|
{"in force not known: varz", noVarz, nil, current, nil, false, "is not known: the server's last configuration load", "wait"},
|
|
{"in force not known: stat", noStat, nil, current, nil, false, "is not known: when the bus's user list was written", "wait"},
|
|
{"not allowed, list behind", notAllowed, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
|
{"not allowed, bus step pending", notAllowed, nil, stepped, nil, false, "`bus upgrade", "`push anchor`"},
|
|
{"not allowed, list uncomposed", notAllowed, nil, uncomposed, nil, false, "`plan anchor`", "`push anchor`"},
|
|
{"older module, allowed, list current", `{"allowed":true}`, nil, current, nil, true, "not confirmed", "push"},
|
|
{"older module, allowed, list behind", `{"allowed":true}`, nil, behind, nil, true, "not confirmed", "push"},
|
|
{"older module, allowed, list unknown", `{"allowed":true}`, nil, current, errors.New("the store is away"), true, "`bus upgrade`", "push"},
|
|
{"older module, not allowed, list behind", `{"allowed":false}`, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
|
{"module cannot answer, list current", "", errors.New("no responders"), current, nil, true, "could not answer", "push"},
|
|
{"module cannot answer, list behind", "", errors.New("no responders"), behind, nil, false, "`push anchor` carries it", ""},
|
|
{"module cannot answer, list unknown", "", errors.New("no responders"), current, errors.New("the store is away"), false, "could not be worked out", ""},
|
|
{"answer unreadable, list behind", `not json`, nil, behind, nil, false, "could not be read", ""},
|
|
} {
|
|
judge := busGrantBy(
|
|
func(context.Context) (string, json.RawMessage, error) {
|
|
return "anchor", json.RawMessage(c.answer), c.askErr
|
|
},
|
|
func(context.Context) (busListState, error) { return c.list, c.listErr })
|
|
held, why, _ := judgeGrantOnce(judge)
|
|
if held != c.held || !strings.Contains(why, c.says) || (c.not != "" && strings.Contains(why, c.not)) {
|
|
t.Errorf("%s: held %v (%q)", c.name, held, why)
|
|
}
|
|
}
|
|
none := busGrantBy(func(context.Context) (string, json.RawMessage, error) { return "", nil, nil }, nil)
|
|
if held, why, _ := judgeGrantOnce(none); held || !strings.Contains(why, "no module holds the bus") {
|
|
t.Errorf("no bus: held %v (%q)", held, why)
|
|
}
|
|
}
|
|
|
|
// judgeGrantOnce is the whole judgement with the composed grant held, as the asker makes it.
|
|
func judgeGrantOnce(bus grantJudge) (bool, string, error) {
|
|
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
|
return grantHeldBy(yes, bus, time.Now)(context.Background())
|
|
}
|
|
|
|
// The orders reviews 22 and 31 of PR 188 found: this controller live before the bus module that answers
|
|
// in_force, with the bus's machine behind on its list — and, after mesh-catalog #166 merged, unpushable until a
|
|
// person's bus upgrade. An older module that allows the ask asks (once, with the caveat said); a new module that
|
|
// has not reloaded names the wait, not a push; one that cannot say whether it reloaded names what is not known.
|
|
func TestAnOlderBusModuleNeitherSilencesTheAsksNorNamesTheWrongRemedy(t *testing.T) {
|
|
for _, c := range []struct {
|
|
name, answer string
|
|
list busListState
|
|
asks bool
|
|
says, not string
|
|
}{
|
|
{"older module, list current", `{"allowed":true}`, busListState{Machine: "anchor"}, true, "not confirmed", ""},
|
|
{"older module, list behind, bus step pending", `{"allowed":true}`,
|
|
busListState{Machine: "anchor", Behind: true, BusStep: "a planned step"}, true, "not confirmed", ""},
|
|
{"new module, not reloaded",
|
|
`{"allowed":true,"in_force":false,"why":"not reloaded","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`,
|
|
busListState{Machine: "anchor"}, false, "within about 5 s", "push"},
|
|
{"new module, reload not known", `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`,
|
|
busListState{Machine: "anchor"}, false, "is not known", "wait"},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var said []string
|
|
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
|
r.a.grantHeld = grantHeldBy(yes, busGrantBy(
|
|
func(context.Context) (string, json.RawMessage, error) {
|
|
return "anchor", json.RawMessage(c.answer), nil
|
|
},
|
|
func(context.Context) (busListState, error) { return c.list, nil }),
|
|
func() time.Time { return r.now })
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
_ = r.a.reconcile(context.Background())
|
|
sent := r.asksSent(t)
|
|
if c.asks {
|
|
if len(sent) != 1 || sent[0].About != heldCondition().Key {
|
|
t.Fatalf("not asked: %+v", sent)
|
|
}
|
|
n := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, c.says) {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("the caveat said %d times: %q", n, said)
|
|
}
|
|
return
|
|
}
|
|
if len(sent) != 0 {
|
|
t.Fatalf("asked while the bus does not hold the grant: %+v", sent)
|
|
}
|
|
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
|
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
|
t.Fatalf("not said as undelivered: %+v", raised)
|
|
}
|
|
summary := raised[len(raised)-1][0].Summary
|
|
if !strings.Contains(summary, c.says) || (c.not != "" && strings.Contains(summary, c.not)) {
|
|
t.Errorf("the condition names the wrong remedy: %q", summary)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The serving asker is wired to the real judgement: with records and a bus it cannot read, the grant is not
|
|
// held, and so nothing is asked (a stub saying yes, or no judgement at all, would ask).
|
|
func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
|
a := servingAsker(&stores{}, nil, nil, nil)
|
|
if a.grantHeld == nil {
|
|
t.Fatal("the serving asker does not judge the bus's grant")
|
|
}
|
|
held, why, err := a.grantHeld(context.Background())
|
|
if err != nil || held || !strings.Contains(why, "could not be worked out") {
|
|
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
|
}
|
|
}
|
|
|
|
// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it.
|
|
func failedPlanCondition(plan string) conditions.Condition {
|
|
p := aFailedPlan(plan, nil)
|
|
f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC))
|
|
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
|
o := watchPlansFailed(f)[0]
|
|
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
|
}
|
|
|
|
// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again".
|
|
func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
c := failedPlanCondition("plan-1792433168676870614")
|
|
r.open = []conditions.Condition{c}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 1 || len(sent[0].Options) != 1 {
|
|
t.Fatalf("asked %+v", sent)
|
|
}
|
|
o := sent[0].Options[0]
|
|
if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" {
|
|
t.Errorf("the option is %+v", o)
|
|
}
|
|
if err := sent[0].Check(r.now); err != nil {
|
|
t.Errorf("the ask is refused as made: %v", err)
|
|
}
|
|
if strings.Contains(sent[0].Explanation, "mesh MCP server") {
|
|
t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation)
|
|
}
|
|
}
|
|
|
|
// 1315/AC2: the warrant calls plans with retry=<plan> and nothing else, once, and is recorded as the hand act;
|
|
// the same warrant heard again does nothing.
|
|
func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) {
|
|
plan := "plan-1792433168676870614"
|
|
r := newAskerRig(t)
|
|
c := failedPlanCondition(plan)
|
|
r.open = []conditions.Condition{c}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, c.Key, "Retry")
|
|
answerWith(t, r, w)
|
|
answerWith(t, r, w)
|
|
if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want {
|
|
t.Fatalf("called %v, want exactly %s", r.called, want)
|
|
}
|
|
if len(r.acts) != 1 {
|
|
t.Fatalf("hand-acts %+v", r.acts)
|
|
}
|
|
act := r.acts[0]
|
|
if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan ||
|
|
act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer {
|
|
t.Errorf("the hand-act is %+v", act)
|
|
}
|
|
if got := r.store[w.Ask]; got.Acted != "done" {
|
|
t.Errorf("kept %+v", got)
|
|
}
|
|
}
|
|
|
|
// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is
|
|
// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before.
|
|
func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) {
|
|
plan := "plan-1792433168676870614"
|
|
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
|
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
|
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
|
for _, tc := range []struct {
|
|
name string
|
|
c conditions.Condition
|
|
label string
|
|
}{
|
|
{"retry", failedPlanCondition(plan), "Retry"},
|
|
{"stop", waiting, "Stop"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"}
|
|
r.open = []conditions.Condition{tc.c}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, tc.c.Key, tc.label)
|
|
answerWith(t, r, w)
|
|
answerWith(t, r, w) // heard again: told once
|
|
if len(r.told) != 1 {
|
|
t.Fatalf("told %+v", r.told)
|
|
}
|
|
got := r.told[0]
|
|
if got.headline != tc.c.Headline {
|
|
t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline)
|
|
}
|
|
said := strings.Join(got.lines, " ")
|
|
if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) {
|
|
t.Errorf("the lines do not carry the answer and the refusal's words: %q", said)
|
|
}
|
|
if why, ok := conditions.Plain(said); !ok {
|
|
t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said)
|
|
}
|
|
if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") {
|
|
t.Errorf("the failure is not recorded as before: %+v", r.acts)
|
|
}
|
|
if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") {
|
|
t.Errorf("kept %+v", kept)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here.
|
|
func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
c := failedPlanCondition("plan-1792433168676870614")
|
|
r.open = []conditions.Condition{c}
|
|
_ = r.a.reconcile(context.Background())
|
|
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
|
if len(r.told) != 0 {
|
|
t.Fatalf("a retry that was done told %+v", r.told)
|
|
}
|
|
|
|
r = newAskerRig(t)
|
|
r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"}
|
|
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
|
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
|
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
|
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
|
r.open = []conditions.Condition{module}
|
|
_ = r.a.reconcile(context.Background())
|
|
answerWith(t, r, r.warrantFor(t, module.Key, "Restart"))
|
|
if len(r.told) != 0 || len(r.acts) != 1 {
|
|
t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts)
|
|
}
|
|
}
|
|
|
|
// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was
|
|
// changed"; and the sentence a refusal falls back to fits the answer given.
|
|
func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) {
|
|
plan := "plan-1792433168676870614"
|
|
r := newAskerRig(t)
|
|
r.refuses = errors.New("mesh-controller did not answer plans within 30s")
|
|
c := failedPlanCondition(plan)
|
|
r.open = []conditions.Condition{c}
|
|
_ = r.a.reconcile(context.Background())
|
|
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
|
if len(r.told) != 1 {
|
|
t.Fatalf("told %+v", r.told)
|
|
}
|
|
said := strings.Join(r.told[0].lines, " ")
|
|
if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") {
|
|
t.Errorf("a call that may have run is told as %q", said)
|
|
}
|
|
if why, ok := conditions.Plain(said); !ok {
|
|
t.Errorf("not plain: %s: %q", why, said)
|
|
}
|
|
}
|
|
|
|
func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) {
|
|
plan := "plan-1792433168676870614"
|
|
for _, tc := range []struct {
|
|
act conditions.Action
|
|
want string
|
|
}{
|
|
{retryAction(plan), "a retry would not run now"},
|
|
{conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"},
|
|
{conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"},
|
|
} {
|
|
// Words the operator's channel withholds: a hash and a dotted name.
|
|
got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act)
|
|
if !strings.Contains(got, tc.want) {
|
|
t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want)
|
|
}
|
|
if _, ok := conditions.Plain(got); !ok {
|
|
t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the
|
|
// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not.
|
|
func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
c := failedPlanCondition("plan-1792433168676870614")
|
|
c.Raised = r.now.Add(-time.Minute)
|
|
r.open = []conditions.Condition{c}
|
|
_ = r.a.reconcile(context.Background())
|
|
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
|
// Still open right after the answer: not asked again at once.
|
|
r.now = r.now.Add(time.Minute)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 1 {
|
|
t.Fatalf("asked again while the same condition stayed open: %d", n)
|
|
}
|
|
// The retry ran, the plan left failed and the condition ended.
|
|
r.now = r.now.Add(time.Minute)
|
|
r.open = nil
|
|
_ = r.a.reconcile(context.Background())
|
|
// It fails again, ten minutes after the answer: a condition raised after the answered ask ended.
|
|
r.now = r.now.Add(8 * time.Minute)
|
|
again := failedPlanCondition("plan-1792433168676870614")
|
|
again.Raised = r.now
|
|
r.open = []conditions.Condition{again}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Fatalf("the second failure was asked %d times in all, want 2", n)
|
|
}
|
|
if len(r.called) != 1 {
|
|
t.Fatalf("something retried without an answer: %v", r.called)
|
|
}
|
|
}
|