jochen
f996cf7b6b
After mesh-catalog #166 merges and before the bus upgrade, the bus's machine cannot be pushed, so any change to the composed list made the list check read behind for good: every ask was withheld and the condition named a push that is refused. An older module's allowed now counts as held, the unconfirmed reload said once; the worst case is one ask refused in the seconds before a reload, which is asked again (issue 369). The list check stays for a module that says not allowed or cannot answer, and its remedy is the plan's own reason when the list cannot be composed, the bus upgrade while sends there wait on it, and a push otherwise. An in_force false is a wait only when the module gives both times; otherwise what is not known is named.
1016 lines
42 KiB
Go
1016 lines
42 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs
|
|
// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level.
|
|
|
|
type memAskedStore map[string]asked
|
|
|
|
// memAskedMu guards every memAskedStore: Change is a compare-and-set as the bus's is.
|
|
var memAskedMu sync.Mutex
|
|
|
|
func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
r, ok := m[id]
|
|
if !ok {
|
|
return nil, nil
|
|
}
|
|
return &r, nil
|
|
}
|
|
func (m memAskedStore) Create(_ context.Context, r asked) error {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
if _, kept := m[r.ID]; kept {
|
|
return errors.New("an ask is kept under that id")
|
|
}
|
|
m[r.ID] = r
|
|
return nil
|
|
}
|
|
func (m memAskedStore) Change(_ context.Context, id string, change func(*asked) bool) (bool, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
r, ok := m[id]
|
|
if !ok || !change(&r) {
|
|
return false, nil
|
|
}
|
|
m[id] = r
|
|
return true, nil
|
|
}
|
|
func (m memAskedStore) All(context.Context) ([]asked, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
var out []asked
|
|
for _, r := range m {
|
|
out = append(out, r)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
type published struct {
|
|
subject, id string
|
|
body []byte
|
|
}
|
|
|
|
type askerRig struct {
|
|
a *asker
|
|
open []conditions.Condition
|
|
store memAskedStore
|
|
sent []published
|
|
called []string
|
|
silenced []string
|
|
acts []link.HandAct
|
|
now time.Time
|
|
}
|
|
|
|
func newAskerRig(t *testing.T) *askerRig {
|
|
r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)}
|
|
r.a = &asker{
|
|
open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil },
|
|
silence: func(_ context.Context, key string, d time.Duration, by, why string) error {
|
|
r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why)
|
|
// As the controller's conditions do (the confirmation review of 2026-10-09, M1): the condition is
|
|
// silenced from now on, so what is asked next sees it silenced.
|
|
for i := range r.open {
|
|
if r.open[i].Key == key {
|
|
r.open[i].Silenced = &conditions.Silence{Until: r.now.Add(d), By: by, Why: why, Since: r.now}
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
store: r.store,
|
|
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
|
r.sent = append(r.sent, published{subject, id, body})
|
|
return nil
|
|
},
|
|
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
|
raw, _ := json.Marshal(args)
|
|
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
|
return nil
|
|
},
|
|
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
|
now: func() time.Time { return r.now },
|
|
logf: t.Logf,
|
|
}
|
|
return r
|
|
}
|
|
|
|
func heldCondition() conditions.Condition {
|
|
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
|
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
|
|
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
|
}
|
|
|
|
func unitsCondition() conditions.Condition {
|
|
key := "machine.shanks.units"
|
|
return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning,
|
|
Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.",
|
|
Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}}
|
|
}
|
|
|
|
func (r *askerRig) asksSent(t *testing.T) []asks.Ask {
|
|
t.Helper()
|
|
var out []asks.Ask
|
|
for _, p := range r.sent {
|
|
if p.subject != asks.AskSubject("mesh-controller") {
|
|
continue
|
|
}
|
|
var q asks.Ask
|
|
if err := json.Unmarshal(p.body, &q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out = append(out, q)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"}
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 2 {
|
|
t.Fatalf("asked %d times: %+v", len(sent), sent)
|
|
}
|
|
byAbout := map[string]asks.Ask{}
|
|
for _, q := range sent {
|
|
byAbout[q.About] = q
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Errorf("%s: %v", q.About, err)
|
|
}
|
|
}
|
|
held := byAbout[heldCondition().Key]
|
|
if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve ||
|
|
held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator ||
|
|
held.OnExpiry == "" {
|
|
t.Errorf("the held delivery is asked %+v", held)
|
|
}
|
|
units := byAbout["machine.shanks.units"]
|
|
if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) {
|
|
t.Errorf("the failed units are asked %+v", units)
|
|
}
|
|
// No second ask while one is open.
|
|
r.now = r.now.Add(time.Minute)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Errorf("asked again while open: %d", n)
|
|
}
|
|
}
|
|
|
|
func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
// The units are silenced, the held delivery lasts past its ask's day.
|
|
units := unitsCondition()
|
|
units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)}
|
|
r.open = []conditions.Condition{heldCondition(), units}
|
|
r.now = r.now.Add(askApproveFor)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cancels int
|
|
for _, p := range r.sent {
|
|
if p.subject == asks.CancelSubject("mesh-controller") {
|
|
cancels++
|
|
}
|
|
}
|
|
if cancels != 1 {
|
|
t.Errorf("cancels %d, want the silenced one's", cancels)
|
|
}
|
|
if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key {
|
|
t.Errorf("the expired ask was not asked again: %+v", sent)
|
|
}
|
|
}
|
|
|
|
// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label.
|
|
func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant {
|
|
t.Helper()
|
|
for _, a := range r.store {
|
|
if a.Condition != condition || a.State != askOpen {
|
|
continue
|
|
}
|
|
for _, o := range a.Ask.Options {
|
|
if o.Label == label {
|
|
return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen,
|
|
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now,
|
|
AskDigest: a.Ask.Digest(),
|
|
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("no open ask about %s offers %s", condition, label)
|
|
return asks.Warrant{}
|
|
}
|
|
|
|
func answerWith(t *testing.T, r *askerRig, w asks.Warrant) {
|
|
t.Helper()
|
|
body, _ := json.Marshal(w)
|
|
if err := r.a.Decided(context.Background(), body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantIsActedOnOnce(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
answerWith(t, r, w)
|
|
answerWith(t, r, w) // heard again
|
|
if len(r.called) != 1 {
|
|
t.Fatalf("called %v", r.called)
|
|
}
|
|
want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}`
|
|
if r.called[0] != want {
|
|
t.Errorf("called\n %s\nwant\n %s", r.called[0], want)
|
|
}
|
|
if len(r.acts) != 1 {
|
|
t.Fatalf("hand-acts %+v", r.acts)
|
|
}
|
|
act := r.acts[0]
|
|
if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" ||
|
|
act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" ||
|
|
act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" {
|
|
t.Errorf("the hand-act %+v", act)
|
|
}
|
|
if !personsDecision(act) {
|
|
t.Error("an act on a warrant counts as a repair")
|
|
}
|
|
if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
|
t.Errorf("kept %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
|
|
for name, change := range map[string]func(*asks.Warrant){
|
|
"another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" },
|
|
"an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" },
|
|
"an option not offered": func(w *asks.Warrant) { w.Option = "delete" },
|
|
"another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge },
|
|
"no person": func(w *asks.Warrant) { w.By = nil },
|
|
"another ask's digest": func(w *asks.Warrant) { w.AskDigest = "sha256:0000" },
|
|
"no ask's digest": func(w *asks.Warrant) { w.AskDigest = "" },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
change(&w)
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts)+len(r.silenced) != 0 {
|
|
t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
|
|
plan := "plan-1791454185265004861"
|
|
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
|
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
|
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
|
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
|
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
|
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
|
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
|
for _, tc := range []struct {
|
|
c conditions.Condition
|
|
label string
|
|
want string
|
|
}{
|
|
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
|
|
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
|
|
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
|
|
} {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{tc.c}
|
|
_ = r.a.reconcile(context.Background())
|
|
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
|
|
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, "machine.shanks.units", "Silence for a week")
|
|
w.Level, w.Proofs = asks.Acknowledge, nil
|
|
w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14",
|
|
Verified: "a desk click: whoever was at the operator's session on g14"}
|
|
w.Channel = "desk-channel"
|
|
answerWith(t, r, w)
|
|
if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") {
|
|
t.Fatalf("silenced %v, called %v", r.silenced, r.called)
|
|
}
|
|
if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 {
|
|
t.Errorf("%+v", r.acts)
|
|
}
|
|
}
|
|
|
|
func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time"
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) ||
|
|
!strings.HasPrefix(r.store[w.Ask].Acted, "nothing") {
|
|
t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask])
|
|
}
|
|
// And a choice for a condition that ended meanwhile does nothing either.
|
|
r2 := newAskerRig(t)
|
|
r2.open = []conditions.Condition{heldCondition()}
|
|
_ = r2.a.reconcile(context.Background())
|
|
w2 := r2.warrantFor(t, heldCondition().Key, "Release")
|
|
r2.open = nil
|
|
answerWith(t, r2, w2)
|
|
if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" {
|
|
t.Errorf("%v %+v", r2.called, r2.store[w2.Ask])
|
|
}
|
|
}
|
|
|
|
func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) {
|
|
if id != w.Ask {
|
|
return nil, errors.New("another ask")
|
|
}
|
|
return &w, nil
|
|
}
|
|
r.now = r.now.Add(askCatchUpAfter)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") {
|
|
t.Errorf("called %v", r.called)
|
|
}
|
|
if n := len(r.asksSent(t)); n != 1 {
|
|
t.Errorf("asked again after the answer: %d", n)
|
|
}
|
|
}
|
|
|
|
// After review (2026-10-08): a refused ask is not asked again at once; issue 369: nor is the refusal believed
|
|
// for ever — it is asked again after a wait that doubles with each refusal in a row, and at once when the
|
|
// channels change.
|
|
func TestAnAskTheRouterRefusedIsAskedAgainAfterAGrowingWait(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
|
r.a.channels = func(context.Context) string { return channels }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
refuse := func() {
|
|
t.Helper()
|
|
sent := r.asksSent(t)
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
|
Outcome: asks.OutcomeRefused, Words: "no channel can carry any of its answers now", At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
first := r.asksSent(t)[0]
|
|
refuse()
|
|
if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") {
|
|
t.Fatalf("the refusal was kept as %+v", got)
|
|
}
|
|
// Each wait: nothing asked before it ends, asked once when it has.
|
|
for i, wait := range []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute,
|
|
16 * time.Minute, 30 * time.Minute, 30 * time.Minute} {
|
|
before := len(r.asksSent(t))
|
|
r.now = r.now.Add(wait - 10*time.Second)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != before {
|
|
t.Fatalf("refusal %d: asked again before %s", i+1, wait)
|
|
}
|
|
r.now = r.now.Add(10 * time.Second)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != before+1 {
|
|
t.Fatalf("refusal %d: not asked again after %s (%d asks)", i+1, wait, n)
|
|
}
|
|
refuse()
|
|
}
|
|
before := len(r.asksSent(t))
|
|
channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true"
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != before+1 {
|
|
t.Errorf("not asked again once the channels changed: %d", n-before)
|
|
}
|
|
}
|
|
|
|
// After review: at most three asks open at once, the most urgent first, then the oldest.
|
|
func TestAtMostThreeAsksAreOpenTheMostUrgentFirst(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var open []conditions.Condition
|
|
for i := 0; i < 4; i++ {
|
|
c := unitsCondition()
|
|
c.Key = "machine.m" + string(rune('a'+i)) + ".units"
|
|
c.Actions = []conditions.Action{conditions.SilenceAction(c.Key)}
|
|
c.Raised = r.now.Add(-time.Duration(10-i) * time.Hour)
|
|
open = append(open, c)
|
|
}
|
|
urgent := heldCondition()
|
|
urgent.Severity, urgent.Raised = conditions.Urgent, r.now.Add(-time.Minute)
|
|
r.open = append(open, urgent)
|
|
_ = r.a.reconcile(context.Background())
|
|
sent := r.asksSent(t)
|
|
if len(sent) != askMostOpen || sent[0].About != urgent.Key || sent[1].About != "machine.ma.units" || sent[2].About != "machine.mb.units" {
|
|
var about []string
|
|
for _, q := range sent {
|
|
about = append(about, q.About)
|
|
}
|
|
t.Fatalf("asked %v", about)
|
|
}
|
|
}
|
|
|
|
// After review: nothing is asked while no router takes asks under the controller's name, and that is said once.
|
|
func TestNothingIsAskedWithoutARouter(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var said []string
|
|
r.a.logf = func(f string, a ...any) { said = append(said, f) }
|
|
r.a.routerHere = func(context.Context) (bool, error) { return false, nil }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
_ = r.a.reconcile(context.Background())
|
|
if len(r.asksSent(t)) != 0 {
|
|
t.Error("asked with no router")
|
|
}
|
|
n := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, "no router takes asks") {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("said %d times", n)
|
|
}
|
|
}
|
|
|
|
// After review: the condition's words keep where an answer is given without a channel; the ask's text does not.
|
|
func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) {
|
|
c := heldCondition()
|
|
if !strings.Contains(c.Explanation, FromMeshMCPServer) {
|
|
t.Fatalf("the condition lost where it is answered: %q", c.Explanation)
|
|
}
|
|
q, _ := askOf("x", c, partsOf(c)[0], time.Now())
|
|
if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") {
|
|
t.Errorf("the ask says %q", q.Explanation)
|
|
}
|
|
if askApproveFor >= 24*time.Hour {
|
|
t.Errorf("an approving ask lasts %s, which the SDK may refuse at its bound", askApproveFor)
|
|
}
|
|
}
|
|
|
|
// After review (security finding 9): a warrant is acted on only for an ask open in the controller's own record,
|
|
// once the claim stands, and never when given after the ask expired.
|
|
func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
late := w
|
|
late.At = r.store[w.Ask].Ask.Expires.Add(time.Minute)
|
|
body, _ := json.Marshal(late)
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Fatalf("acted on a warrant given after the ask expired: %v", r.called)
|
|
}
|
|
// Claimed already by another delivery: nothing done here.
|
|
kept := r.store[w.Ask]
|
|
kept.Acted = "acting"
|
|
r.store[w.Ask] = kept
|
|
body, _ = json.Marshal(w)
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Fatalf("acted though the claim was another's: %v", r.called)
|
|
}
|
|
// Cancelled in its own record: refused.
|
|
kept.Acted, kept.State = "", askCancelled
|
|
r.store[w.Ask] = kept
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Errorf("acted on a cancelled ask: %v", r.called)
|
|
}
|
|
}
|
|
|
|
// novox/hq ADR 0259 §6: a warrant authorises the act its option bound when the controller asked, and no
|
|
// other. A record of the act changed after the ask — another delivery, another machine, another argument —
|
|
// is refused and nothing is performed.
|
|
func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) {
|
|
for name, change := range map[string]func(*conditions.Action){
|
|
"another argument": func(a *conditions.Action) {
|
|
a.Arguments = map[string]string{"id": "novox/mesh-controller@000000000000"}
|
|
},
|
|
"another verb": func(a *conditions.Action) { a.Verb = "mesh-delivery.stop" },
|
|
"another machine": func(a *conditions.Action) { a.Machine = "anchor" },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
kept := r.store[w.Ask]
|
|
acts := append([]conditions.Action(nil), kept.Actions...)
|
|
i := kept.Options[w.Option]
|
|
change(&acts[i])
|
|
kept.Actions = acts
|
|
r.store[w.Ask] = kept
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts) != 0 {
|
|
t.Errorf("performed an act the option did not bind: %v %v", r.called, r.acts)
|
|
}
|
|
})
|
|
}
|
|
// Every option of an ask binds its act.
|
|
q, _ := askOf("x", heldCondition(), partsOf(heldCondition())[0], time.Now())
|
|
for _, o := range q.Options {
|
|
if o.Binds == "" {
|
|
t.Errorf("the option %s binds nothing", o.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Failure is loud (novox/hq ADR 0259, the self-review of 2026-10-09): a condition that needs the operator and
|
|
// could not be asked on any channel — no router, or the router refused the ask — is a condition of its own,
|
|
// cleared once it can be asked again.
|
|
func TestAnAskThatCannotBeDeliveredIsSaid(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
|
routerHere := false
|
|
r.a.routerHere = func(context.Context) (bool, error) { return routerHere, nil }
|
|
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
|
r.a.channels = func(context.Context) string { return channels }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || got[0].Kind != "asks-undelivered" ||
|
|
!strings.Contains(got[0].Summary, heldCondition().Key) || !strings.Contains(got[0].Summary, "no router") {
|
|
t.Fatalf("no router, said as %+v", got)
|
|
}
|
|
|
|
routerHere = true
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("asked, and still said undelivered: %+v", got)
|
|
}
|
|
first := r.asksSent(t)[0]
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused,
|
|
Words: "no channel can carry any of its answers now", At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no channel can carry") {
|
|
t.Fatalf("the router's refusal, said as %+v", got)
|
|
}
|
|
if why, ok := conditions.PlainWords(conditions.Words{Headline: last()[0].Headline, Explanation: last()[0].Explanation,
|
|
Needs: last()[0].Needs, Resolved: last()[0].Resolved}, ""); !ok {
|
|
t.Errorf("not plain: %s", why)
|
|
}
|
|
r.open = nil
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Errorf("nothing needs asking, and still said: %+v", got)
|
|
}
|
|
}
|
|
|
|
// The review of 2026-10-09 (M1): an acknowledging answer never shares an ask with an authorising one. A
|
|
// condition offering Restart and Silence is asked twice — Restart alone, about the condition, and Silence
|
|
// alone, apart — so Silence chosen on a channel that only acknowledges leaves the Restart ask open.
|
|
func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
key := "module.shanks.plex.down"
|
|
c := conditions.Condition{Key: key, Kind: "module-down", Severity: conditions.Urgent, Headline: "Plex down on shanks",
|
|
Explanation: "Needs you: restart it, or silence this.", Needs: "restart it, or silence this.",
|
|
Actions: []conditions.Action{
|
|
{Label: "Restart", Verb: "node-service-manager.restart", Machine: "shanks", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "plex"}},
|
|
conditions.SilenceAction(key)}}
|
|
r.open = []conditions.Condition{c}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 2 {
|
|
t.Fatalf("asked %d time(s): %+v", len(sent), sent)
|
|
}
|
|
for _, q := range sent {
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Errorf("%s: %v", q.About, err)
|
|
}
|
|
levels := map[asks.Level]bool{}
|
|
for _, o := range q.Options {
|
|
levels[o.Level] = true
|
|
}
|
|
if len(levels) != 1 {
|
|
t.Errorf("the ask about %s mixes levels: %+v", q.About, q.Options)
|
|
}
|
|
}
|
|
byAbout := map[string]asks.Ask{}
|
|
for _, q := range sent {
|
|
byAbout[q.About] = q
|
|
}
|
|
if q := byAbout[key]; len(q.Options) != 1 || q.Options[0].Label != "Restart" {
|
|
t.Errorf("the condition's own ask: %+v", q)
|
|
}
|
|
if q := byAbout[key+".acknowledge"]; len(q.Options) != 1 || q.Options[0].Level != asks.Acknowledge {
|
|
t.Errorf("the acknowledging ask: %+v", q)
|
|
}
|
|
// Silence chosen: performed, and the Restart ask stays open, never asked twice.
|
|
answerWith(t, r, r.warrantFor(t, key, "Silence for a week"))
|
|
if len(r.silenced) != 1 || len(r.called) != 0 {
|
|
t.Fatalf("silenced %v called %v", r.silenced, r.called)
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
open := 0
|
|
for _, a := range r.store {
|
|
if a.State == askOpen && a.Condition == key {
|
|
open++
|
|
if a.Part != "" || a.Ask.Options[0].Label != "Restart" {
|
|
t.Errorf("the open ask is %+v", a)
|
|
}
|
|
}
|
|
}
|
|
if open != 1 || len(r.asksSent(t)) != 2 {
|
|
t.Errorf("after the silence: %d open, %d asked", open, len(r.asksSent(t)))
|
|
}
|
|
// And the approval still answers: Restart chosen on a channel that proves who answered is performed.
|
|
answerWith(t, r, r.warrantFor(t, key, "Restart"))
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "node-service-manager.restart@shanks") {
|
|
t.Errorf("the approval kept through a silence was not performed: %v", r.called)
|
|
}
|
|
}
|
|
|
|
// novox/hq issue 353: the controller's grant to ask is composed from the router's assignment and reaches the
|
|
// bus only when its machine is pushed. Between the two, the bus refuses every ask (measured 2026-10-09, 17:54 to
|
|
// 17:56 local: seven refusals of mesh.seat.operator-channel.accept.ask.mesh-controller). So nothing is asked
|
|
// while the bus's user list is behind, it is said once, the conditions that need the operator are raised as
|
|
// undelivered with what to do, and the asks go out once the bus holds the grant.
|
|
func TestNothingIsAskedWhileTheBusLacksTheControllersGrant(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var said []string
|
|
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
held := false
|
|
r.a.grantHeld = func(context.Context) (bool, string, error) {
|
|
return held, "the bus's user list on anchor is behind what the mesh composes; `push anchor` carries it", nil
|
|
}
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
_ = r.a.reconcile(context.Background())
|
|
if len(r.asksSent(t)) != 0 {
|
|
t.Error("asked while the bus lacks the grant")
|
|
}
|
|
n := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, "does not hold the controller's grant") {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("said %d times: %q", n, said)
|
|
}
|
|
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
|
!strings.Contains(raised[len(raised)-1][0].Summary, "`push anchor` carries it") ||
|
|
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
|
t.Fatalf("not said as a condition with what to do: %+v", raised)
|
|
}
|
|
held = true
|
|
_ = r.a.reconcile(context.Background())
|
|
if sent := r.asksSent(t); len(sent) != 1 || sent[0].About != heldCondition().Key {
|
|
t.Errorf("not asked once the bus holds the grant: %+v", sent)
|
|
}
|
|
if last := raised[len(raised)-1]; len(last) != 0 {
|
|
t.Errorf("the undelivered condition was not cleared: %+v", last)
|
|
}
|
|
}
|
|
|
|
// novox/hq issue 369: the router refused while its channels had not yet said they could send; they could twenty
|
|
// minutes later, and "Questions for you not delivered" repeated that refusal for eleven hours, escalating on it.
|
|
// The ask is made again; while the router's word on it is awaited the condition stands unchanged (neither
|
|
// cleared nor raised again); once the router took it, the condition clears; a new refusal is said in its own
|
|
// words.
|
|
func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
|
r.a.channels = func(context.Context) string { return "channel/telegram=telegram@anchor[choice]own:true" }
|
|
r.open = []conditions.Condition{unitsCondition()}
|
|
refuse := func(words string) {
|
|
t.Helper()
|
|
sent := r.asksSent(t)
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
|
Outcome: asks.OutcomeRefused, Words: words, At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
refuse("no channel can carry any of its answers now: telegram: telegram has not said whether it can send")
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "telegram has not said") {
|
|
t.Fatalf("the refusal, said as %+v", got)
|
|
}
|
|
// The wait ends: asked again; until the router's word on it is in, the condition stands as it was.
|
|
r.now = r.now.Add(askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Fatalf("not asked again: %d asks", n)
|
|
}
|
|
if got := last(); len(got) != 1 {
|
|
t.Fatalf("cleared while the router's word on the new ask was awaited: the condition would flap")
|
|
}
|
|
r.now = r.now.Add(askVerdictWait / 2)
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 {
|
|
t.Fatalf("cleared inside the verdict wait: the condition would flap")
|
|
}
|
|
// Refused again, now for a reason of today: said in those words.
|
|
refuse("no channel can carry any of its answers now: telegram: no account is linked")
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no account is linked") ||
|
|
strings.Contains(got[0].Summary, "has not said") {
|
|
t.Fatalf("an old refusal's words repeated: %+v", got)
|
|
}
|
|
// Asked again after the doubled wait, and taken: no refusal comes, and the condition clears.
|
|
r.now = r.now.Add(2 * askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 3 {
|
|
t.Fatalf("not asked again after the doubled wait: %d asks", n)
|
|
}
|
|
r.now = r.now.Add(askVerdictWait)
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("taken, and still said undelivered: %+v", got)
|
|
}
|
|
if n := len(r.asksSent(t)); n != 3 {
|
|
t.Fatalf("an ask taken was asked again: %d asks", n)
|
|
}
|
|
// The operator answers it; the condition stays open (its fix takes a while): the old refusal is not said again.
|
|
taken := r.asksSent(t)[2]
|
|
r.store.Change(context.Background(), taken.ID, func(x *asked) bool {
|
|
x.State, x.Ended = string(asks.OutcomeChosen), r.now
|
|
return true
|
|
})
|
|
for i := 0; i < 5; i++ {
|
|
r.now = r.now.Add(askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("an answered ask brought its old refusal back: %+v", got)
|
|
}
|
|
}
|
|
// Its explanation opens with one verdict, not two.
|
|
r.open = append(r.open, heldCondition())
|
|
_ = r.a.reconcile(context.Background())
|
|
refuse("no channel can carry any of its answers now")
|
|
_ = r.a.reconcile(context.Background())
|
|
if e := conditions.Verdict(last()[0].Needs, last()[0].Explanation); strings.Count(e, "Needs you") != 1 {
|
|
t.Errorf("the explanation says its verdict twice: %q", e)
|
|
}
|
|
}
|
|
|
|
// hq issue 353, the review: the bus holds the controller's grant to ask only when both facts hold — the mesh
|
|
// composes the controller's own grant, and the bus's module says it is in force on the running server — and
|
|
// each fact not known is not held, said with why.
|
|
func TestTheGrantIsHeldOnlyWhenComposedAndInForceOnTheBus(t *testing.T) {
|
|
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
|
no := func(why string) grantJudge {
|
|
return func(context.Context) (bool, string, error) { return false, why, nil }
|
|
}
|
|
broken := func(context.Context) (bool, string, error) { return false, "", errors.New("the store is away") }
|
|
for _, c := range []struct {
|
|
name string
|
|
composed, inForce grantJudge
|
|
held bool
|
|
says string
|
|
}{
|
|
{"composed and in force", yes, yes, true, ""},
|
|
{"not composed", no("no router is assigned"), yes, false, "no router is assigned"},
|
|
{"composed, written, not reloaded", yes, no("the server has not reloaded"), false, "has not reloaded"},
|
|
{"the composition unknown", broken, yes, false, "could not be worked out: the store is away"},
|
|
{"the bus unknown", yes, broken, false, "could not be worked out: the store is away"},
|
|
{"nothing judges the bus", yes, nil, false, "not judged"},
|
|
} {
|
|
held, why, err := grantHeldBy(c.composed, c.inForce, time.Now)(context.Background())
|
|
if err != nil || held != c.held || !strings.Contains(why, c.says) {
|
|
t.Errorf("%s: held %v (%q, %v)", c.name, held, why, err)
|
|
}
|
|
}
|
|
// Remembered for grantLookEvery, then judged again.
|
|
now := time.Date(2026, 10, 9, 18, 0, 0, 0, time.UTC)
|
|
inForce := false
|
|
judge := grantHeldBy(yes, func(context.Context) (bool, string, error) { return inForce, "not yet", nil },
|
|
func() time.Time { return now })
|
|
if held, _, _ := judge(context.Background()); held {
|
|
t.Fatal("held before the bus enforces it")
|
|
}
|
|
inForce = true
|
|
if held, _, _ := judge(context.Background()); held {
|
|
t.Error("judged again inside grantLookEvery")
|
|
}
|
|
now = now.Add(grantLookEvery)
|
|
if held, _, _ := judge(context.Background()); !held {
|
|
t.Error("not judged again after grantLookEvery")
|
|
}
|
|
}
|
|
|
|
// The controller's own grant, as composed: present while a router that takes asks is assigned, absent
|
|
// otherwise — whatever else in the user list changes.
|
|
func TestTheControllersOwnGrantToAskIsReadFromTheComposition(t *testing.T) {
|
|
router := broker.Declared{Module: "messenger", Holds: []broker.Seat{{Name: broker.AsksSeat, Scope: "mesh",
|
|
Accepts: []string{"ask", "cancel"}, Emits: []string{"decided"}, ByCaller: []string{"ask", "cancel", "decided"}}}}
|
|
other := broker.Declared{Module: "dunst"}
|
|
with := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {router, other}}}
|
|
without := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {other}}}
|
|
if may, err := controllerMayAsk(with); err != nil || !may {
|
|
t.Errorf("with a router assigned: %v %v", may, err)
|
|
}
|
|
if may, err := controllerMayAsk(without); err != nil || may {
|
|
t.Errorf("with no router assigned: %v %v", may, err)
|
|
}
|
|
}
|
|
|
|
// The bus module's answer, with the record's word on the list sent as the check only where the answer does
|
|
// not settle it (reviews 22 and 31 of PR 188); each way it is not held names its own remedy.
|
|
func TestTheBusModulesAnswerNamesTheRemedyForEachCase(t *testing.T) {
|
|
const (
|
|
reloaded = `{"allowed":true,"in_force":true,"file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:56:05Z"}`
|
|
notLoaded = `{"allowed":true,"in_force":false,"why":"the user list on the machine allows it, and the server has not reloaded since it was written","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`
|
|
noVarz = `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`
|
|
noStat = `{"allowed":true,"in_force":false,"why":"when the bus's user list was written could not be read inside mesh-broker-nats: exit 1"}`
|
|
notAllowed = `{"allowed":false,"in_force":false,"why":"the user list on the machine does not allow it"}`
|
|
)
|
|
current := busListState{Machine: "anchor"}
|
|
behind := busListState{Machine: "anchor", Behind: true}
|
|
stepped := busListState{Machine: "anchor", Behind: true, BusStep: "sending anchor would replace the bus; `bus upgrade --why …`"}
|
|
uncomposed := busListState{Machine: "anchor", Behind: true, Uncomposed: errors.New("module foo requires a provision nobody provides")}
|
|
for _, c := range []struct {
|
|
name, answer string
|
|
askErr error
|
|
list busListState
|
|
listErr error
|
|
held bool
|
|
says, not string
|
|
}{
|
|
{"in force", reloaded, nil, behind, nil, true, "", "push"},
|
|
{"written, not reloaded", notLoaded, nil, current, nil, false, "within about 5 s", "push"},
|
|
{"in force not known: varz", noVarz, nil, current, nil, false, "is not known: the server's last configuration load", "wait"},
|
|
{"in force not known: stat", noStat, nil, current, nil, false, "is not known: when the bus's user list was written", "wait"},
|
|
{"not allowed, list behind", notAllowed, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
|
{"not allowed, bus step pending", notAllowed, nil, stepped, nil, false, "`bus upgrade", "`push anchor`"},
|
|
{"not allowed, list uncomposed", notAllowed, nil, uncomposed, nil, false, "`plan anchor`", "`push anchor`"},
|
|
{"older module, allowed, list current", `{"allowed":true}`, nil, current, nil, true, "not confirmed", "push"},
|
|
{"older module, allowed, list behind", `{"allowed":true}`, nil, behind, nil, true, "not confirmed", "push"},
|
|
{"older module, allowed, list unknown", `{"allowed":true}`, nil, current, errors.New("the store is away"), true, "`bus upgrade`", "push"},
|
|
{"older module, not allowed, list behind", `{"allowed":false}`, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
|
{"module cannot answer, list current", "", errors.New("no responders"), current, nil, true, "could not answer", "push"},
|
|
{"module cannot answer, list behind", "", errors.New("no responders"), behind, nil, false, "`push anchor` carries it", ""},
|
|
{"module cannot answer, list unknown", "", errors.New("no responders"), current, errors.New("the store is away"), false, "could not be worked out", ""},
|
|
{"answer unreadable, list behind", `not json`, nil, behind, nil, false, "could not be read", ""},
|
|
} {
|
|
judge := busGrantBy(
|
|
func(context.Context) (string, json.RawMessage, error) {
|
|
return "anchor", json.RawMessage(c.answer), c.askErr
|
|
},
|
|
func(context.Context) (busListState, error) { return c.list, c.listErr })
|
|
held, why, _ := judgeGrantOnce(judge)
|
|
if held != c.held || !strings.Contains(why, c.says) || (c.not != "" && strings.Contains(why, c.not)) {
|
|
t.Errorf("%s: held %v (%q)", c.name, held, why)
|
|
}
|
|
}
|
|
none := busGrantBy(func(context.Context) (string, json.RawMessage, error) { return "", nil, nil }, nil)
|
|
if held, why, _ := judgeGrantOnce(none); held || !strings.Contains(why, "no module holds the bus") {
|
|
t.Errorf("no bus: held %v (%q)", held, why)
|
|
}
|
|
}
|
|
|
|
// judgeGrantOnce is the whole judgement with the composed grant held, as the asker makes it.
|
|
func judgeGrantOnce(bus grantJudge) (bool, string, error) {
|
|
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
|
return grantHeldBy(yes, bus, time.Now)(context.Background())
|
|
}
|
|
|
|
// The orders reviews 22 and 31 of PR 188 found: this controller live before the bus module that answers
|
|
// in_force, with the bus's machine behind on its list — and, after mesh-catalog #166 merged, unpushable until a
|
|
// person's bus upgrade. An older module that allows the ask asks (once, with the caveat said); a new module that
|
|
// has not reloaded names the wait, not a push; one that cannot say whether it reloaded names what is not known.
|
|
func TestAnOlderBusModuleNeitherSilencesTheAsksNorNamesTheWrongRemedy(t *testing.T) {
|
|
for _, c := range []struct {
|
|
name, answer string
|
|
list busListState
|
|
asks bool
|
|
says, not string
|
|
}{
|
|
{"older module, list current", `{"allowed":true}`, busListState{Machine: "anchor"}, true, "not confirmed", ""},
|
|
{"older module, list behind, bus step pending", `{"allowed":true}`,
|
|
busListState{Machine: "anchor", Behind: true, BusStep: "a planned step"}, true, "not confirmed", ""},
|
|
{"new module, not reloaded",
|
|
`{"allowed":true,"in_force":false,"why":"not reloaded","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`,
|
|
busListState{Machine: "anchor"}, false, "within about 5 s", "push"},
|
|
{"new module, reload not known", `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`,
|
|
busListState{Machine: "anchor"}, false, "is not known", "wait"},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var said []string
|
|
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
|
r.a.grantHeld = grantHeldBy(yes, busGrantBy(
|
|
func(context.Context) (string, json.RawMessage, error) {
|
|
return "anchor", json.RawMessage(c.answer), nil
|
|
},
|
|
func(context.Context) (busListState, error) { return c.list, nil }),
|
|
func() time.Time { return r.now })
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
_ = r.a.reconcile(context.Background())
|
|
sent := r.asksSent(t)
|
|
if c.asks {
|
|
if len(sent) != 1 || sent[0].About != heldCondition().Key {
|
|
t.Fatalf("not asked: %+v", sent)
|
|
}
|
|
n := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, c.says) {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("the caveat said %d times: %q", n, said)
|
|
}
|
|
return
|
|
}
|
|
if len(sent) != 0 {
|
|
t.Fatalf("asked while the bus does not hold the grant: %+v", sent)
|
|
}
|
|
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
|
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
|
t.Fatalf("not said as undelivered: %+v", raised)
|
|
}
|
|
summary := raised[len(raised)-1][0].Summary
|
|
if !strings.Contains(summary, c.says) || (c.not != "" && strings.Contains(summary, c.not)) {
|
|
t.Errorf("the condition names the wrong remedy: %q", summary)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The serving asker is wired to the real judgement: with records and a bus it cannot read, the grant is not
|
|
// held, and so nothing is asked (a stub saying yes, or no judgement at all, would ask).
|
|
func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
|
a := servingAsker(&stores{}, nil, nil, nil)
|
|
if a.grantHeld == nil {
|
|
t.Fatal("the serving asker does not judge the bus's grant")
|
|
}
|
|
held, why, err := a.grantHeld(context.Background())
|
|
if err != nil || held || !strings.Contains(why, "could not be worked out") {
|
|
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
|
}
|
|
}
|