jochen
2e0a7d1cbd
mesh/merge-gate pass: builds mesh-controller → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery-group group fix/373-one-message-per-condition delivering: 0 of 2 delivered
mesh/delivery delivered
A refusal stood for its part whenever no ask was open, so after the retry was taken and answered, "Questions for you not delivered" came back with the old words for an hour, then again after each answer. A refusal is now current only while no ask was made after it that the router did not refuse; the verdict wait applies only to that newer ask. Tests reconcile inside the wait and after an answer.
811 lines
31 KiB
Go
811 lines
31 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs
|
|
// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level.
|
|
|
|
type memAskedStore map[string]asked
|
|
|
|
// memAskedMu guards every memAskedStore: Change is a compare-and-set as the bus's is.
|
|
var memAskedMu sync.Mutex
|
|
|
|
func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
r, ok := m[id]
|
|
if !ok {
|
|
return nil, nil
|
|
}
|
|
return &r, nil
|
|
}
|
|
func (m memAskedStore) Create(_ context.Context, r asked) error {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
if _, kept := m[r.ID]; kept {
|
|
return errors.New("an ask is kept under that id")
|
|
}
|
|
m[r.ID] = r
|
|
return nil
|
|
}
|
|
func (m memAskedStore) Change(_ context.Context, id string, change func(*asked) bool) (bool, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
r, ok := m[id]
|
|
if !ok || !change(&r) {
|
|
return false, nil
|
|
}
|
|
m[id] = r
|
|
return true, nil
|
|
}
|
|
func (m memAskedStore) All(context.Context) ([]asked, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
var out []asked
|
|
for _, r := range m {
|
|
out = append(out, r)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
type published struct {
|
|
subject, id string
|
|
body []byte
|
|
}
|
|
|
|
type askerRig struct {
|
|
a *asker
|
|
open []conditions.Condition
|
|
store memAskedStore
|
|
sent []published
|
|
called []string
|
|
silenced []string
|
|
acts []link.HandAct
|
|
now time.Time
|
|
}
|
|
|
|
func newAskerRig(t *testing.T) *askerRig {
|
|
r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)}
|
|
r.a = &asker{
|
|
open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil },
|
|
silence: func(_ context.Context, key string, d time.Duration, by, why string) error {
|
|
r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why)
|
|
// As the controller's conditions do (the confirmation review of 2026-10-09, M1): the condition is
|
|
// silenced from now on, so what is asked next sees it silenced.
|
|
for i := range r.open {
|
|
if r.open[i].Key == key {
|
|
r.open[i].Silenced = &conditions.Silence{Until: r.now.Add(d), By: by, Why: why, Since: r.now}
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
store: r.store,
|
|
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
|
r.sent = append(r.sent, published{subject, id, body})
|
|
return nil
|
|
},
|
|
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
|
raw, _ := json.Marshal(args)
|
|
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
|
return nil
|
|
},
|
|
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
|
now: func() time.Time { return r.now },
|
|
logf: t.Logf,
|
|
}
|
|
return r
|
|
}
|
|
|
|
func heldCondition() conditions.Condition {
|
|
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
|
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
|
|
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
|
}
|
|
|
|
func unitsCondition() conditions.Condition {
|
|
key := "machine.shanks.units"
|
|
return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning,
|
|
Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.",
|
|
Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}}
|
|
}
|
|
|
|
func (r *askerRig) asksSent(t *testing.T) []asks.Ask {
|
|
t.Helper()
|
|
var out []asks.Ask
|
|
for _, p := range r.sent {
|
|
if p.subject != asks.AskSubject("mesh-controller") {
|
|
continue
|
|
}
|
|
var q asks.Ask
|
|
if err := json.Unmarshal(p.body, &q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out = append(out, q)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"}
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 2 {
|
|
t.Fatalf("asked %d times: %+v", len(sent), sent)
|
|
}
|
|
byAbout := map[string]asks.Ask{}
|
|
for _, q := range sent {
|
|
byAbout[q.About] = q
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Errorf("%s: %v", q.About, err)
|
|
}
|
|
}
|
|
held := byAbout[heldCondition().Key]
|
|
if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve ||
|
|
held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator ||
|
|
held.OnExpiry == "" {
|
|
t.Errorf("the held delivery is asked %+v", held)
|
|
}
|
|
units := byAbout["machine.shanks.units"]
|
|
if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) {
|
|
t.Errorf("the failed units are asked %+v", units)
|
|
}
|
|
// No second ask while one is open.
|
|
r.now = r.now.Add(time.Minute)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Errorf("asked again while open: %d", n)
|
|
}
|
|
}
|
|
|
|
func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
// The units are silenced, the held delivery lasts past its ask's day.
|
|
units := unitsCondition()
|
|
units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)}
|
|
r.open = []conditions.Condition{heldCondition(), units}
|
|
r.now = r.now.Add(askApproveFor)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cancels int
|
|
for _, p := range r.sent {
|
|
if p.subject == asks.CancelSubject("mesh-controller") {
|
|
cancels++
|
|
}
|
|
}
|
|
if cancels != 1 {
|
|
t.Errorf("cancels %d, want the silenced one's", cancels)
|
|
}
|
|
if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key {
|
|
t.Errorf("the expired ask was not asked again: %+v", sent)
|
|
}
|
|
}
|
|
|
|
// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label.
|
|
func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant {
|
|
t.Helper()
|
|
for _, a := range r.store {
|
|
if a.Condition != condition || a.State != askOpen {
|
|
continue
|
|
}
|
|
for _, o := range a.Ask.Options {
|
|
if o.Label == label {
|
|
return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen,
|
|
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now,
|
|
AskDigest: a.Ask.Digest(),
|
|
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("no open ask about %s offers %s", condition, label)
|
|
return asks.Warrant{}
|
|
}
|
|
|
|
func answerWith(t *testing.T, r *askerRig, w asks.Warrant) {
|
|
t.Helper()
|
|
body, _ := json.Marshal(w)
|
|
if err := r.a.Decided(context.Background(), body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantIsActedOnOnce(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
answerWith(t, r, w)
|
|
answerWith(t, r, w) // heard again
|
|
if len(r.called) != 1 {
|
|
t.Fatalf("called %v", r.called)
|
|
}
|
|
want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}`
|
|
if r.called[0] != want {
|
|
t.Errorf("called\n %s\nwant\n %s", r.called[0], want)
|
|
}
|
|
if len(r.acts) != 1 {
|
|
t.Fatalf("hand-acts %+v", r.acts)
|
|
}
|
|
act := r.acts[0]
|
|
if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" ||
|
|
act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" ||
|
|
act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" {
|
|
t.Errorf("the hand-act %+v", act)
|
|
}
|
|
if !personsDecision(act) {
|
|
t.Error("an act on a warrant counts as a repair")
|
|
}
|
|
if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
|
t.Errorf("kept %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
|
|
for name, change := range map[string]func(*asks.Warrant){
|
|
"another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" },
|
|
"an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" },
|
|
"an option not offered": func(w *asks.Warrant) { w.Option = "delete" },
|
|
"another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge },
|
|
"no person": func(w *asks.Warrant) { w.By = nil },
|
|
"another ask's digest": func(w *asks.Warrant) { w.AskDigest = "sha256:0000" },
|
|
"no ask's digest": func(w *asks.Warrant) { w.AskDigest = "" },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
change(&w)
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts)+len(r.silenced) != 0 {
|
|
t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
|
|
plan := "plan-1791454185265004861"
|
|
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
|
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
|
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
|
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
|
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
|
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
|
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
|
for _, tc := range []struct {
|
|
c conditions.Condition
|
|
label string
|
|
want string
|
|
}{
|
|
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
|
|
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
|
|
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
|
|
} {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{tc.c}
|
|
_ = r.a.reconcile(context.Background())
|
|
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
|
|
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, "machine.shanks.units", "Silence for a week")
|
|
w.Level, w.Proofs = asks.Acknowledge, nil
|
|
w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14",
|
|
Verified: "a desk click: whoever was at the operator's session on g14"}
|
|
w.Channel = "desk-channel"
|
|
answerWith(t, r, w)
|
|
if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") {
|
|
t.Fatalf("silenced %v, called %v", r.silenced, r.called)
|
|
}
|
|
if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 {
|
|
t.Errorf("%+v", r.acts)
|
|
}
|
|
}
|
|
|
|
func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time"
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) ||
|
|
!strings.HasPrefix(r.store[w.Ask].Acted, "nothing") {
|
|
t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask])
|
|
}
|
|
// And a choice for a condition that ended meanwhile does nothing either.
|
|
r2 := newAskerRig(t)
|
|
r2.open = []conditions.Condition{heldCondition()}
|
|
_ = r2.a.reconcile(context.Background())
|
|
w2 := r2.warrantFor(t, heldCondition().Key, "Release")
|
|
r2.open = nil
|
|
answerWith(t, r2, w2)
|
|
if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" {
|
|
t.Errorf("%v %+v", r2.called, r2.store[w2.Ask])
|
|
}
|
|
}
|
|
|
|
func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) {
|
|
if id != w.Ask {
|
|
return nil, errors.New("another ask")
|
|
}
|
|
return &w, nil
|
|
}
|
|
r.now = r.now.Add(askCatchUpAfter)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") {
|
|
t.Errorf("called %v", r.called)
|
|
}
|
|
if n := len(r.asksSent(t)); n != 1 {
|
|
t.Errorf("asked again after the answer: %d", n)
|
|
}
|
|
}
|
|
|
|
// After review (2026-10-08): a refused ask is not asked again at once; issue 369: nor is the refusal believed
|
|
// for ever — it is asked again after a wait that doubles with each refusal in a row, and at once when the
|
|
// channels change.
|
|
func TestAnAskTheRouterRefusedIsAskedAgainAfterAGrowingWait(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
|
r.a.channels = func(context.Context) string { return channels }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
refuse := func() {
|
|
t.Helper()
|
|
sent := r.asksSent(t)
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
|
Outcome: asks.OutcomeRefused, Words: "no channel can carry any of its answers now", At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
first := r.asksSent(t)[0]
|
|
refuse()
|
|
if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") {
|
|
t.Fatalf("the refusal was kept as %+v", got)
|
|
}
|
|
// Each wait: nothing asked before it ends, asked once when it has.
|
|
for i, wait := range []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute,
|
|
16 * time.Minute, 30 * time.Minute, 30 * time.Minute} {
|
|
before := len(r.asksSent(t))
|
|
r.now = r.now.Add(wait - 10*time.Second)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != before {
|
|
t.Fatalf("refusal %d: asked again before %s", i+1, wait)
|
|
}
|
|
r.now = r.now.Add(10 * time.Second)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != before+1 {
|
|
t.Fatalf("refusal %d: not asked again after %s (%d asks)", i+1, wait, n)
|
|
}
|
|
refuse()
|
|
}
|
|
before := len(r.asksSent(t))
|
|
channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true"
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != before+1 {
|
|
t.Errorf("not asked again once the channels changed: %d", n-before)
|
|
}
|
|
}
|
|
|
|
// After review: at most three asks open at once, the most urgent first, then the oldest.
|
|
func TestAtMostThreeAsksAreOpenTheMostUrgentFirst(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var open []conditions.Condition
|
|
for i := 0; i < 4; i++ {
|
|
c := unitsCondition()
|
|
c.Key = "machine.m" + string(rune('a'+i)) + ".units"
|
|
c.Actions = []conditions.Action{conditions.SilenceAction(c.Key)}
|
|
c.Raised = r.now.Add(-time.Duration(10-i) * time.Hour)
|
|
open = append(open, c)
|
|
}
|
|
urgent := heldCondition()
|
|
urgent.Severity, urgent.Raised = conditions.Urgent, r.now.Add(-time.Minute)
|
|
r.open = append(open, urgent)
|
|
_ = r.a.reconcile(context.Background())
|
|
sent := r.asksSent(t)
|
|
if len(sent) != askMostOpen || sent[0].About != urgent.Key || sent[1].About != "machine.ma.units" || sent[2].About != "machine.mb.units" {
|
|
var about []string
|
|
for _, q := range sent {
|
|
about = append(about, q.About)
|
|
}
|
|
t.Fatalf("asked %v", about)
|
|
}
|
|
}
|
|
|
|
// After review: nothing is asked while no router takes asks under the controller's name, and that is said once.
|
|
func TestNothingIsAskedWithoutARouter(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var said []string
|
|
r.a.logf = func(f string, a ...any) { said = append(said, f) }
|
|
r.a.routerHere = func(context.Context) (bool, error) { return false, nil }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
_ = r.a.reconcile(context.Background())
|
|
if len(r.asksSent(t)) != 0 {
|
|
t.Error("asked with no router")
|
|
}
|
|
n := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, "no router takes asks") {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("said %d times", n)
|
|
}
|
|
}
|
|
|
|
// After review: the condition's words keep where an answer is given without a channel; the ask's text does not.
|
|
func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) {
|
|
c := heldCondition()
|
|
if !strings.Contains(c.Explanation, FromMeshMCPServer) {
|
|
t.Fatalf("the condition lost where it is answered: %q", c.Explanation)
|
|
}
|
|
q, _ := askOf("x", c, partsOf(c)[0], time.Now())
|
|
if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") {
|
|
t.Errorf("the ask says %q", q.Explanation)
|
|
}
|
|
if askApproveFor >= 24*time.Hour {
|
|
t.Errorf("an approving ask lasts %s, which the SDK may refuse at its bound", askApproveFor)
|
|
}
|
|
}
|
|
|
|
// After review (security finding 9): a warrant is acted on only for an ask open in the controller's own record,
|
|
// once the claim stands, and never when given after the ask expired.
|
|
func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
late := w
|
|
late.At = r.store[w.Ask].Ask.Expires.Add(time.Minute)
|
|
body, _ := json.Marshal(late)
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Fatalf("acted on a warrant given after the ask expired: %v", r.called)
|
|
}
|
|
// Claimed already by another delivery: nothing done here.
|
|
kept := r.store[w.Ask]
|
|
kept.Acted = "acting"
|
|
r.store[w.Ask] = kept
|
|
body, _ = json.Marshal(w)
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Fatalf("acted though the claim was another's: %v", r.called)
|
|
}
|
|
// Cancelled in its own record: refused.
|
|
kept.Acted, kept.State = "", askCancelled
|
|
r.store[w.Ask] = kept
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Errorf("acted on a cancelled ask: %v", r.called)
|
|
}
|
|
}
|
|
|
|
// novox/hq ADR 0259 §6: a warrant authorises the act its option bound when the controller asked, and no
|
|
// other. A record of the act changed after the ask — another delivery, another machine, another argument —
|
|
// is refused and nothing is performed.
|
|
func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) {
|
|
for name, change := range map[string]func(*conditions.Action){
|
|
"another argument": func(a *conditions.Action) {
|
|
a.Arguments = map[string]string{"id": "novox/mesh-controller@000000000000"}
|
|
},
|
|
"another verb": func(a *conditions.Action) { a.Verb = "mesh-delivery.stop" },
|
|
"another machine": func(a *conditions.Action) { a.Machine = "anchor" },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
kept := r.store[w.Ask]
|
|
acts := append([]conditions.Action(nil), kept.Actions...)
|
|
i := kept.Options[w.Option]
|
|
change(&acts[i])
|
|
kept.Actions = acts
|
|
r.store[w.Ask] = kept
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts) != 0 {
|
|
t.Errorf("performed an act the option did not bind: %v %v", r.called, r.acts)
|
|
}
|
|
})
|
|
}
|
|
// Every option of an ask binds its act.
|
|
q, _ := askOf("x", heldCondition(), partsOf(heldCondition())[0], time.Now())
|
|
for _, o := range q.Options {
|
|
if o.Binds == "" {
|
|
t.Errorf("the option %s binds nothing", o.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Failure is loud (novox/hq ADR 0259, the self-review of 2026-10-09): a condition that needs the operator and
|
|
// could not be asked on any channel — no router, or the router refused the ask — is a condition of its own,
|
|
// cleared once it can be asked again.
|
|
func TestAnAskThatCannotBeDeliveredIsSaid(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
|
routerHere := false
|
|
r.a.routerHere = func(context.Context) (bool, error) { return routerHere, nil }
|
|
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
|
r.a.channels = func(context.Context) string { return channels }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || got[0].Kind != "asks-undelivered" ||
|
|
!strings.Contains(got[0].Summary, heldCondition().Key) || !strings.Contains(got[0].Summary, "no router") {
|
|
t.Fatalf("no router, said as %+v", got)
|
|
}
|
|
|
|
routerHere = true
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("asked, and still said undelivered: %+v", got)
|
|
}
|
|
first := r.asksSent(t)[0]
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused,
|
|
Words: "no channel can carry any of its answers now", At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no channel can carry") {
|
|
t.Fatalf("the router's refusal, said as %+v", got)
|
|
}
|
|
if why, ok := conditions.PlainWords(conditions.Words{Headline: last()[0].Headline, Explanation: last()[0].Explanation,
|
|
Needs: last()[0].Needs, Resolved: last()[0].Resolved}, ""); !ok {
|
|
t.Errorf("not plain: %s", why)
|
|
}
|
|
r.open = nil
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Errorf("nothing needs asking, and still said: %+v", got)
|
|
}
|
|
}
|
|
|
|
// The review of 2026-10-09 (M1): an acknowledging answer never shares an ask with an authorising one. A
|
|
// condition offering Restart and Silence is asked twice — Restart alone, about the condition, and Silence
|
|
// alone, apart — so Silence chosen on a channel that only acknowledges leaves the Restart ask open.
|
|
func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
key := "module.shanks.plex.down"
|
|
c := conditions.Condition{Key: key, Kind: "module-down", Severity: conditions.Urgent, Headline: "Plex down on shanks",
|
|
Explanation: "Needs you: restart it, or silence this.", Needs: "restart it, or silence this.",
|
|
Actions: []conditions.Action{
|
|
{Label: "Restart", Verb: "node-service-manager.restart", Machine: "shanks", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "plex"}},
|
|
conditions.SilenceAction(key)}}
|
|
r.open = []conditions.Condition{c}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 2 {
|
|
t.Fatalf("asked %d time(s): %+v", len(sent), sent)
|
|
}
|
|
for _, q := range sent {
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Errorf("%s: %v", q.About, err)
|
|
}
|
|
levels := map[asks.Level]bool{}
|
|
for _, o := range q.Options {
|
|
levels[o.Level] = true
|
|
}
|
|
if len(levels) != 1 {
|
|
t.Errorf("the ask about %s mixes levels: %+v", q.About, q.Options)
|
|
}
|
|
}
|
|
byAbout := map[string]asks.Ask{}
|
|
for _, q := range sent {
|
|
byAbout[q.About] = q
|
|
}
|
|
if q := byAbout[key]; len(q.Options) != 1 || q.Options[0].Label != "Restart" {
|
|
t.Errorf("the condition's own ask: %+v", q)
|
|
}
|
|
if q := byAbout[key+".acknowledge"]; len(q.Options) != 1 || q.Options[0].Level != asks.Acknowledge {
|
|
t.Errorf("the acknowledging ask: %+v", q)
|
|
}
|
|
// Silence chosen: performed, and the Restart ask stays open, never asked twice.
|
|
answerWith(t, r, r.warrantFor(t, key, "Silence for a week"))
|
|
if len(r.silenced) != 1 || len(r.called) != 0 {
|
|
t.Fatalf("silenced %v called %v", r.silenced, r.called)
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
open := 0
|
|
for _, a := range r.store {
|
|
if a.State == askOpen && a.Condition == key {
|
|
open++
|
|
if a.Part != "" || a.Ask.Options[0].Label != "Restart" {
|
|
t.Errorf("the open ask is %+v", a)
|
|
}
|
|
}
|
|
}
|
|
if open != 1 || len(r.asksSent(t)) != 2 {
|
|
t.Errorf("after the silence: %d open, %d asked", open, len(r.asksSent(t)))
|
|
}
|
|
// And the approval still answers: Restart chosen on a channel that proves who answered is performed.
|
|
answerWith(t, r, r.warrantFor(t, key, "Restart"))
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "node-service-manager.restart@shanks") {
|
|
t.Errorf("the approval kept through a silence was not performed: %v", r.called)
|
|
}
|
|
}
|
|
|
|
// novox/hq issue 353: the controller's grant to ask is composed from the router's assignment and reaches the
|
|
// bus only when its machine is pushed. Between the two, the bus refuses every ask (measured 2026-10-09, 17:54 to
|
|
// 17:56 local: seven refusals of mesh.seat.operator-channel.accept.ask.mesh-controller). So nothing is asked
|
|
// while the bus's user list is behind, it is said once, the conditions that need the operator are raised as
|
|
// undelivered with what to do, and the asks go out once the bus holds the grant.
|
|
func TestNothingIsAskedWhileTheBusLacksTheControllersGrant(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var said []string
|
|
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
held := false
|
|
r.a.grantHeld = func(context.Context) (bool, string, error) {
|
|
return held, "the bus's user list on anchor is behind what the mesh composes; `push anchor` carries it", nil
|
|
}
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
_ = r.a.reconcile(context.Background())
|
|
if len(r.asksSent(t)) != 0 {
|
|
t.Error("asked while the bus lacks the grant")
|
|
}
|
|
n := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, "does not hold the controller's grant") {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("said %d times: %q", n, said)
|
|
}
|
|
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
|
!strings.Contains(raised[len(raised)-1][0].Summary, "`push anchor` carries it") ||
|
|
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
|
t.Fatalf("not said as a condition with what to do: %+v", raised)
|
|
}
|
|
held = true
|
|
_ = r.a.reconcile(context.Background())
|
|
if sent := r.asksSent(t); len(sent) != 1 || sent[0].About != heldCondition().Key {
|
|
t.Errorf("not asked once the bus holds the grant: %+v", sent)
|
|
}
|
|
if last := raised[len(raised)-1]; len(last) != 0 {
|
|
t.Errorf("the undelivered condition was not cleared: %+v", last)
|
|
}
|
|
}
|
|
|
|
// novox/hq issue 369: the router refused while its channels had not yet said they could send; they could twenty
|
|
// minutes later, and "Questions for you not delivered" repeated that refusal for eleven hours, escalating on it.
|
|
// The ask is made again; while the router's word on it is awaited the condition stands unchanged (neither
|
|
// cleared nor raised again); once the router took it, the condition clears; a new refusal is said in its own
|
|
// words.
|
|
func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
|
r.a.channels = func(context.Context) string { return "channel/telegram=telegram@anchor[choice]own:true" }
|
|
r.open = []conditions.Condition{unitsCondition()}
|
|
refuse := func(words string) {
|
|
t.Helper()
|
|
sent := r.asksSent(t)
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: sent[len(sent)-1].ID, Asker: "mesh-controller",
|
|
Outcome: asks.OutcomeRefused, Words: words, At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
refuse("no channel can carry any of its answers now: telegram: telegram has not said whether it can send")
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "telegram has not said") {
|
|
t.Fatalf("the refusal, said as %+v", got)
|
|
}
|
|
// The wait ends: asked again; until the router's word on it is in, the condition stands as it was.
|
|
r.now = r.now.Add(askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Fatalf("not asked again: %d asks", n)
|
|
}
|
|
if got := last(); len(got) != 1 {
|
|
t.Fatalf("cleared while the router's word on the new ask was awaited: the condition would flap")
|
|
}
|
|
r.now = r.now.Add(askVerdictWait / 2)
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 {
|
|
t.Fatalf("cleared inside the verdict wait: the condition would flap")
|
|
}
|
|
// Refused again, now for a reason of today: said in those words.
|
|
refuse("no channel can carry any of its answers now: telegram: no account is linked")
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no account is linked") ||
|
|
strings.Contains(got[0].Summary, "has not said") {
|
|
t.Fatalf("an old refusal's words repeated: %+v", got)
|
|
}
|
|
// Asked again after the doubled wait, and taken: no refusal comes, and the condition clears.
|
|
r.now = r.now.Add(2 * askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 3 {
|
|
t.Fatalf("not asked again after the doubled wait: %d asks", n)
|
|
}
|
|
r.now = r.now.Add(askVerdictWait)
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("taken, and still said undelivered: %+v", got)
|
|
}
|
|
if n := len(r.asksSent(t)); n != 3 {
|
|
t.Fatalf("an ask taken was asked again: %d asks", n)
|
|
}
|
|
// The operator answers it; the condition stays open (its fix takes a while): the old refusal is not said again.
|
|
taken := r.asksSent(t)[2]
|
|
r.store.Change(context.Background(), taken.ID, func(x *asked) bool {
|
|
x.State, x.Ended = string(asks.OutcomeChosen), r.now
|
|
return true
|
|
})
|
|
for i := 0; i < 5; i++ {
|
|
r.now = r.now.Add(askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("an answered ask brought its old refusal back: %+v", got)
|
|
}
|
|
}
|
|
// Its explanation opens with one verdict, not two.
|
|
r.open = append(r.open, heldCondition())
|
|
_ = r.a.reconcile(context.Background())
|
|
refuse("no channel can carry any of its answers now")
|
|
_ = r.a.reconcile(context.Background())
|
|
if e := conditions.Verdict(last()[0].Needs, last()[0].Explanation); strings.Count(e, "Needs you") != 1 {
|
|
t.Errorf("the explanation says its verdict twice: %q", e)
|
|
}
|
|
}
|