jschoubben
74d35600a6
Choosing Silence silenced the condition, the condition was no longer wanted, and the next reconcile cancelled the Restart or Release ask beside it: an acknowledgement, which any desk click may give, took an approval back. An open approval ask now stays until it is answered or expires while its condition is open and silenced with the same answers. The test silences as the controller does; it failed before (0 open) and passes, and the kept Restart is performed on its warrant.
657 lines
25 KiB
Go
657 lines
25 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs
|
|
// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level.
|
|
|
|
type memAskedStore map[string]asked
|
|
|
|
// memAskedMu guards every memAskedStore: Change is a compare-and-set as the bus's is.
|
|
var memAskedMu sync.Mutex
|
|
|
|
func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
r, ok := m[id]
|
|
if !ok {
|
|
return nil, nil
|
|
}
|
|
return &r, nil
|
|
}
|
|
func (m memAskedStore) Create(_ context.Context, r asked) error {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
if _, kept := m[r.ID]; kept {
|
|
return errors.New("an ask is kept under that id")
|
|
}
|
|
m[r.ID] = r
|
|
return nil
|
|
}
|
|
func (m memAskedStore) Change(_ context.Context, id string, change func(*asked) bool) (bool, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
r, ok := m[id]
|
|
if !ok || !change(&r) {
|
|
return false, nil
|
|
}
|
|
m[id] = r
|
|
return true, nil
|
|
}
|
|
func (m memAskedStore) All(context.Context) ([]asked, error) {
|
|
memAskedMu.Lock()
|
|
defer memAskedMu.Unlock()
|
|
var out []asked
|
|
for _, r := range m {
|
|
out = append(out, r)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
type published struct {
|
|
subject, id string
|
|
body []byte
|
|
}
|
|
|
|
type askerRig struct {
|
|
a *asker
|
|
open []conditions.Condition
|
|
store memAskedStore
|
|
sent []published
|
|
called []string
|
|
silenced []string
|
|
acts []link.HandAct
|
|
now time.Time
|
|
}
|
|
|
|
func newAskerRig(t *testing.T) *askerRig {
|
|
r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)}
|
|
r.a = &asker{
|
|
open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil },
|
|
silence: func(_ context.Context, key string, d time.Duration, by, why string) error {
|
|
r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why)
|
|
// As the controller's conditions do (the confirmation review of 2026-10-09, M1): the condition is
|
|
// silenced from now on, so what is asked next sees it silenced.
|
|
for i := range r.open {
|
|
if r.open[i].Key == key {
|
|
r.open[i].Silenced = &conditions.Silence{Until: r.now.Add(d), By: by, Why: why, Since: r.now}
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
store: r.store,
|
|
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
|
r.sent = append(r.sent, published{subject, id, body})
|
|
return nil
|
|
},
|
|
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
|
raw, _ := json.Marshal(args)
|
|
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
|
return nil
|
|
},
|
|
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
|
now: func() time.Time { return r.now },
|
|
logf: t.Logf,
|
|
}
|
|
return r
|
|
}
|
|
|
|
func heldCondition() conditions.Condition {
|
|
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
|
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
|
|
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
|
}
|
|
|
|
func unitsCondition() conditions.Condition {
|
|
key := "machine.shanks.units"
|
|
return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning,
|
|
Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.",
|
|
Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}}
|
|
}
|
|
|
|
func (r *askerRig) asksSent(t *testing.T) []asks.Ask {
|
|
t.Helper()
|
|
var out []asks.Ask
|
|
for _, p := range r.sent {
|
|
if p.subject != asks.AskSubject("mesh-controller") {
|
|
continue
|
|
}
|
|
var q asks.Ask
|
|
if err := json.Unmarshal(p.body, &q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out = append(out, q)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"}
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 2 {
|
|
t.Fatalf("asked %d times: %+v", len(sent), sent)
|
|
}
|
|
byAbout := map[string]asks.Ask{}
|
|
for _, q := range sent {
|
|
byAbout[q.About] = q
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Errorf("%s: %v", q.About, err)
|
|
}
|
|
}
|
|
held := byAbout[heldCondition().Key]
|
|
if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve ||
|
|
held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator ||
|
|
held.OnExpiry == "" {
|
|
t.Errorf("the held delivery is asked %+v", held)
|
|
}
|
|
units := byAbout["machine.shanks.units"]
|
|
if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) {
|
|
t.Errorf("the failed units are asked %+v", units)
|
|
}
|
|
// No second ask while one is open.
|
|
r.now = r.now.Add(time.Minute)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Errorf("asked again while open: %d", n)
|
|
}
|
|
}
|
|
|
|
func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
// The units are silenced, the held delivery lasts past its ask's day.
|
|
units := unitsCondition()
|
|
units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)}
|
|
r.open = []conditions.Condition{heldCondition(), units}
|
|
r.now = r.now.Add(askApproveFor)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cancels int
|
|
for _, p := range r.sent {
|
|
if p.subject == asks.CancelSubject("mesh-controller") {
|
|
cancels++
|
|
}
|
|
}
|
|
if cancels != 1 {
|
|
t.Errorf("cancels %d, want the silenced one's", cancels)
|
|
}
|
|
if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key {
|
|
t.Errorf("the expired ask was not asked again: %+v", sent)
|
|
}
|
|
}
|
|
|
|
// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label.
|
|
func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant {
|
|
t.Helper()
|
|
for _, a := range r.store {
|
|
if a.Condition != condition || a.State != askOpen {
|
|
continue
|
|
}
|
|
for _, o := range a.Ask.Options {
|
|
if o.Label == label {
|
|
return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen,
|
|
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now,
|
|
AskDigest: a.Ask.Digest(),
|
|
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("no open ask about %s offers %s", condition, label)
|
|
return asks.Warrant{}
|
|
}
|
|
|
|
func answerWith(t *testing.T, r *askerRig, w asks.Warrant) {
|
|
t.Helper()
|
|
body, _ := json.Marshal(w)
|
|
if err := r.a.Decided(context.Background(), body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantIsActedOnOnce(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
answerWith(t, r, w)
|
|
answerWith(t, r, w) // heard again
|
|
if len(r.called) != 1 {
|
|
t.Fatalf("called %v", r.called)
|
|
}
|
|
want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}`
|
|
if r.called[0] != want {
|
|
t.Errorf("called\n %s\nwant\n %s", r.called[0], want)
|
|
}
|
|
if len(r.acts) != 1 {
|
|
t.Fatalf("hand-acts %+v", r.acts)
|
|
}
|
|
act := r.acts[0]
|
|
if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" ||
|
|
act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" ||
|
|
act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" {
|
|
t.Errorf("the hand-act %+v", act)
|
|
}
|
|
if !personsDecision(act) {
|
|
t.Error("an act on a warrant counts as a repair")
|
|
}
|
|
if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
|
t.Errorf("kept %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
|
|
for name, change := range map[string]func(*asks.Warrant){
|
|
"another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" },
|
|
"an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" },
|
|
"an option not offered": func(w *asks.Warrant) { w.Option = "delete" },
|
|
"another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge },
|
|
"no person": func(w *asks.Warrant) { w.By = nil },
|
|
"another ask's digest": func(w *asks.Warrant) { w.AskDigest = "sha256:0000" },
|
|
"no ask's digest": func(w *asks.Warrant) { w.AskDigest = "" },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
change(&w)
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts)+len(r.silenced) != 0 {
|
|
t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
|
|
plan := "plan-1791454185265004861"
|
|
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
|
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
|
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
|
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
|
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
|
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
|
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
|
for _, tc := range []struct {
|
|
c conditions.Condition
|
|
label string
|
|
want string
|
|
}{
|
|
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
|
|
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
|
|
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
|
|
} {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{tc.c}
|
|
_ = r.a.reconcile(context.Background())
|
|
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
|
|
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, "machine.shanks.units", "Silence for a week")
|
|
w.Level, w.Proofs = asks.Acknowledge, nil
|
|
w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14",
|
|
Verified: "a desk click: whoever was at the operator's session on g14"}
|
|
w.Channel = "desk-channel"
|
|
answerWith(t, r, w)
|
|
if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") {
|
|
t.Fatalf("silenced %v, called %v", r.silenced, r.called)
|
|
}
|
|
if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 {
|
|
t.Errorf("%+v", r.acts)
|
|
}
|
|
}
|
|
|
|
func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time"
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) ||
|
|
!strings.HasPrefix(r.store[w.Ask].Acted, "nothing") {
|
|
t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask])
|
|
}
|
|
// And a choice for a condition that ended meanwhile does nothing either.
|
|
r2 := newAskerRig(t)
|
|
r2.open = []conditions.Condition{heldCondition()}
|
|
_ = r2.a.reconcile(context.Background())
|
|
w2 := r2.warrantFor(t, heldCondition().Key, "Release")
|
|
r2.open = nil
|
|
answerWith(t, r2, w2)
|
|
if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" {
|
|
t.Errorf("%v %+v", r2.called, r2.store[w2.Ask])
|
|
}
|
|
}
|
|
|
|
func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) {
|
|
if id != w.Ask {
|
|
return nil, errors.New("another ask")
|
|
}
|
|
return &w, nil
|
|
}
|
|
r.now = r.now.Add(askCatchUpAfter)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") {
|
|
t.Errorf("called %v", r.called)
|
|
}
|
|
if n := len(r.asksSent(t)); n != 1 {
|
|
t.Errorf("asked again after the answer: %d", n)
|
|
}
|
|
}
|
|
|
|
// After review (2026-10-08): a refused ask is not asked again until its answers or the channels change.
|
|
func TestAnAskTheRouterRefusedWaitsUntilSomethingChanges(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
|
r.a.channels = func(context.Context) string { return channels }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
first := r.asksSent(t)[0]
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused,
|
|
Words: "no channel can carry any of its answers now", At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") {
|
|
t.Fatalf("the refusal was kept as %+v", got)
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
r.now = r.now.Add(askEvery)
|
|
_ = r.a.reconcile(context.Background())
|
|
}
|
|
if n := len(r.asksSent(t)); n != 1 {
|
|
t.Fatalf("asked again %d time(s) though nothing changed", n-1)
|
|
}
|
|
channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true"
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Errorf("not asked again once the channels changed: %d", n)
|
|
}
|
|
}
|
|
|
|
// After review: at most three asks open at once, the most urgent first, then the oldest.
|
|
func TestAtMostThreeAsksAreOpenTheMostUrgentFirst(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var open []conditions.Condition
|
|
for i := 0; i < 4; i++ {
|
|
c := unitsCondition()
|
|
c.Key = "machine.m" + string(rune('a'+i)) + ".units"
|
|
c.Actions = []conditions.Action{conditions.SilenceAction(c.Key)}
|
|
c.Raised = r.now.Add(-time.Duration(10-i) * time.Hour)
|
|
open = append(open, c)
|
|
}
|
|
urgent := heldCondition()
|
|
urgent.Severity, urgent.Raised = conditions.Urgent, r.now.Add(-time.Minute)
|
|
r.open = append(open, urgent)
|
|
_ = r.a.reconcile(context.Background())
|
|
sent := r.asksSent(t)
|
|
if len(sent) != askMostOpen || sent[0].About != urgent.Key || sent[1].About != "machine.ma.units" || sent[2].About != "machine.mb.units" {
|
|
var about []string
|
|
for _, q := range sent {
|
|
about = append(about, q.About)
|
|
}
|
|
t.Fatalf("asked %v", about)
|
|
}
|
|
}
|
|
|
|
// After review: nothing is asked while no router takes asks under the controller's name, and that is said once.
|
|
func TestNothingIsAskedWithoutARouter(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var said []string
|
|
r.a.logf = func(f string, a ...any) { said = append(said, f) }
|
|
r.a.routerHere = func(context.Context) (bool, error) { return false, nil }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
_ = r.a.reconcile(context.Background())
|
|
if len(r.asksSent(t)) != 0 {
|
|
t.Error("asked with no router")
|
|
}
|
|
n := 0
|
|
for _, s := range said {
|
|
if strings.Contains(s, "no router takes asks") {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("said %d times", n)
|
|
}
|
|
}
|
|
|
|
// After review: the condition's words keep where an answer is given without a channel; the ask's text does not.
|
|
func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) {
|
|
c := heldCondition()
|
|
if !strings.Contains(c.Explanation, FromMeshMCPServer) {
|
|
t.Fatalf("the condition lost where it is answered: %q", c.Explanation)
|
|
}
|
|
q, _ := askOf("x", c, partsOf(c)[0], time.Now())
|
|
if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") {
|
|
t.Errorf("the ask says %q", q.Explanation)
|
|
}
|
|
if askApproveFor >= 24*time.Hour {
|
|
t.Errorf("an approving ask lasts %s, which the SDK may refuse at its bound", askApproveFor)
|
|
}
|
|
}
|
|
|
|
// After review (security finding 9): a warrant is acted on only for an ask open in the controller's own record,
|
|
// once the claim stands, and never when given after the ask expired.
|
|
func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
late := w
|
|
late.At = r.store[w.Ask].Ask.Expires.Add(time.Minute)
|
|
body, _ := json.Marshal(late)
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Fatalf("acted on a warrant given after the ask expired: %v", r.called)
|
|
}
|
|
// Claimed already by another delivery: nothing done here.
|
|
kept := r.store[w.Ask]
|
|
kept.Acted = "acting"
|
|
r.store[w.Ask] = kept
|
|
body, _ = json.Marshal(w)
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Fatalf("acted though the claim was another's: %v", r.called)
|
|
}
|
|
// Cancelled in its own record: refused.
|
|
kept.Acted, kept.State = "", askCancelled
|
|
r.store[w.Ask] = kept
|
|
_ = r.a.Decided(context.Background(), body)
|
|
if len(r.called) != 0 {
|
|
t.Errorf("acted on a cancelled ask: %v", r.called)
|
|
}
|
|
}
|
|
|
|
// novox/hq ADR 0259 §6: a warrant authorises the act its option bound when the controller asked, and no
|
|
// other. A record of the act changed after the ask — another delivery, another machine, another argument —
|
|
// is refused and nothing is performed.
|
|
func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) {
|
|
for name, change := range map[string]func(*conditions.Action){
|
|
"another argument": func(a *conditions.Action) {
|
|
a.Arguments = map[string]string{"id": "novox/mesh-controller@000000000000"}
|
|
},
|
|
"another verb": func(a *conditions.Action) { a.Verb = "mesh-delivery.stop" },
|
|
"another machine": func(a *conditions.Action) { a.Machine = "anchor" },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
kept := r.store[w.Ask]
|
|
acts := append([]conditions.Action(nil), kept.Actions...)
|
|
i := kept.Options[w.Option]
|
|
change(&acts[i])
|
|
kept.Actions = acts
|
|
r.store[w.Ask] = kept
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts) != 0 {
|
|
t.Errorf("performed an act the option did not bind: %v %v", r.called, r.acts)
|
|
}
|
|
})
|
|
}
|
|
// Every option of an ask binds its act.
|
|
q, _ := askOf("x", heldCondition(), partsOf(heldCondition())[0], time.Now())
|
|
for _, o := range q.Options {
|
|
if o.Binds == "" {
|
|
t.Errorf("the option %s binds nothing", o.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Failure is loud (novox/hq ADR 0259, the self-review of 2026-10-09): a condition that needs the operator and
|
|
// could not be asked on any channel — no router, or the router refused the ask — is a condition of its own,
|
|
// cleared once it can be asked again.
|
|
func TestAnAskThatCannotBeDeliveredIsSaid(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
var raised [][]conditions.Observation
|
|
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
|
raised = append(raised, obs)
|
|
return nil
|
|
}
|
|
last := func() []conditions.Observation { return raised[len(raised)-1] }
|
|
routerHere := false
|
|
r.a.routerHere = func(context.Context) (bool, error) { return routerHere, nil }
|
|
channels := "channel/telegram=telegram@anchor[choice]own:true"
|
|
r.a.channels = func(context.Context) string { return channels }
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || got[0].Kind != "asks-undelivered" ||
|
|
!strings.Contains(got[0].Summary, heldCondition().Key) || !strings.Contains(got[0].Summary, "no router") {
|
|
t.Fatalf("no router, said as %+v", got)
|
|
}
|
|
|
|
routerHere = true
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Fatalf("asked, and still said undelivered: %+v", got)
|
|
}
|
|
first := r.asksSent(t)[0]
|
|
refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused,
|
|
Words: "no channel can carry any of its answers now", At: r.now})
|
|
if err := r.a.Decided(context.Background(), refusal); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no channel can carry") {
|
|
t.Fatalf("the router's refusal, said as %+v", got)
|
|
}
|
|
if why, ok := conditions.PlainWords(conditions.Words{Headline: last()[0].Headline, Explanation: last()[0].Explanation,
|
|
Needs: last()[0].Needs, Resolved: last()[0].Resolved}, ""); !ok {
|
|
t.Errorf("not plain: %s", why)
|
|
}
|
|
r.open = nil
|
|
_ = r.a.reconcile(context.Background())
|
|
if got := last(); len(got) != 0 {
|
|
t.Errorf("nothing needs asking, and still said: %+v", got)
|
|
}
|
|
}
|
|
|
|
// The review of 2026-10-09 (M1): an acknowledging answer never shares an ask with an authorising one. A
|
|
// condition offering Restart and Silence is asked twice — Restart alone, about the condition, and Silence
|
|
// alone, apart — so Silence chosen on a channel that only acknowledges leaves the Restart ask open.
|
|
func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
key := "module.shanks.plex.down"
|
|
c := conditions.Condition{Key: key, Kind: "module-down", Severity: conditions.Urgent, Headline: "Plex down on shanks",
|
|
Explanation: "Needs you: restart it, or silence this.", Needs: "restart it, or silence this.",
|
|
Actions: []conditions.Action{
|
|
{Label: "Restart", Verb: "node-service-manager.restart", Machine: "shanks", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "plex"}},
|
|
conditions.SilenceAction(key)}}
|
|
r.open = []conditions.Condition{c}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 2 {
|
|
t.Fatalf("asked %d time(s): %+v", len(sent), sent)
|
|
}
|
|
for _, q := range sent {
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Errorf("%s: %v", q.About, err)
|
|
}
|
|
levels := map[asks.Level]bool{}
|
|
for _, o := range q.Options {
|
|
levels[o.Level] = true
|
|
}
|
|
if len(levels) != 1 {
|
|
t.Errorf("the ask about %s mixes levels: %+v", q.About, q.Options)
|
|
}
|
|
}
|
|
byAbout := map[string]asks.Ask{}
|
|
for _, q := range sent {
|
|
byAbout[q.About] = q
|
|
}
|
|
if q := byAbout[key]; len(q.Options) != 1 || q.Options[0].Label != "Restart" {
|
|
t.Errorf("the condition's own ask: %+v", q)
|
|
}
|
|
if q := byAbout[key+".acknowledge"]; len(q.Options) != 1 || q.Options[0].Level != asks.Acknowledge {
|
|
t.Errorf("the acknowledging ask: %+v", q)
|
|
}
|
|
// Silence chosen: performed, and the Restart ask stays open, never asked twice.
|
|
answerWith(t, r, r.warrantFor(t, key, "Silence for a week"))
|
|
if len(r.silenced) != 1 || len(r.called) != 0 {
|
|
t.Fatalf("silenced %v called %v", r.silenced, r.called)
|
|
}
|
|
_ = r.a.reconcile(context.Background())
|
|
open := 0
|
|
for _, a := range r.store {
|
|
if a.State == askOpen && a.Condition == key {
|
|
open++
|
|
if a.Part != "" || a.Ask.Options[0].Label != "Restart" {
|
|
t.Errorf("the open ask is %+v", a)
|
|
}
|
|
}
|
|
}
|
|
if open != 1 || len(r.asksSent(t)) != 2 {
|
|
t.Errorf("after the silence: %d open, %d asked", open, len(r.asksSent(t)))
|
|
}
|
|
// And the approval still answers: Restart chosen on a channel that proves who answered is performed.
|
|
answerWith(t, r, r.warrantFor(t, key, "Restart"))
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "node-service-manager.restart@shanks") {
|
|
t.Errorf("the approval kept through a silence was not performed: %v", r.called)
|
|
}
|
|
}
|