343 lines
13 KiB
Go
343 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs
|
|
// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level.
|
|
|
|
type memAskedStore map[string]asked
|
|
|
|
func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) {
|
|
r, ok := m[id]
|
|
if !ok {
|
|
return nil, nil
|
|
}
|
|
return &r, nil
|
|
}
|
|
func (m memAskedStore) Put(_ context.Context, r asked) error { m[r.ID] = r; return nil }
|
|
func (m memAskedStore) All(context.Context) ([]asked, error) {
|
|
var out []asked
|
|
for _, r := range m {
|
|
out = append(out, r)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
type published struct {
|
|
subject, id string
|
|
body []byte
|
|
}
|
|
|
|
type askerRig struct {
|
|
a *asker
|
|
open []conditions.Condition
|
|
store memAskedStore
|
|
sent []published
|
|
called []string
|
|
silenced []string
|
|
acts []link.HandAct
|
|
now time.Time
|
|
}
|
|
|
|
func newAskerRig(t *testing.T) *askerRig {
|
|
r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)}
|
|
r.a = &asker{
|
|
open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil },
|
|
silence: func(_ context.Context, key string, d time.Duration, by, why string) error {
|
|
r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why)
|
|
return nil
|
|
},
|
|
store: r.store,
|
|
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
|
r.sent = append(r.sent, published{subject, id, body})
|
|
return nil
|
|
},
|
|
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
|
raw, _ := json.Marshal(args)
|
|
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
|
return nil
|
|
},
|
|
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
|
now: func() time.Time { return r.now },
|
|
logf: t.Logf,
|
|
}
|
|
return r
|
|
}
|
|
|
|
func heldCondition() conditions.Condition {
|
|
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
|
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
|
|
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
|
}
|
|
|
|
func unitsCondition() conditions.Condition {
|
|
key := "machine.shanks.units"
|
|
return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning,
|
|
Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.",
|
|
Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}}
|
|
}
|
|
|
|
func (r *askerRig) asksSent(t *testing.T) []asks.Ask {
|
|
t.Helper()
|
|
var out []asks.Ask
|
|
for _, p := range r.sent {
|
|
if p.subject != asks.AskSubject("mesh-controller") {
|
|
continue
|
|
}
|
|
var q asks.Ask
|
|
if err := json.Unmarshal(p.body, &q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out = append(out, q)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"}
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 2 {
|
|
t.Fatalf("asked %d times: %+v", len(sent), sent)
|
|
}
|
|
byAbout := map[string]asks.Ask{}
|
|
for _, q := range sent {
|
|
byAbout[q.About] = q
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Errorf("%s: %v", q.About, err)
|
|
}
|
|
}
|
|
held := byAbout[heldCondition().Key]
|
|
if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve ||
|
|
held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator ||
|
|
held.OnExpiry == "" {
|
|
t.Errorf("the held delivery is asked %+v", held)
|
|
}
|
|
units := byAbout["machine.shanks.units"]
|
|
if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) {
|
|
t.Errorf("the failed units are asked %+v", units)
|
|
}
|
|
// No second ask while one is open.
|
|
r.now = r.now.Add(time.Minute)
|
|
_ = r.a.reconcile(context.Background())
|
|
if n := len(r.asksSent(t)); n != 2 {
|
|
t.Errorf("asked again while open: %d", n)
|
|
}
|
|
}
|
|
|
|
func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition(), unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
// The units are silenced, the held delivery lasts past its ask's day.
|
|
units := unitsCondition()
|
|
units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)}
|
|
r.open = []conditions.Condition{heldCondition(), units}
|
|
r.now = r.now.Add(askApproveFor)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cancels int
|
|
for _, p := range r.sent {
|
|
if p.subject == asks.CancelSubject("mesh-controller") {
|
|
cancels++
|
|
}
|
|
}
|
|
if cancels != 1 {
|
|
t.Errorf("cancels %d, want the silenced one's", cancels)
|
|
}
|
|
if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key {
|
|
t.Errorf("the expired ask was not asked again: %+v", sent)
|
|
}
|
|
}
|
|
|
|
// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label.
|
|
func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant {
|
|
t.Helper()
|
|
for _, a := range r.store {
|
|
if a.Condition != condition || a.State != askOpen {
|
|
continue
|
|
}
|
|
for _, o := range a.Ask.Options {
|
|
if o.Label == label {
|
|
return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen,
|
|
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now,
|
|
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("no open ask about %s offers %s", condition, label)
|
|
return asks.Warrant{}
|
|
}
|
|
|
|
func answerWith(t *testing.T, r *askerRig, w asks.Warrant) {
|
|
t.Helper()
|
|
body, _ := json.Marshal(w)
|
|
if err := r.a.Decided(context.Background(), body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantIsActedOnOnce(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
answerWith(t, r, w)
|
|
answerWith(t, r, w) // heard again
|
|
if len(r.called) != 1 {
|
|
t.Fatalf("called %v", r.called)
|
|
}
|
|
want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}`
|
|
if r.called[0] != want {
|
|
t.Errorf("called\n %s\nwant\n %s", r.called[0], want)
|
|
}
|
|
if len(r.acts) != 1 {
|
|
t.Fatalf("hand-acts %+v", r.acts)
|
|
}
|
|
act := r.acts[0]
|
|
if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" ||
|
|
act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" ||
|
|
act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" {
|
|
t.Errorf("the hand-act %+v", act)
|
|
}
|
|
if !personsDecision(act) {
|
|
t.Error("an act on a warrant counts as a repair")
|
|
}
|
|
if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
|
t.Errorf("kept %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
|
|
for name, change := range map[string]func(*asks.Warrant){
|
|
"another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" },
|
|
"an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" },
|
|
"an option not offered": func(w *asks.Warrant) { w.Option = "delete" },
|
|
"another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge },
|
|
"no person": func(w *asks.Warrant) { w.By = nil },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
change(&w)
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts)+len(r.silenced) != 0 {
|
|
t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
|
|
plan := "plan-1791454185265004861"
|
|
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
|
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
|
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
|
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
|
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
|
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
|
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
|
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
|
for _, tc := range []struct {
|
|
c conditions.Condition
|
|
label string
|
|
want string
|
|
}{
|
|
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
|
|
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
|
|
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
|
|
} {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{tc.c}
|
|
_ = r.a.reconcile(context.Background())
|
|
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
|
|
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{unitsCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, "machine.shanks.units", "Silence for a week")
|
|
w.Level, w.Proofs = asks.Acknowledge, nil
|
|
w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14",
|
|
Verified: "a desk click: whoever was at the operator's session on g14"}
|
|
w.Channel = "desk-channel"
|
|
answerWith(t, r, w)
|
|
if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") {
|
|
t.Fatalf("silenced %v, called %v", r.silenced, r.called)
|
|
}
|
|
if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 {
|
|
t.Errorf("%+v", r.acts)
|
|
}
|
|
}
|
|
|
|
func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Release")
|
|
w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time"
|
|
answerWith(t, r, w)
|
|
if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) ||
|
|
!strings.HasPrefix(r.store[w.Ask].Acted, "nothing") {
|
|
t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask])
|
|
}
|
|
// And a choice for a condition that ended meanwhile does nothing either.
|
|
r2 := newAskerRig(t)
|
|
r2.open = []conditions.Condition{heldCondition()}
|
|
_ = r2.a.reconcile(context.Background())
|
|
w2 := r2.warrantFor(t, heldCondition().Key, "Release")
|
|
r2.open = nil
|
|
answerWith(t, r2, w2)
|
|
if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" {
|
|
t.Errorf("%v %+v", r2.called, r2.store[w2.Ask])
|
|
}
|
|
}
|
|
|
|
func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) {
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{heldCondition()}
|
|
_ = r.a.reconcile(context.Background())
|
|
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
|
r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) {
|
|
if id != w.Ask {
|
|
return nil, errors.New("another ask")
|
|
}
|
|
return &w, nil
|
|
}
|
|
r.now = r.now.Add(askCatchUpAfter)
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") {
|
|
t.Errorf("called %v", r.called)
|
|
}
|
|
if n := len(r.asksSent(t)); n != 1 {
|
|
t.Errorf("asked again after the answer: %d", n)
|
|
}
|
|
}
|