Ask the operator for a condition's answers and act on the warrant, so release, stop, start and restart can be answered from any channel that proves who answered (hq ADR 0259)
This commit is contained in:
@@ -0,0 +1,478 @@
|
||||
package main
|
||||
|
||||
// The controller asks, and acts on the operator's warrant (novox/hq ADR 0259 §6). It holds no channel, no
|
||||
// identity and no factor: it asks the router like any other module, and performs the answer chosen with its
|
||||
// own grant.
|
||||
//
|
||||
// - **For every open, unsilenced condition that needs the operator and names its answers**, one ask is
|
||||
// published on the `operator-channel` seat under the controller's own name: the condition's words, its
|
||||
// actions as options at their levels (Silence acknowledges; Release, Stop, Start and Restart approve),
|
||||
// answered by the operator, expiring after a day (a week when every option only acknowledges). A
|
||||
// condition that clears, is silenced, or changes its answers has its ask cancelled; an ask that expired
|
||||
// unanswered is asked again while the condition lasts. Each ask is kept in the controller's bucket
|
||||
// `asked`, so a restart neither asks twice nor forgets.
|
||||
// - **On a warrant**, heard on the seat's event under the controller's own name (which only the router may
|
||||
// say), the controller acts once per ask: only for an ask it holds, only for the option it offered at
|
||||
// that option's level, and only while the condition is still open. It performs the action as itself —
|
||||
// a silence through its own conditions, any other through the verb the action names — with the warrant's
|
||||
// words as its why, and records it in the hand-act log as the operator's decision, naming the channel,
|
||||
// the ask and the proofs. An ask that ended without a choice is recorded and nothing is done.
|
||||
// - **A warrant it missed** while away is read from the router's record of its asks, under its own name.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The asker's name on the seat: the controller's module.
|
||||
const askerName = broker.ControllerSeat
|
||||
|
||||
// How long an ask lasts: a day when an answer approves, a week when every answer only acknowledges.
|
||||
const (
|
||||
askApproveFor = 24 * time.Hour
|
||||
askAcknowledgeFor = 7 * 24 * time.Hour
|
||||
// askEvery is how often what is open is asked about again, beside every change.
|
||||
askEvery = time.Minute
|
||||
// askCatchUpAfter is how old an open ask is before the router's record of it is read: a warrant heard
|
||||
// on the event needs no reading.
|
||||
askCatchUpAfter = 2 * time.Minute
|
||||
// askAgainAfterAnswer is how long a condition the operator answered is not asked about again with the
|
||||
// same answers: what was chosen takes a while to clear it, and asking again at once would ask twice.
|
||||
askAgainAfterAnswer = time.Hour
|
||||
)
|
||||
|
||||
// What became of an ask, as the controller keeps it.
|
||||
const (
|
||||
askOpen = "open"
|
||||
askCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// asked is one ask the controller made, as it keeps it.
|
||||
type asked struct {
|
||||
ID string `json:"id"`
|
||||
Condition string `json:"condition"`
|
||||
Ask asks.Ask `json:"ask"`
|
||||
Actions []conditions.Action `json:"actions"`
|
||||
// Options are the actions by option id.
|
||||
Options map[string]int `json:"options"`
|
||||
State string `json:"state"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Ended time.Time `json:"ended,omitempty"`
|
||||
Warrant *asks.Warrant `json:"warrant,omitempty"`
|
||||
// Acted is what the controller did on the warrant: empty before it did anything, "acting" while it acts,
|
||||
// then "done", "failed: …" or "nothing: …". Anything but empty is never acted on again.
|
||||
Acted string `json:"acted,omitempty"`
|
||||
}
|
||||
|
||||
// askedStore keeps the asks (broker.AskedBucket).
|
||||
type askedStore interface {
|
||||
Get(ctx context.Context, id string) (*asked, error)
|
||||
Put(ctx context.Context, a asked) error
|
||||
All(ctx context.Context) ([]asked, error)
|
||||
}
|
||||
|
||||
// asker is the controller asking the operator and acting on the answer.
|
||||
type asker struct {
|
||||
open func(ctx context.Context) ([]conditions.Condition, error)
|
||||
silence func(ctx context.Context, key string, d time.Duration, by, why string) error
|
||||
store askedStore
|
||||
// publish puts a message on a subject's stream, de-duplicated by id.
|
||||
publish func(ctx context.Context, subject string, body []byte, id string) error
|
||||
// call performs an action's verb with its arguments, as the controller.
|
||||
call func(ctx context.Context, a conditions.Action, args map[string]string) error
|
||||
// record writes the hand-act log.
|
||||
record func(ctx context.Context, act link.HandAct) error
|
||||
// routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing.
|
||||
routerRecord func(ctx context.Context, id string) (*asks.Warrant, error)
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func (a *asker) nudge() {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.nudged == nil {
|
||||
a.nudged = make(chan struct{}, 1)
|
||||
}
|
||||
ch := a.nudged
|
||||
a.mu.Unlock()
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep asks until ctx ends: now, on every change of a condition, and every askEvery.
|
||||
func (a *asker) keep(ctx context.Context) {
|
||||
a.nudge()
|
||||
tick := time.NewTicker(askEvery)
|
||||
defer tick.Stop()
|
||||
a.mu.Lock()
|
||||
nudged := a.nudged
|
||||
a.mu.Unlock()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
case <-nudged:
|
||||
}
|
||||
if err := a.reconcile(ctx); err != nil {
|
||||
a.logf("what the operator is asked could not be brought up to date: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wants says whether a condition is one to ask about now.
|
||||
func wants(c conditions.Condition, now time.Time) bool {
|
||||
return len(c.Actions) > 0 && c.Needs != "" && !c.SilencedAt(now)
|
||||
}
|
||||
|
||||
func sameAsked(a []conditions.Action, b []conditions.Action) bool {
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
return string(x) == string(y)
|
||||
}
|
||||
|
||||
// reconcile brings what is asked in line with what is open.
|
||||
func (a *asker) reconcile(ctx context.Context) error {
|
||||
now := a.now()
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
all, err := a.store.All(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition := map[string]asked{}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen {
|
||||
if prior, held := byCondition[r.Condition]; !held || r.Opened.After(prior.Opened) {
|
||||
byCondition[r.Condition] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// A warrant missed while away, read from the router's record.
|
||||
if a.routerRecord != nil {
|
||||
for _, r := range byCondition {
|
||||
if now.Sub(r.Opened) < askCatchUpAfter {
|
||||
continue
|
||||
}
|
||||
if w, err := a.routerRecord(ctx, r.ID); err == nil && w != nil {
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(ctx, body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if all, err = a.store.All(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition = map[string]asked{}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen {
|
||||
byCondition[r.Condition] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// What the operator answered lately, by condition: not asked again at once.
|
||||
answered := map[string]asked{}
|
||||
for _, r := range all {
|
||||
if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer {
|
||||
answered[r.Condition] = r
|
||||
}
|
||||
}
|
||||
wanted := map[string]bool{}
|
||||
sort.Slice(open, func(i, j int) bool { return open[i].Key < open[j].Key })
|
||||
for _, c := range open {
|
||||
if !wants(c, now) {
|
||||
continue
|
||||
}
|
||||
wanted[c.Key] = true
|
||||
if r, done := answered[c.Key]; done && sameAsked(r.Actions, c.Actions) {
|
||||
if _, held := byCondition[c.Key]; !held {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r, held := byCondition[c.Key]; held {
|
||||
switch {
|
||||
case !sameAsked(r.Actions, c.Actions):
|
||||
if err := a.cancel(ctx, r, "its answers changed"); err != nil {
|
||||
return err
|
||||
}
|
||||
case !now.Before(r.Ask.Expires):
|
||||
// Expired unanswered: the router says so too; asked again below while it lasts.
|
||||
r.State, r.Ended = string(asks.OutcomeExpired), now
|
||||
if err := a.store.Put(ctx, r); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := a.ask(ctx, c); err != nil {
|
||||
a.logf("the operator could not be asked about %s: %v", c.Key, err)
|
||||
}
|
||||
}
|
||||
for key, r := range byCondition {
|
||||
if !wanted[key] {
|
||||
if err := a.cancel(ctx, r, "the condition ended, was silenced or needs nothing now"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// optionID is an action's label as an option's id: "Silence for a week" is silence-for-a-week.
|
||||
func optionID(label string) string {
|
||||
var b strings.Builder
|
||||
dash := false
|
||||
for _, r := range strings.ToLower(label) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
dash = false
|
||||
case !dash && b.Len() > 0:
|
||||
b.WriteByte('-')
|
||||
dash = true
|
||||
}
|
||||
}
|
||||
return strings.TrimSuffix(b.String(), "-")
|
||||
}
|
||||
|
||||
// doesWords is what an action does, in the words an option says it with.
|
||||
func doesWords(act conditions.Action) string {
|
||||
switch {
|
||||
case act.Arguments["silence"] != "":
|
||||
return "nothing more is said of it for a week"
|
||||
case act.Verb == "mesh-delivery.release":
|
||||
return "the delivery goes on"
|
||||
case act.Verb == "mesh-delivery.stop":
|
||||
return "the delivery ends"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["go"] != "":
|
||||
return "the delivery starts"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
|
||||
return "the delivery is stopped"
|
||||
case strings.HasSuffix(act.Verb, ".restart"):
|
||||
return "its service is restarted on " + act.Machine
|
||||
}
|
||||
return strings.ToLower(act.Label)
|
||||
}
|
||||
|
||||
// askOf is the ask a condition is asked with.
|
||||
func askOf(id string, c conditions.Condition, now time.Time) (asks.Ask, map[string]int) {
|
||||
q := asks.Ask{ID: id, Headline: c.Headline, Explanation: c.Explanation, Who: asks.Operator,
|
||||
OnExpiry: "nothing is done, and you are asked again while it lasts", About: c.Key,
|
||||
Urgent: c.Severity == conditions.Urgent}
|
||||
options := map[string]int{}
|
||||
approves := false
|
||||
for i, act := range c.Actions {
|
||||
level := asks.Level(act.Level)
|
||||
if level == "" {
|
||||
level = asks.Approve // an action that says nothing of its level is never taken for less
|
||||
}
|
||||
approves = approves || level != asks.Acknowledge
|
||||
oid := optionID(act.Label)
|
||||
options[oid] = i
|
||||
q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesWords(act), Level: level})
|
||||
}
|
||||
q.Expires = now.Add(askAcknowledgeFor)
|
||||
if approves {
|
||||
q.Expires = now.Add(askApproveFor)
|
||||
}
|
||||
return q, options
|
||||
}
|
||||
|
||||
func newAskID() string {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return "c" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// ask publishes one ask about a condition, and keeps it.
|
||||
func (a *asker) ask(ctx context.Context, c conditions.Condition) error {
|
||||
now := a.now()
|
||||
id := newAskID()
|
||||
q, options := askOf(id, c, now)
|
||||
if err := q.Check(now); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.publish(ctx, asks.AskSubject(askerName), body, "ask."+id); err != nil {
|
||||
return err
|
||||
}
|
||||
a.logf("asked the operator about %s (%s): %d answer(s)", c.Key, id, len(q.Options))
|
||||
return a.store.Put(ctx, asked{ID: id, Condition: c.Key, Ask: q, Actions: c.Actions, Options: options,
|
||||
State: askOpen, Opened: now})
|
||||
}
|
||||
|
||||
// cancel takes an ask back.
|
||||
func (a *asker) cancel(ctx context.Context, r asked, why string) error {
|
||||
body, _ := json.Marshal(map[string]string{"id": r.ID})
|
||||
if err := a.publish(ctx, asks.CancelSubject(askerName), body, "cancel."+r.ID); err != nil {
|
||||
a.logf("the ask %s about %s could not be taken back (%v); taken back at the next look", r.ID, r.Condition, err)
|
||||
return nil
|
||||
}
|
||||
a.logf("took back the ask %s about %s: %s", r.ID, r.Condition, why)
|
||||
r.State, r.Ended = askCancelled, a.now()
|
||||
return a.store.Put(ctx, r)
|
||||
}
|
||||
|
||||
// Decided takes the router's word on one of the controller's asks (link.Decider). An error is returned only
|
||||
// when what was decided could not be kept, so the word is held and heard again.
|
||||
func (a *asker) Decided(ctx context.Context, body []byte) error {
|
||||
var w asks.Warrant
|
||||
if err := json.Unmarshal(body, &w); err != nil {
|
||||
a.logf("the router's word on an ask could not be read; ignored: %v", err)
|
||||
return nil
|
||||
}
|
||||
if w.Asker != askerName {
|
||||
a.logf("REFUSED a warrant for %s's ask %s: the controller acts only on its own", w.Asker, w.Ask)
|
||||
return nil
|
||||
}
|
||||
r, err := a.store.Get(ctx, w.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r == nil {
|
||||
a.logf("REFUSED a warrant for the ask %s, which the controller does not hold", w.Ask)
|
||||
return nil
|
||||
}
|
||||
if r.Acted != "" {
|
||||
return nil // heard again: acted on once
|
||||
}
|
||||
now := a.now()
|
||||
if w.Outcome != asks.OutcomeChosen {
|
||||
r.State, r.Ended, r.Warrant = string(w.Outcome), now, &w
|
||||
r.Acted = "nothing: the ask " + string(w.Outcome)
|
||||
if w.Words != "" {
|
||||
r.Acted += ": " + w.Words
|
||||
}
|
||||
a.logf("the ask %s about %s ended %s; nothing is done", r.ID, r.Condition, w.Outcome)
|
||||
return a.store.Put(ctx, *r)
|
||||
}
|
||||
option, err := w.For(askerName, r.Ask)
|
||||
if err != nil {
|
||||
a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err)
|
||||
return nil
|
||||
}
|
||||
index, offered := r.Options[option.ID]
|
||||
if !offered || index >= len(r.Actions) {
|
||||
a.logf("REFUSED a warrant for the ask %s: it chose %s, which no action stands for", r.ID, option.ID)
|
||||
return nil
|
||||
}
|
||||
act := r.Actions[index]
|
||||
r.State, r.Warrant = string(asks.OutcomeChosen), &w
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stillOpen := false
|
||||
for _, c := range open {
|
||||
stillOpen = stillOpen || c.Key == r.Condition
|
||||
}
|
||||
if !stillOpen {
|
||||
// The asker checks the state is still what it asked about before it acts (to-be 46 §10, step 7).
|
||||
r.Ended, r.Acted = now, "nothing: the condition ended before the answer"
|
||||
a.logf("%s, for %s, which ended meanwhile: nothing is done", w.Says(), r.Condition)
|
||||
return a.store.Put(ctx, *r)
|
||||
}
|
||||
r.Acted = "acting"
|
||||
if err := a.store.Put(ctx, *r); err != nil {
|
||||
return err
|
||||
}
|
||||
why := fmt.Sprintf("%s (ask %s)", w.Says(), r.ID)
|
||||
args := map[string]string{}
|
||||
for k, v := range act.Arguments {
|
||||
args[k] = v
|
||||
}
|
||||
if v, takes := args["why"]; takes && v == "" {
|
||||
args["why"] = why
|
||||
}
|
||||
var acted error
|
||||
if act.Arguments["silence"] != "" {
|
||||
acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why)
|
||||
} else {
|
||||
acted = a.call(ctx, act, args)
|
||||
}
|
||||
r.Ended = a.now()
|
||||
r.Acted = "done"
|
||||
if acted != nil {
|
||||
r.Acted = "failed: " + acted.Error()
|
||||
}
|
||||
if err := a.store.Put(ctx, *r); err != nil {
|
||||
return err
|
||||
}
|
||||
verbArgs := []string{act.Verb}
|
||||
if act.Machine != "" {
|
||||
verbArgs = append(verbArgs, "on "+act.Machine)
|
||||
}
|
||||
keys := make([]string, 0, len(args))
|
||||
for k := range args {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if k != "why" {
|
||||
verbArgs = append(verbArgs, k+"="+args[k])
|
||||
}
|
||||
}
|
||||
if err := a.record(ctx, link.HandAct{Verb: handActWarrant, Args: verbArgs, Why: why, By: byWords(w),
|
||||
Cause: conditions.CauseOperatorAnswer, Condition: r.Condition, Via: viaWords(w), Ask: r.ID,
|
||||
Proofs: w.Proofs, RequestedBy: r.Condition, Outcome: r.Acted}); err != nil {
|
||||
a.logf("%s was done, and could NOT be recorded in the hand-act log: %v", why, err)
|
||||
}
|
||||
a.logf("%s: %s", why, r.Acted)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision,
|
||||
// never a repair (handActVerbs).
|
||||
const handActWarrant = "warrant"
|
||||
|
||||
// byWords is who chose, as the hand-act log says it: "the operator, as telegram identity 42".
|
||||
func byWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return "the operator"
|
||||
}
|
||||
return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
|
||||
}
|
||||
|
||||
// viaWords is the channel an answer came through: its module and kind, and how the sender was known.
|
||||
func viaWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return w.Channel
|
||||
}
|
||||
via := w.Channel + " (" + w.By.Kind + ")"
|
||||
if w.By.Verified != "" {
|
||||
via += ", " + w.By.Verified
|
||||
}
|
||||
return via
|
||||
}
|
||||
|
||||
// errNotGranted is an action whose verb the controller's grant does not name.
|
||||
var errNotGranted = errors.New("the controller's grant does not name this verb")
|
||||
@@ -0,0 +1,342 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs
|
||||
// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level.
|
||||
|
||||
type memAskedStore map[string]asked
|
||||
|
||||
func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) {
|
||||
r, ok := m[id]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
func (m memAskedStore) Put(_ context.Context, r asked) error { m[r.ID] = r; return nil }
|
||||
func (m memAskedStore) All(context.Context) ([]asked, error) {
|
||||
var out []asked
|
||||
for _, r := range m {
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type published struct {
|
||||
subject, id string
|
||||
body []byte
|
||||
}
|
||||
|
||||
type askerRig struct {
|
||||
a *asker
|
||||
open []conditions.Condition
|
||||
store memAskedStore
|
||||
sent []published
|
||||
called []string
|
||||
silenced []string
|
||||
acts []link.HandAct
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func newAskerRig(t *testing.T) *askerRig {
|
||||
r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)}
|
||||
r.a = &asker{
|
||||
open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil },
|
||||
silence: func(_ context.Context, key string, d time.Duration, by, why string) error {
|
||||
r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why)
|
||||
return nil
|
||||
},
|
||||
store: r.store,
|
||||
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
||||
r.sent = append(r.sent, published{subject, id, body})
|
||||
return nil
|
||||
},
|
||||
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
||||
raw, _ := json.Marshal(args)
|
||||
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
||||
return nil
|
||||
},
|
||||
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
||||
now: func() time.Time { return r.now },
|
||||
logf: t.Logf,
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func heldCondition() conditions.Condition {
|
||||
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
||||
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
|
||||
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||
}
|
||||
|
||||
func unitsCondition() conditions.Condition {
|
||||
key := "machine.shanks.units"
|
||||
return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning,
|
||||
Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.",
|
||||
Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}}
|
||||
}
|
||||
|
||||
func (r *askerRig) asksSent(t *testing.T) []asks.Ask {
|
||||
t.Helper()
|
||||
var out []asks.Ask
|
||||
for _, p := range r.sent {
|
||||
if p.subject != asks.AskSubject("mesh-controller") {
|
||||
continue
|
||||
}
|
||||
var q asks.Ask
|
||||
if err := json.Unmarshal(p.body, &q); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, q)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"}
|
||||
r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := r.asksSent(t)
|
||||
if len(sent) != 2 {
|
||||
t.Fatalf("asked %d times: %+v", len(sent), sent)
|
||||
}
|
||||
byAbout := map[string]asks.Ask{}
|
||||
for _, q := range sent {
|
||||
byAbout[q.About] = q
|
||||
if err := q.Check(r.now); err != nil {
|
||||
t.Errorf("%s: %v", q.About, err)
|
||||
}
|
||||
}
|
||||
held := byAbout[heldCondition().Key]
|
||||
if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve ||
|
||||
held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator ||
|
||||
held.OnExpiry == "" {
|
||||
t.Errorf("the held delivery is asked %+v", held)
|
||||
}
|
||||
units := byAbout["machine.shanks.units"]
|
||||
if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) {
|
||||
t.Errorf("the failed units are asked %+v", units)
|
||||
}
|
||||
// No second ask while one is open.
|
||||
r.now = r.now.Add(time.Minute)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != 2 {
|
||||
t.Errorf("asked again while open: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition(), unitsCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
// The units are silenced, the held delivery lasts past its ask's day.
|
||||
units := unitsCondition()
|
||||
units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)}
|
||||
r.open = []conditions.Condition{heldCondition(), units}
|
||||
r.now = r.now.Add(askApproveFor)
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var cancels int
|
||||
for _, p := range r.sent {
|
||||
if p.subject == asks.CancelSubject("mesh-controller") {
|
||||
cancels++
|
||||
}
|
||||
}
|
||||
if cancels != 1 {
|
||||
t.Errorf("cancels %d, want the silenced one's", cancels)
|
||||
}
|
||||
if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key {
|
||||
t.Errorf("the expired ask was not asked again: %+v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label.
|
||||
func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant {
|
||||
t.Helper()
|
||||
for _, a := range r.store {
|
||||
if a.Condition != condition || a.State != askOpen {
|
||||
continue
|
||||
}
|
||||
for _, o := range a.Ask.Options {
|
||||
if o.Label == label {
|
||||
return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen,
|
||||
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now,
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("no open ask about %s offers %s", condition, label)
|
||||
return asks.Warrant{}
|
||||
}
|
||||
|
||||
func answerWith(t *testing.T, r *askerRig, w asks.Warrant) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(w)
|
||||
if err := r.a.Decided(context.Background(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarrantIsActedOnOnce(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Release")
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w) // heard again
|
||||
if len(r.called) != 1 {
|
||||
t.Fatalf("called %v", r.called)
|
||||
}
|
||||
want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}`
|
||||
if r.called[0] != want {
|
||||
t.Errorf("called\n %s\nwant\n %s", r.called[0], want)
|
||||
}
|
||||
if len(r.acts) != 1 {
|
||||
t.Fatalf("hand-acts %+v", r.acts)
|
||||
}
|
||||
act := r.acts[0]
|
||||
if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" ||
|
||||
act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" ||
|
||||
act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" {
|
||||
t.Errorf("the hand-act %+v", act)
|
||||
}
|
||||
if !personsDecision(act) {
|
||||
t.Error("an act on a warrant counts as a repair")
|
||||
}
|
||||
if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" {
|
||||
t.Errorf("kept %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
|
||||
for name, change := range map[string]func(*asks.Warrant){
|
||||
"another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" },
|
||||
"an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" },
|
||||
"an option not offered": func(w *asks.Warrant) { w.Option = "delete" },
|
||||
"another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge },
|
||||
"no person": func(w *asks.Warrant) { w.By = nil },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
||||
change(&w)
|
||||
answerWith(t, r, w)
|
||||
if len(r.called)+len(r.acts)+len(r.silenced) != 0 {
|
||||
t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
|
||||
plan := "plan-1791454185265004861"
|
||||
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
||||
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
||||
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
||||
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
||||
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
||||
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
||||
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
||||
for _, tc := range []struct {
|
||||
c conditions.Condition
|
||||
label string
|
||||
want string
|
||||
}{
|
||||
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
|
||||
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
|
||||
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
|
||||
} {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{tc.c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
|
||||
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
|
||||
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{unitsCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, "machine.shanks.units", "Silence for a week")
|
||||
w.Level, w.Proofs = asks.Acknowledge, nil
|
||||
w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14",
|
||||
Verified: "a desk click: whoever was at the operator's session on g14"}
|
||||
w.Channel = "desk-channel"
|
||||
answerWith(t, r, w)
|
||||
if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") {
|
||||
t.Fatalf("silenced %v, called %v", r.silenced, r.called)
|
||||
}
|
||||
if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 {
|
||||
t.Errorf("%+v", r.acts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Release")
|
||||
w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time"
|
||||
answerWith(t, r, w)
|
||||
if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) ||
|
||||
!strings.HasPrefix(r.store[w.Ask].Acted, "nothing") {
|
||||
t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask])
|
||||
}
|
||||
// And a choice for a condition that ended meanwhile does nothing either.
|
||||
r2 := newAskerRig(t)
|
||||
r2.open = []conditions.Condition{heldCondition()}
|
||||
_ = r2.a.reconcile(context.Background())
|
||||
w2 := r2.warrantFor(t, heldCondition().Key, "Release")
|
||||
r2.open = nil
|
||||
answerWith(t, r2, w2)
|
||||
if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" {
|
||||
t.Errorf("%v %+v", r2.called, r2.store[w2.Ask])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, heldCondition().Key, "Stop")
|
||||
r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) {
|
||||
if id != w.Ask {
|
||||
return nil, errors.New("another ask")
|
||||
}
|
||||
return &w, nil
|
||||
}
|
||||
r.now = r.now.Add(askCatchUpAfter)
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") {
|
||||
t.Errorf("called %v", r.called)
|
||||
}
|
||||
if n := len(r.asksSent(t)); n != 1 {
|
||||
t.Errorf("asked again after the answer: %d", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package main
|
||||
|
||||
// The asker on the bus: its asks in the controller's bucket `asked`, its asks and cancels published on the
|
||||
// seat under the controller's name, the verbs a warrant chooses called with the controller's grant, and the
|
||||
// router's record of its asks read under its name (novox/hq ADR 0259).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// askerFrom is the serving controller's asker; nil in any other process.
|
||||
var askerFrom *asker
|
||||
|
||||
// askWithin is how long a verb a warrant chose is given to answer.
|
||||
const askWithin = time.Minute
|
||||
|
||||
type busAsked struct{ conn *nats.Conn }
|
||||
|
||||
func (b busAsked) kv(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
js, err := jetstream.New(b.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return js.KeyValue(ctx, broker.AskedBucket)
|
||||
}
|
||||
|
||||
func (b busAsked) Get(ctx context.Context, id string) (*asked, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := kv.Get(ctx, id)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r asked
|
||||
return &r, json.Unmarshal(e.Value(), &r)
|
||||
}
|
||||
|
||||
func (b busAsked) Put(ctx context.Context, r asked) error {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = kv.Put(ctx, r.ID, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busAsked) All(ctx context.Context) ([]asked, error) {
|
||||
kv, err := b.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = lister.Stop() }()
|
||||
var out []asked
|
||||
for k := range lister.Keys() {
|
||||
e, err := kv.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var r asked
|
||||
if json.Unmarshal(e.Value(), &r) == nil {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// callAction performs an action's verb as the controller, through the grant that names it.
|
||||
func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
return func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
seat, verb, ok := strings.Cut(a.Verb, ".")
|
||||
if !ok {
|
||||
return fmt.Errorf("%q names no seat and verb", a.Verb)
|
||||
}
|
||||
body := map[string]any{}
|
||||
for k, v := range args {
|
||||
body[k] = v
|
||||
}
|
||||
if seat == catalogue.DeliverySeat {
|
||||
_, err := askDeliveryOwner(ctx, conn, verb, body)
|
||||
return err
|
||||
}
|
||||
granted := false
|
||||
for _, v := range broker.VerbsTheControllerActsOnAWarrant {
|
||||
granted = granted || (v.Seat == seat && v.Verb == verb)
|
||||
}
|
||||
if !granted {
|
||||
return fmt.Errorf("%s: %w", a.Verb, errNotGranted)
|
||||
}
|
||||
var answer link.Answer
|
||||
var err error
|
||||
if a.Machine != "" {
|
||||
answer, err = link.AskSeatTool(ctx, conn, seat, verb, a.Machine, body, askWithin)
|
||||
} else {
|
||||
answer, err = link.AskMeshSeatTool(ctx, conn, seat, verb, body, askWithin)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s refused: %s", a.Verb, answer.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// routerRecordOf reads the router's record of one of the controller's asks, under its name, and answers
|
||||
// how it ended when it did: the bucket is the one the asks seat's declarer names as its records.
|
||||
func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.Context, id string) (*asks.Warrant, error) {
|
||||
return func(ctx context.Context, id string) (*asks.Warrant, error) {
|
||||
bucket, err := asksRecords(ctx, inv)
|
||||
if err != nil || bucket == "" {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+askerName+"."+id, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reply.Header.Get("Status") != "" {
|
||||
return nil, nil // none, or not readable: the event says it
|
||||
}
|
||||
var rec struct {
|
||||
State string `json:"state"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
if json.Unmarshal(reply.Data, &rec) != nil || rec.State == "open" || rec.Warrant == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return rec.Warrant, nil
|
||||
}
|
||||
}
|
||||
|
||||
// asksRecords is the bucket the asks seat's declarer keeps its record of asks in.
|
||||
func asksRecords(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
declared, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, m := range declared {
|
||||
for _, s := range m.DefinesSeats {
|
||||
if s.Name == broker.AsksSeat && len(s.Records) > 0 {
|
||||
return broker.BucketName(m.Module, s.Records[0]), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// startAsking makes the serving controller's asker and hands it the router's words.
|
||||
func startAsking(ctx context.Context, open *stores, server *link.Server, conn *nats.Conn, keeper *conditions.Keeper) {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the operator cannot be asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
a := &asker{
|
||||
open: keeper.Open,
|
||||
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
||||
_, err := keeper.Silence(ctx, key, d, by, why)
|
||||
return err
|
||||
},
|
||||
store: busAsked{conn: conn},
|
||||
publish: func(ctx context.Context, subject string, body []byte, id string) error {
|
||||
_, err := js.Publish(ctx, subject, body, jetstream.WithMsgID(id))
|
||||
return err
|
||||
},
|
||||
call: callAction(conn),
|
||||
record: func(ctx context.Context, act link.HandAct) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
},
|
||||
routerRecord: routerRecordOf(conn, open.inventory),
|
||||
now: time.Now,
|
||||
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
}
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error)
|
||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||
// does nothing).
|
||||
Changed: statusFrom.nudge,
|
||||
Changed: func() { statusFrom.nudge(); askerFrom.nudge() },
|
||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
||||
}
|
||||
|
||||
@@ -136,12 +136,13 @@ func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, verb := range []string{"stalled", "close"} {
|
||||
// And release and stop, which the operator's warrant chooses (novox/hq ADR 0259).
|
||||
for _, verb := range []string{"stalled", "close", "release", "stop"} {
|
||||
if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) {
|
||||
t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
}
|
||||
if _, err := askDeliveryOwner(t.Context(), nil, "stop", nil); err == nil || !strings.Contains(err.Error(), "grant") {
|
||||
if _, err := askDeliveryOwner(t.Context(), nil, "retire-history", nil); err == nil || !strings.Contains(err.Error(), "grant") {
|
||||
t.Fatalf("a verb the grant does not name was asked: %v", err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
|
||||
@@ -60,11 +60,18 @@ var handActVerbs = []handActVerb{
|
||||
// a person's word (ADR 0242), which the push itself reads from what it carried (recorded_push.go).
|
||||
{Verb: "push", Decision: "a recorded build moves only by a person's push: that push is the word its " +
|
||||
"upgrade policy asks for (ADR 0242)", DecidedWhen: pushedRecorded},
|
||||
{Verb: "plans stop"},
|
||||
// Stopping or starting a walk the operator chose on a warrant (novox/hq ADR 0259) is their decision.
|
||||
{Verb: "plans stop", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)",
|
||||
DecidedFor: []string{conditions.CauseOperatorAnswer}},
|
||||
{Verb: "plans close"},
|
||||
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
|
||||
// not trusted with it — either is a repair the owner should have made.
|
||||
{Verb: "plans go"},
|
||||
// not trusted with it — either is a repair the owner should have made. Unless the operator chose it on
|
||||
// a warrant (ADR 0259).
|
||||
{Verb: "plans go", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)",
|
||||
DecidedFor: []string{conditions.CauseOperatorAnswer}},
|
||||
// An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a
|
||||
// channel that proved who answered, performed by the controller as itself.
|
||||
{Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"},
|
||||
{Verb: "broker consumer-reset"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
|
||||
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
|
||||
|
||||
@@ -402,6 +402,7 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
|
||||
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
||||
namesWords(plain, 3)),
|
||||
Needs: needs,
|
||||
Actions: moduleActions(node, rs),
|
||||
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
||||
}
|
||||
|
||||
|
||||
@@ -680,13 +680,29 @@ func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severit
|
||||
}
|
||||
|
||||
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
|
||||
// Start and Stop are asked of the operator (novox/hq ADR 0259), so the words do not say where: the router
|
||||
// says where each can be answered.
|
||||
func waitingNeeds(severity conditions.Severity) string {
|
||||
if severity == conditions.Urgent {
|
||||
return "start it, or stop it, " + FromMeshMCPServer
|
||||
return "start it, or stop it."
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// waitingActions are the answers to a walk waiting past its urgent bound: start it, or stop it — the plan's
|
||||
// own verbs, approved by the operator (novox/hq ADR 0259). None before the bound.
|
||||
func waitingActions(plan string, severity conditions.Severity) []conditions.Action {
|
||||
if severity != conditions.Urgent || plan == "" {
|
||||
return nil
|
||||
}
|
||||
return []conditions.Action{
|
||||
{Label: "Start", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"go": plan, "why": "", "cause": conditions.CauseOperatorAnswer}},
|
||||
{Label: "Stop", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"stop": plan, "why": "", "cause": conditions.CauseOperatorAnswer}},
|
||||
}
|
||||
}
|
||||
|
||||
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
|
||||
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
|
||||
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
|
||||
@@ -701,11 +717,35 @@ func moduleNeeds(node string, rs []inventory.ResourceHealth) string {
|
||||
}
|
||||
}
|
||||
if unit != "" {
|
||||
return fmt.Sprintf("restart its service %s on %s %s", unit, node, FromMeshMCPServer)
|
||||
// Asked of the operator (moduleActions): the router says where it can be answered.
|
||||
return fmt.Sprintf("restart its service %s on %s.", unit, node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// moduleActions are the answers to a module unhealthy on a machine: restart its failed service there,
|
||||
// approved by the operator (novox/hq ADR 0259) — none when the mesh restarts it, or a new login is what it
|
||||
// waits for.
|
||||
func moduleActions(node string, rs []inventory.ResourceHealth) []conditions.Action {
|
||||
for _, r := range rs {
|
||||
if strings.Contains(r.Reason, "relogin needed") {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
for _, r := range rs {
|
||||
if r.Kind != link.KindUnit || r.Target == "" {
|
||||
continue
|
||||
}
|
||||
scope := "system"
|
||||
if r.Account != "" {
|
||||
scope = "user"
|
||||
}
|
||||
return []conditions.Action{{Label: "Restart", Verb: "node-service-manager.restart", Machine: node,
|
||||
Level: conditions.LevelApprove, Arguments: map[string]string{"unit": r.Target, "scope": scope}}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FromMeshMCPServer ends what the operator needs when no notification can do it (ADR 0258), naming the mesh MCP
|
||||
// server (the glossary's word; "console" is retired): the answer is not an
|
||||
// acknowledgement, so it is given where the operator is known to be the one asking, until answers are
|
||||
@@ -776,15 +816,19 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
// Words only: releasing or stopping a delivery is not an acknowledgement, so no desk click
|
||||
// performs it (ADR 0258).
|
||||
// Asked of the operator, approved on a channel that proves who answered (novox/hq ADR 0259); the
|
||||
// router says where each can be answered, so the words do not.
|
||||
release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"id": l.ID, "why": ""}}
|
||||
stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"id": l.ID, "why": ""}}
|
||||
switch held {
|
||||
case "held":
|
||||
needs = "release it, or stop it, " + FromMeshMCPServer
|
||||
needs, actions = "release it, or stop it.", []conditions.Action{release, stop}
|
||||
case "ready", "checked":
|
||||
needs = "merge its pull request, or close it."
|
||||
default:
|
||||
needs = "stop it " + FromMeshMCPServer
|
||||
needs, actions = "stop it.", []conditions.Action{stop}
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
||||
|
||||
@@ -65,13 +65,21 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
|
||||
t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary)
|
||||
}
|
||||
|
||||
// Past four hours it is urgent, and offers the controller's own answers.
|
||||
// Past four hours it is urgent, and asks the operator to start or stop it (novox/hq ADR 0259): the plan's
|
||||
// own verbs, approved, which the controller performs on the warrant. The router says where to answer.
|
||||
f.waits[0].since = now.Add(-5 * time.Hour)
|
||||
got = watchWaits(f)
|
||||
plainExample(t, got[0], "openrazer delivery waiting to start",
|
||||
"Needs you: start it, or stop it, from the mesh MCP server; this notification cannot do it. The change to openrazer is merged and built, and mesh-delivery (the "+
|
||||
"Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+
|
||||
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
|
||||
"be stuck.")
|
||||
"be stuck.", "Start", "Stop")
|
||||
for i, want := range []string{"go", "stop"} {
|
||||
a := got[0].Actions[i]
|
||||
if a.Verb != "mesh-controller.plans" || a.Arguments[want] != "plan-1791454185265004861" ||
|
||||
a.Level != conditions.LevelApprove || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
|
||||
t.Errorf("%s: %+v", a.Label, a)
|
||||
}
|
||||
}
|
||||
|
||||
// Many modules are counted, not listed in the headline.
|
||||
f.waits[0].modules = []string{"a", "b", "c", "d"}
|
||||
@@ -82,16 +90,20 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
|
||||
}
|
||||
|
||||
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
|
||||
// account's own service manager (exit-code)". Restarting is not an acknowledgement, so it is said in words
|
||||
// and offered as no answer (ADR 0258).
|
||||
// account's own service manager (exit-code)". Restarting is not an acknowledgement: it is asked of the
|
||||
// operator at the approve level (novox/hq ADR 0259), so a desk click never performs it (ADR 0258).
|
||||
func TestAModuleUnhealthyAsksForARestartInWords(t *testing.T) {
|
||||
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
|
||||
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
|
||||
Resource: "openrazer-daemon", Target: "openrazer-daemon.service", Account: "jochen",
|
||||
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
|
||||
plainExample(t, o, "openrazer not working on g14",
|
||||
"Needs you: restart its service openrazer-daemon on g14 from the mesh MCP server; this notification cannot do it. "+
|
||||
"Needs you: restart its service openrazer-daemon on g14. "+
|
||||
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon "+
|
||||
"as it runs again.")
|
||||
"as it runs again.", "Restart")
|
||||
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" || a.Level != conditions.LevelApprove ||
|
||||
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
|
||||
t.Errorf("restart: %+v", a)
|
||||
}
|
||||
// An account waiting for a new login (ADR 0252) asks for the login, held to the plain rule.
|
||||
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
|
||||
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
|
||||
@@ -153,8 +165,13 @@ func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) {
|
||||
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
||||
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
|
||||
plainExample(t, got[0], "Delivery of hq held for 36 hours",
|
||||
"Needs you: release it, or stop it, from the mesh MCP server; this notification cannot do it. A delivery of hq has been held for 36 hours, past its limit.",
|
||||
)
|
||||
"Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
|
||||
"Release", "Stop")
|
||||
for i, verb := range []string{"mesh-delivery.release", "mesh-delivery.stop"} {
|
||||
if a := got[0].Actions[i]; a.Verb != verb || a.Arguments["id"] != "novox/hq@055550802096" || a.Level != conditions.LevelApprove {
|
||||
t.Errorf("%+v", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
|
||||
|
||||
@@ -378,6 +378,7 @@ func watchWaits(f *signalFacts) []conditions.Observation {
|
||||
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
|
||||
Explanation: walkWaitingWords(w, in, severity),
|
||||
Needs: waitingNeeds(severity),
|
||||
Actions: waitingActions(w.id, severity),
|
||||
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -700,6 +700,9 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
|
||||
// under the lease and the brake, every act said.
|
||||
healers := newHealing(open, keeper, bus, server.JetStream())
|
||||
go healers.keep(watching)
|
||||
// And the asker (novox/hq ADR 0259): what needs the operator and names its answers is asked of them,
|
||||
// and the answer chosen is performed on its warrant.
|
||||
startAsking(watching, open, server, bus.Conn, keeper)
|
||||
go forgettingOldHeals(watching, open.inventory)
|
||||
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||
|
||||
Reference in New Issue
Block a user