Ask the operator for a condition's answers and act on the warrant, so release, stop, start and restart can be answered from any channel that proves who answered (hq ADR 0259)
This commit is contained in:
jochen
@@ -0,0 +1,478 @@
|
||||
package main
|
||||
|
||||
// The controller asks, and acts on the operator's warrant (novox/hq ADR 0259 §6). It holds no channel, no
|
||||
// identity and no factor: it asks the router like any other module, and performs the answer chosen with its
|
||||
// own grant.
|
||||
//
|
||||
// - **For every open, unsilenced condition that needs the operator and names its answers**, one ask is
|
||||
// published on the `operator-channel` seat under the controller's own name: the condition's words, its
|
||||
// actions as options at their levels (Silence acknowledges; Release, Stop, Start and Restart approve),
|
||||
// answered by the operator, expiring after a day (a week when every option only acknowledges). A
|
||||
// condition that clears, is silenced, or changes its answers has its ask cancelled; an ask that expired
|
||||
// unanswered is asked again while the condition lasts. Each ask is kept in the controller's bucket
|
||||
// `asked`, so a restart neither asks twice nor forgets.
|
||||
// - **On a warrant**, heard on the seat's event under the controller's own name (which only the router may
|
||||
// say), the controller acts once per ask: only for an ask it holds, only for the option it offered at
|
||||
// that option's level, and only while the condition is still open. It performs the action as itself —
|
||||
// a silence through its own conditions, any other through the verb the action names — with the warrant's
|
||||
// words as its why, and records it in the hand-act log as the operator's decision, naming the channel,
|
||||
// the ask and the proofs. An ask that ended without a choice is recorded and nothing is done.
|
||||
// - **A warrant it missed** while away is read from the router's record of its asks, under its own name.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The asker's name on the seat: the controller's module.
|
||||
const askerName = broker.ControllerSeat
|
||||
|
||||
// How long an ask lasts: a day when an answer approves, a week when every answer only acknowledges.
|
||||
const (
|
||||
askApproveFor = 24 * time.Hour
|
||||
askAcknowledgeFor = 7 * 24 * time.Hour
|
||||
// askEvery is how often what is open is asked about again, beside every change.
|
||||
askEvery = time.Minute
|
||||
// askCatchUpAfter is how old an open ask is before the router's record of it is read: a warrant heard
|
||||
// on the event needs no reading.
|
||||
askCatchUpAfter = 2 * time.Minute
|
||||
// askAgainAfterAnswer is how long a condition the operator answered is not asked about again with the
|
||||
// same answers: what was chosen takes a while to clear it, and asking again at once would ask twice.
|
||||
askAgainAfterAnswer = time.Hour
|
||||
)
|
||||
|
||||
// What became of an ask, as the controller keeps it.
|
||||
const (
|
||||
askOpen = "open"
|
||||
askCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// asked is one ask the controller made, as it keeps it.
|
||||
type asked struct {
|
||||
ID string `json:"id"`
|
||||
Condition string `json:"condition"`
|
||||
Ask asks.Ask `json:"ask"`
|
||||
Actions []conditions.Action `json:"actions"`
|
||||
// Options are the actions by option id.
|
||||
Options map[string]int `json:"options"`
|
||||
State string `json:"state"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Ended time.Time `json:"ended,omitempty"`
|
||||
Warrant *asks.Warrant `json:"warrant,omitempty"`
|
||||
// Acted is what the controller did on the warrant: empty before it did anything, "acting" while it acts,
|
||||
// then "done", "failed: …" or "nothing: …". Anything but empty is never acted on again.
|
||||
Acted string `json:"acted,omitempty"`
|
||||
}
|
||||
|
||||
// askedStore keeps the asks (broker.AskedBucket).
|
||||
type askedStore interface {
|
||||
Get(ctx context.Context, id string) (*asked, error)
|
||||
Put(ctx context.Context, a asked) error
|
||||
All(ctx context.Context) ([]asked, error)
|
||||
}
|
||||
|
||||
// asker is the controller asking the operator and acting on the answer.
|
||||
type asker struct {
|
||||
open func(ctx context.Context) ([]conditions.Condition, error)
|
||||
silence func(ctx context.Context, key string, d time.Duration, by, why string) error
|
||||
store askedStore
|
||||
// publish puts a message on a subject's stream, de-duplicated by id.
|
||||
publish func(ctx context.Context, subject string, body []byte, id string) error
|
||||
// call performs an action's verb with its arguments, as the controller.
|
||||
call func(ctx context.Context, a conditions.Action, args map[string]string) error
|
||||
// record writes the hand-act log.
|
||||
record func(ctx context.Context, act link.HandAct) error
|
||||
// routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing.
|
||||
routerRecord func(ctx context.Context, id string) (*asks.Warrant, error)
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func (a *asker) nudge() {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.nudged == nil {
|
||||
a.nudged = make(chan struct{}, 1)
|
||||
}
|
||||
ch := a.nudged
|
||||
a.mu.Unlock()
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep asks until ctx ends: now, on every change of a condition, and every askEvery.
|
||||
func (a *asker) keep(ctx context.Context) {
|
||||
a.nudge()
|
||||
tick := time.NewTicker(askEvery)
|
||||
defer tick.Stop()
|
||||
a.mu.Lock()
|
||||
nudged := a.nudged
|
||||
a.mu.Unlock()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
case <-nudged:
|
||||
}
|
||||
if err := a.reconcile(ctx); err != nil {
|
||||
a.logf("what the operator is asked could not be brought up to date: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wants says whether a condition is one to ask about now.
|
||||
func wants(c conditions.Condition, now time.Time) bool {
|
||||
return len(c.Actions) > 0 && c.Needs != "" && !c.SilencedAt(now)
|
||||
}
|
||||
|
||||
func sameAsked(a []conditions.Action, b []conditions.Action) bool {
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
return string(x) == string(y)
|
||||
}
|
||||
|
||||
// reconcile brings what is asked in line with what is open.
|
||||
func (a *asker) reconcile(ctx context.Context) error {
|
||||
now := a.now()
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
all, err := a.store.All(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition := map[string]asked{}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen {
|
||||
if prior, held := byCondition[r.Condition]; !held || r.Opened.After(prior.Opened) {
|
||||
byCondition[r.Condition] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// A warrant missed while away, read from the router's record.
|
||||
if a.routerRecord != nil {
|
||||
for _, r := range byCondition {
|
||||
if now.Sub(r.Opened) < askCatchUpAfter {
|
||||
continue
|
||||
}
|
||||
if w, err := a.routerRecord(ctx, r.ID); err == nil && w != nil {
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(ctx, body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if all, err = a.store.All(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
byCondition = map[string]asked{}
|
||||
for _, r := range all {
|
||||
if r.State == askOpen {
|
||||
byCondition[r.Condition] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
// What the operator answered lately, by condition: not asked again at once.
|
||||
answered := map[string]asked{}
|
||||
for _, r := range all {
|
||||
if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer {
|
||||
answered[r.Condition] = r
|
||||
}
|
||||
}
|
||||
wanted := map[string]bool{}
|
||||
sort.Slice(open, func(i, j int) bool { return open[i].Key < open[j].Key })
|
||||
for _, c := range open {
|
||||
if !wants(c, now) {
|
||||
continue
|
||||
}
|
||||
wanted[c.Key] = true
|
||||
if r, done := answered[c.Key]; done && sameAsked(r.Actions, c.Actions) {
|
||||
if _, held := byCondition[c.Key]; !held {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r, held := byCondition[c.Key]; held {
|
||||
switch {
|
||||
case !sameAsked(r.Actions, c.Actions):
|
||||
if err := a.cancel(ctx, r, "its answers changed"); err != nil {
|
||||
return err
|
||||
}
|
||||
case !now.Before(r.Ask.Expires):
|
||||
// Expired unanswered: the router says so too; asked again below while it lasts.
|
||||
r.State, r.Ended = string(asks.OutcomeExpired), now
|
||||
if err := a.store.Put(ctx, r); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := a.ask(ctx, c); err != nil {
|
||||
a.logf("the operator could not be asked about %s: %v", c.Key, err)
|
||||
}
|
||||
}
|
||||
for key, r := range byCondition {
|
||||
if !wanted[key] {
|
||||
if err := a.cancel(ctx, r, "the condition ended, was silenced or needs nothing now"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// optionID is an action's label as an option's id: "Silence for a week" is silence-for-a-week.
|
||||
func optionID(label string) string {
|
||||
var b strings.Builder
|
||||
dash := false
|
||||
for _, r := range strings.ToLower(label) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
dash = false
|
||||
case !dash && b.Len() > 0:
|
||||
b.WriteByte('-')
|
||||
dash = true
|
||||
}
|
||||
}
|
||||
return strings.TrimSuffix(b.String(), "-")
|
||||
}
|
||||
|
||||
// doesWords is what an action does, in the words an option says it with.
|
||||
func doesWords(act conditions.Action) string {
|
||||
switch {
|
||||
case act.Arguments["silence"] != "":
|
||||
return "nothing more is said of it for a week"
|
||||
case act.Verb == "mesh-delivery.release":
|
||||
return "the delivery goes on"
|
||||
case act.Verb == "mesh-delivery.stop":
|
||||
return "the delivery ends"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["go"] != "":
|
||||
return "the delivery starts"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
|
||||
return "the delivery is stopped"
|
||||
case strings.HasSuffix(act.Verb, ".restart"):
|
||||
return "its service is restarted on " + act.Machine
|
||||
}
|
||||
return strings.ToLower(act.Label)
|
||||
}
|
||||
|
||||
// askOf is the ask a condition is asked with.
|
||||
func askOf(id string, c conditions.Condition, now time.Time) (asks.Ask, map[string]int) {
|
||||
q := asks.Ask{ID: id, Headline: c.Headline, Explanation: c.Explanation, Who: asks.Operator,
|
||||
OnExpiry: "nothing is done, and you are asked again while it lasts", About: c.Key,
|
||||
Urgent: c.Severity == conditions.Urgent}
|
||||
options := map[string]int{}
|
||||
approves := false
|
||||
for i, act := range c.Actions {
|
||||
level := asks.Level(act.Level)
|
||||
if level == "" {
|
||||
level = asks.Approve // an action that says nothing of its level is never taken for less
|
||||
}
|
||||
approves = approves || level != asks.Acknowledge
|
||||
oid := optionID(act.Label)
|
||||
options[oid] = i
|
||||
q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesWords(act), Level: level})
|
||||
}
|
||||
q.Expires = now.Add(askAcknowledgeFor)
|
||||
if approves {
|
||||
q.Expires = now.Add(askApproveFor)
|
||||
}
|
||||
return q, options
|
||||
}
|
||||
|
||||
func newAskID() string {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return "c" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// ask publishes one ask about a condition, and keeps it.
|
||||
func (a *asker) ask(ctx context.Context, c conditions.Condition) error {
|
||||
now := a.now()
|
||||
id := newAskID()
|
||||
q, options := askOf(id, c, now)
|
||||
if err := q.Check(now); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(q)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.publish(ctx, asks.AskSubject(askerName), body, "ask."+id); err != nil {
|
||||
return err
|
||||
}
|
||||
a.logf("asked the operator about %s (%s): %d answer(s)", c.Key, id, len(q.Options))
|
||||
return a.store.Put(ctx, asked{ID: id, Condition: c.Key, Ask: q, Actions: c.Actions, Options: options,
|
||||
State: askOpen, Opened: now})
|
||||
}
|
||||
|
||||
// cancel takes an ask back.
|
||||
func (a *asker) cancel(ctx context.Context, r asked, why string) error {
|
||||
body, _ := json.Marshal(map[string]string{"id": r.ID})
|
||||
if err := a.publish(ctx, asks.CancelSubject(askerName), body, "cancel."+r.ID); err != nil {
|
||||
a.logf("the ask %s about %s could not be taken back (%v); taken back at the next look", r.ID, r.Condition, err)
|
||||
return nil
|
||||
}
|
||||
a.logf("took back the ask %s about %s: %s", r.ID, r.Condition, why)
|
||||
r.State, r.Ended = askCancelled, a.now()
|
||||
return a.store.Put(ctx, r)
|
||||
}
|
||||
|
||||
// Decided takes the router's word on one of the controller's asks (link.Decider). An error is returned only
|
||||
// when what was decided could not be kept, so the word is held and heard again.
|
||||
func (a *asker) Decided(ctx context.Context, body []byte) error {
|
||||
var w asks.Warrant
|
||||
if err := json.Unmarshal(body, &w); err != nil {
|
||||
a.logf("the router's word on an ask could not be read; ignored: %v", err)
|
||||
return nil
|
||||
}
|
||||
if w.Asker != askerName {
|
||||
a.logf("REFUSED a warrant for %s's ask %s: the controller acts only on its own", w.Asker, w.Ask)
|
||||
return nil
|
||||
}
|
||||
r, err := a.store.Get(ctx, w.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r == nil {
|
||||
a.logf("REFUSED a warrant for the ask %s, which the controller does not hold", w.Ask)
|
||||
return nil
|
||||
}
|
||||
if r.Acted != "" {
|
||||
return nil // heard again: acted on once
|
||||
}
|
||||
now := a.now()
|
||||
if w.Outcome != asks.OutcomeChosen {
|
||||
r.State, r.Ended, r.Warrant = string(w.Outcome), now, &w
|
||||
r.Acted = "nothing: the ask " + string(w.Outcome)
|
||||
if w.Words != "" {
|
||||
r.Acted += ": " + w.Words
|
||||
}
|
||||
a.logf("the ask %s about %s ended %s; nothing is done", r.ID, r.Condition, w.Outcome)
|
||||
return a.store.Put(ctx, *r)
|
||||
}
|
||||
option, err := w.For(askerName, r.Ask)
|
||||
if err != nil {
|
||||
a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err)
|
||||
return nil
|
||||
}
|
||||
index, offered := r.Options[option.ID]
|
||||
if !offered || index >= len(r.Actions) {
|
||||
a.logf("REFUSED a warrant for the ask %s: it chose %s, which no action stands for", r.ID, option.ID)
|
||||
return nil
|
||||
}
|
||||
act := r.Actions[index]
|
||||
r.State, r.Warrant = string(asks.OutcomeChosen), &w
|
||||
open, err := a.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stillOpen := false
|
||||
for _, c := range open {
|
||||
stillOpen = stillOpen || c.Key == r.Condition
|
||||
}
|
||||
if !stillOpen {
|
||||
// The asker checks the state is still what it asked about before it acts (to-be 46 §10, step 7).
|
||||
r.Ended, r.Acted = now, "nothing: the condition ended before the answer"
|
||||
a.logf("%s, for %s, which ended meanwhile: nothing is done", w.Says(), r.Condition)
|
||||
return a.store.Put(ctx, *r)
|
||||
}
|
||||
r.Acted = "acting"
|
||||
if err := a.store.Put(ctx, *r); err != nil {
|
||||
return err
|
||||
}
|
||||
why := fmt.Sprintf("%s (ask %s)", w.Says(), r.ID)
|
||||
args := map[string]string{}
|
||||
for k, v := range act.Arguments {
|
||||
args[k] = v
|
||||
}
|
||||
if v, takes := args["why"]; takes && v == "" {
|
||||
args["why"] = why
|
||||
}
|
||||
var acted error
|
||||
if act.Arguments["silence"] != "" {
|
||||
acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why)
|
||||
} else {
|
||||
acted = a.call(ctx, act, args)
|
||||
}
|
||||
r.Ended = a.now()
|
||||
r.Acted = "done"
|
||||
if acted != nil {
|
||||
r.Acted = "failed: " + acted.Error()
|
||||
}
|
||||
if err := a.store.Put(ctx, *r); err != nil {
|
||||
return err
|
||||
}
|
||||
verbArgs := []string{act.Verb}
|
||||
if act.Machine != "" {
|
||||
verbArgs = append(verbArgs, "on "+act.Machine)
|
||||
}
|
||||
keys := make([]string, 0, len(args))
|
||||
for k := range args {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if k != "why" {
|
||||
verbArgs = append(verbArgs, k+"="+args[k])
|
||||
}
|
||||
}
|
||||
if err := a.record(ctx, link.HandAct{Verb: handActWarrant, Args: verbArgs, Why: why, By: byWords(w),
|
||||
Cause: conditions.CauseOperatorAnswer, Condition: r.Condition, Via: viaWords(w), Ask: r.ID,
|
||||
Proofs: w.Proofs, RequestedBy: r.Condition, Outcome: r.Acted}); err != nil {
|
||||
a.logf("%s was done, and could NOT be recorded in the hand-act log: %v", why, err)
|
||||
}
|
||||
a.logf("%s: %s", why, r.Acted)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision,
|
||||
// never a repair (handActVerbs).
|
||||
const handActWarrant = "warrant"
|
||||
|
||||
// byWords is who chose, as the hand-act log says it: "the operator, as telegram identity 42".
|
||||
func byWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return "the operator"
|
||||
}
|
||||
return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
|
||||
}
|
||||
|
||||
// viaWords is the channel an answer came through: its module and kind, and how the sender was known.
|
||||
func viaWords(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return w.Channel
|
||||
}
|
||||
via := w.Channel + " (" + w.By.Kind + ")"
|
||||
if w.By.Verified != "" {
|
||||
via += ", " + w.By.Verified
|
||||
}
|
||||
return via
|
||||
}
|
||||
|
||||
// errNotGranted is an action whose verb the controller's grant does not name.
|
||||
var errNotGranted = errors.New("the controller's grant does not name this verb")
|
||||
Reference in New Issue
Block a user