From 8de4dc7951591aee463581c3f6a362d2e22f6163 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 17:14:53 +0200 Subject: [PATCH 01/15] Ask the operator for a condition's answers and act on the warrant, so release, stop, start and restart can be answered from any channel that proves who answered (hq ADR 0259) --- cmd/mesh-controller/asker.go | 478 ++++++++++++++++++ cmd/mesh-controller/asker_test.go | 342 +++++++++++++ cmd/mesh-controller/asker_wire.go | 210 ++++++++ cmd/mesh-controller/conditions.go | 2 +- .../delivery_conditions_test.go | 5 +- cmd/mesh-controller/handacts.go | 13 +- cmd/mesh-controller/module_health.go | 1 + cmd/mesh-controller/plain_words.go | 56 +- cmd/mesh-controller/plain_words_test.go | 37 +- cmd/mesh-controller/signals.go | 1 + cmd/mesh-controller/watchdogs.go | 3 + go.mod | 1 + go.sum | 6 +- internal/broker/controller_asks_test.go | 53 ++ internal/broker/controller_buckets.go | 23 +- internal/broker/nats.go | 25 +- internal/broker/streams.go | 11 + internal/broker/testdata/composed.conf | 4 +- internal/broker/users.go | 20 +- internal/conditions/plain.go | 13 +- internal/link/contracts.go | 4 + internal/link/handacts.go | 10 + internal/link/receive.go | 3 + internal/link/receive_nats.go | 4 +- internal/link/serve.go | 39 +- .../novox/mesh-sdk/go/asks/asks.go | 377 ++++++++++++++ vendor/modules.txt | 3 + 27 files changed, 1711 insertions(+), 33 deletions(-) create mode 100644 cmd/mesh-controller/asker.go create mode 100644 cmd/mesh-controller/asker_test.go create mode 100644 cmd/mesh-controller/asker_wire.go create mode 100644 internal/broker/controller_asks_test.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go new file mode 100644 index 00000000..5896226e --- /dev/null +++ b/cmd/mesh-controller/asker.go @@ -0,0 +1,478 @@ +package main + +// The controller asks, and acts on the operator's warrant (novox/hq ADR 0259 §6). It holds no channel, no +// identity and no factor: it asks the router like any other module, and performs the answer chosen with its +// own grant. +// +// - **For every open, unsilenced condition that needs the operator and names its answers**, one ask is +// published on the `operator-channel` seat under the controller's own name: the condition's words, its +// actions as options at their levels (Silence acknowledges; Release, Stop, Start and Restart approve), +// answered by the operator, expiring after a day (a week when every option only acknowledges). A +// condition that clears, is silenced, or changes its answers has its ask cancelled; an ask that expired +// unanswered is asked again while the condition lasts. Each ask is kept in the controller's bucket +// `asked`, so a restart neither asks twice nor forgets. +// - **On a warrant**, heard on the seat's event under the controller's own name (which only the router may +// say), the controller acts once per ask: only for an ask it holds, only for the option it offered at +// that option's level, and only while the condition is still open. It performs the action as itself — +// a silence through its own conditions, any other through the verb the action names — with the warrant's +// words as its why, and records it in the hand-act log as the operator's decision, naming the channel, +// the ask and the proofs. An ask that ended without a choice is recorded and nothing is done. +// - **A warrant it missed** while away is read from the router's record of its asks, under its own name. + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +// The asker's name on the seat: the controller's module. +const askerName = broker.ControllerSeat + +// How long an ask lasts: a day when an answer approves, a week when every answer only acknowledges. +const ( + askApproveFor = 24 * time.Hour + askAcknowledgeFor = 7 * 24 * time.Hour + // askEvery is how often what is open is asked about again, beside every change. + askEvery = time.Minute + // askCatchUpAfter is how old an open ask is before the router's record of it is read: a warrant heard + // on the event needs no reading. + askCatchUpAfter = 2 * time.Minute + // askAgainAfterAnswer is how long a condition the operator answered is not asked about again with the + // same answers: what was chosen takes a while to clear it, and asking again at once would ask twice. + askAgainAfterAnswer = time.Hour +) + +// What became of an ask, as the controller keeps it. +const ( + askOpen = "open" + askCancelled = "cancelled" +) + +// asked is one ask the controller made, as it keeps it. +type asked struct { + ID string `json:"id"` + Condition string `json:"condition"` + Ask asks.Ask `json:"ask"` + Actions []conditions.Action `json:"actions"` + // Options are the actions by option id. + Options map[string]int `json:"options"` + State string `json:"state"` + Opened time.Time `json:"opened"` + Ended time.Time `json:"ended,omitempty"` + Warrant *asks.Warrant `json:"warrant,omitempty"` + // Acted is what the controller did on the warrant: empty before it did anything, "acting" while it acts, + // then "done", "failed: …" or "nothing: …". Anything but empty is never acted on again. + Acted string `json:"acted,omitempty"` +} + +// askedStore keeps the asks (broker.AskedBucket). +type askedStore interface { + Get(ctx context.Context, id string) (*asked, error) + Put(ctx context.Context, a asked) error + All(ctx context.Context) ([]asked, error) +} + +// asker is the controller asking the operator and acting on the answer. +type asker struct { + open func(ctx context.Context) ([]conditions.Condition, error) + silence func(ctx context.Context, key string, d time.Duration, by, why string) error + store askedStore + // publish puts a message on a subject's stream, de-duplicated by id. + publish func(ctx context.Context, subject string, body []byte, id string) error + // call performs an action's verb with its arguments, as the controller. + call func(ctx context.Context, a conditions.Action, args map[string]string) error + // record writes the hand-act log. + record func(ctx context.Context, act link.HandAct) error + // routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing. + routerRecord func(ctx context.Context, id string) (*asks.Warrant, error) + now func() time.Time + logf func(string, ...any) + + mu sync.Mutex + nudged chan struct{} +} + +func (a *asker) nudge() { + if a == nil { + return + } + a.mu.Lock() + if a.nudged == nil { + a.nudged = make(chan struct{}, 1) + } + ch := a.nudged + a.mu.Unlock() + select { + case ch <- struct{}{}: + default: + } +} + +// keep asks until ctx ends: now, on every change of a condition, and every askEvery. +func (a *asker) keep(ctx context.Context) { + a.nudge() + tick := time.NewTicker(askEvery) + defer tick.Stop() + a.mu.Lock() + nudged := a.nudged + a.mu.Unlock() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + case <-nudged: + } + if err := a.reconcile(ctx); err != nil { + a.logf("what the operator is asked could not be brought up to date: %v", err) + } + } +} + +// wants says whether a condition is one to ask about now. +func wants(c conditions.Condition, now time.Time) bool { + return len(c.Actions) > 0 && c.Needs != "" && !c.SilencedAt(now) +} + +func sameAsked(a []conditions.Action, b []conditions.Action) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} + +// reconcile brings what is asked in line with what is open. +func (a *asker) reconcile(ctx context.Context) error { + now := a.now() + open, err := a.open(ctx) + if err != nil { + return err + } + all, err := a.store.All(ctx) + if err != nil { + return err + } + byCondition := map[string]asked{} + for _, r := range all { + if r.State == askOpen { + if prior, held := byCondition[r.Condition]; !held || r.Opened.After(prior.Opened) { + byCondition[r.Condition] = r + } + } + } + // A warrant missed while away, read from the router's record. + if a.routerRecord != nil { + for _, r := range byCondition { + if now.Sub(r.Opened) < askCatchUpAfter { + continue + } + if w, err := a.routerRecord(ctx, r.ID); err == nil && w != nil { + body, _ := json.Marshal(w) + if err := a.Decided(ctx, body); err != nil { + return err + } + } + } + if all, err = a.store.All(ctx); err != nil { + return err + } + byCondition = map[string]asked{} + for _, r := range all { + if r.State == askOpen { + byCondition[r.Condition] = r + } + } + } + // What the operator answered lately, by condition: not asked again at once. + answered := map[string]asked{} + for _, r := range all { + if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer { + answered[r.Condition] = r + } + } + wanted := map[string]bool{} + sort.Slice(open, func(i, j int) bool { return open[i].Key < open[j].Key }) + for _, c := range open { + if !wants(c, now) { + continue + } + wanted[c.Key] = true + if r, done := answered[c.Key]; done && sameAsked(r.Actions, c.Actions) { + if _, held := byCondition[c.Key]; !held { + continue + } + } + if r, held := byCondition[c.Key]; held { + switch { + case !sameAsked(r.Actions, c.Actions): + if err := a.cancel(ctx, r, "its answers changed"); err != nil { + return err + } + case !now.Before(r.Ask.Expires): + // Expired unanswered: the router says so too; asked again below while it lasts. + r.State, r.Ended = string(asks.OutcomeExpired), now + if err := a.store.Put(ctx, r); err != nil { + return err + } + default: + continue + } + } + if err := a.ask(ctx, c); err != nil { + a.logf("the operator could not be asked about %s: %v", c.Key, err) + } + } + for key, r := range byCondition { + if !wanted[key] { + if err := a.cancel(ctx, r, "the condition ended, was silenced or needs nothing now"); err != nil { + return err + } + } + } + return nil +} + +// optionID is an action's label as an option's id: "Silence for a week" is silence-for-a-week. +func optionID(label string) string { + var b strings.Builder + dash := false + for _, r := range strings.ToLower(label) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + dash = false + case !dash && b.Len() > 0: + b.WriteByte('-') + dash = true + } + } + return strings.TrimSuffix(b.String(), "-") +} + +// doesWords is what an action does, in the words an option says it with. +func doesWords(act conditions.Action) string { + switch { + case act.Arguments["silence"] != "": + return "nothing more is said of it for a week" + case act.Verb == "mesh-delivery.release": + return "the delivery goes on" + case act.Verb == "mesh-delivery.stop": + return "the delivery ends" + case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["go"] != "": + return "the delivery starts" + case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "": + return "the delivery is stopped" + case strings.HasSuffix(act.Verb, ".restart"): + return "its service is restarted on " + act.Machine + } + return strings.ToLower(act.Label) +} + +// askOf is the ask a condition is asked with. +func askOf(id string, c conditions.Condition, now time.Time) (asks.Ask, map[string]int) { + q := asks.Ask{ID: id, Headline: c.Headline, Explanation: c.Explanation, Who: asks.Operator, + OnExpiry: "nothing is done, and you are asked again while it lasts", About: c.Key, + Urgent: c.Severity == conditions.Urgent} + options := map[string]int{} + approves := false + for i, act := range c.Actions { + level := asks.Level(act.Level) + if level == "" { + level = asks.Approve // an action that says nothing of its level is never taken for less + } + approves = approves || level != asks.Acknowledge + oid := optionID(act.Label) + options[oid] = i + q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesWords(act), Level: level}) + } + q.Expires = now.Add(askAcknowledgeFor) + if approves { + q.Expires = now.Add(askApproveFor) + } + return q, options +} + +func newAskID() string { + var b [8]byte + _, _ = rand.Read(b[:]) + return "c" + hex.EncodeToString(b[:]) +} + +// ask publishes one ask about a condition, and keeps it. +func (a *asker) ask(ctx context.Context, c conditions.Condition) error { + now := a.now() + id := newAskID() + q, options := askOf(id, c, now) + if err := q.Check(now); err != nil { + return err + } + body, err := json.Marshal(q) + if err != nil { + return err + } + if err := a.publish(ctx, asks.AskSubject(askerName), body, "ask."+id); err != nil { + return err + } + a.logf("asked the operator about %s (%s): %d answer(s)", c.Key, id, len(q.Options)) + return a.store.Put(ctx, asked{ID: id, Condition: c.Key, Ask: q, Actions: c.Actions, Options: options, + State: askOpen, Opened: now}) +} + +// cancel takes an ask back. +func (a *asker) cancel(ctx context.Context, r asked, why string) error { + body, _ := json.Marshal(map[string]string{"id": r.ID}) + if err := a.publish(ctx, asks.CancelSubject(askerName), body, "cancel."+r.ID); err != nil { + a.logf("the ask %s about %s could not be taken back (%v); taken back at the next look", r.ID, r.Condition, err) + return nil + } + a.logf("took back the ask %s about %s: %s", r.ID, r.Condition, why) + r.State, r.Ended = askCancelled, a.now() + return a.store.Put(ctx, r) +} + +// Decided takes the router's word on one of the controller's asks (link.Decider). An error is returned only +// when what was decided could not be kept, so the word is held and heard again. +func (a *asker) Decided(ctx context.Context, body []byte) error { + var w asks.Warrant + if err := json.Unmarshal(body, &w); err != nil { + a.logf("the router's word on an ask could not be read; ignored: %v", err) + return nil + } + if w.Asker != askerName { + a.logf("REFUSED a warrant for %s's ask %s: the controller acts only on its own", w.Asker, w.Ask) + return nil + } + r, err := a.store.Get(ctx, w.Ask) + if err != nil { + return err + } + if r == nil { + a.logf("REFUSED a warrant for the ask %s, which the controller does not hold", w.Ask) + return nil + } + if r.Acted != "" { + return nil // heard again: acted on once + } + now := a.now() + if w.Outcome != asks.OutcomeChosen { + r.State, r.Ended, r.Warrant = string(w.Outcome), now, &w + r.Acted = "nothing: the ask " + string(w.Outcome) + if w.Words != "" { + r.Acted += ": " + w.Words + } + a.logf("the ask %s about %s ended %s; nothing is done", r.ID, r.Condition, w.Outcome) + return a.store.Put(ctx, *r) + } + option, err := w.For(askerName, r.Ask) + if err != nil { + a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err) + return nil + } + index, offered := r.Options[option.ID] + if !offered || index >= len(r.Actions) { + a.logf("REFUSED a warrant for the ask %s: it chose %s, which no action stands for", r.ID, option.ID) + return nil + } + act := r.Actions[index] + r.State, r.Warrant = string(asks.OutcomeChosen), &w + open, err := a.open(ctx) + if err != nil { + return err + } + stillOpen := false + for _, c := range open { + stillOpen = stillOpen || c.Key == r.Condition + } + if !stillOpen { + // The asker checks the state is still what it asked about before it acts (to-be 46 §10, step 7). + r.Ended, r.Acted = now, "nothing: the condition ended before the answer" + a.logf("%s, for %s, which ended meanwhile: nothing is done", w.Says(), r.Condition) + return a.store.Put(ctx, *r) + } + r.Acted = "acting" + if err := a.store.Put(ctx, *r); err != nil { + return err + } + why := fmt.Sprintf("%s (ask %s)", w.Says(), r.ID) + args := map[string]string{} + for k, v := range act.Arguments { + args[k] = v + } + if v, takes := args["why"]; takes && v == "" { + args["why"] = why + } + var acted error + if act.Arguments["silence"] != "" { + acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why) + } else { + acted = a.call(ctx, act, args) + } + r.Ended = a.now() + r.Acted = "done" + if acted != nil { + r.Acted = "failed: " + acted.Error() + } + if err := a.store.Put(ctx, *r); err != nil { + return err + } + verbArgs := []string{act.Verb} + if act.Machine != "" { + verbArgs = append(verbArgs, "on "+act.Machine) + } + keys := make([]string, 0, len(args)) + for k := range args { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if k != "why" { + verbArgs = append(verbArgs, k+"="+args[k]) + } + } + if err := a.record(ctx, link.HandAct{Verb: handActWarrant, Args: verbArgs, Why: why, By: byWords(w), + Cause: conditions.CauseOperatorAnswer, Condition: r.Condition, Via: viaWords(w), Ask: r.ID, + Proofs: w.Proofs, RequestedBy: r.Condition, Outcome: r.Acted}); err != nil { + a.logf("%s was done, and could NOT be recorded in the hand-act log: %v", why, err) + } + a.logf("%s: %s", why, r.Acted) + return nil +} + +// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision, +// never a repair (handActVerbs). +const handActWarrant = "warrant" + +// byWords is who chose, as the hand-act log says it: "the operator, as telegram identity 42". +func byWords(w asks.Warrant) string { + if w.By == nil { + return "the operator" + } + return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) +} + +// viaWords is the channel an answer came through: its module and kind, and how the sender was known. +func viaWords(w asks.Warrant) string { + if w.By == nil { + return w.Channel + } + via := w.Channel + " (" + w.By.Kind + ")" + if w.By.Verified != "" { + via += ", " + w.By.Verified + } + return via +} + +// errNotGranted is an action whose verb the controller's grant does not name. +var errNotGranted = errors.New("the controller's grant does not name this verb") diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go new file mode 100644 index 00000000..e3100eb4 --- /dev/null +++ b/cmd/mesh-controller/asker_test.go @@ -0,0 +1,342 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +// novox/hq ADR 0259 §6: the controller asks the operator for the answers its conditions name, and performs +// the one chosen on the router's warrant — once, for its own ask, the option offered, at its level. + +type memAskedStore map[string]asked + +func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) { + r, ok := m[id] + if !ok { + return nil, nil + } + return &r, nil +} +func (m memAskedStore) Put(_ context.Context, r asked) error { m[r.ID] = r; return nil } +func (m memAskedStore) All(context.Context) ([]asked, error) { + var out []asked + for _, r := range m { + out = append(out, r) + } + return out, nil +} + +type published struct { + subject, id string + body []byte +} + +type askerRig struct { + a *asker + open []conditions.Condition + store memAskedStore + sent []published + called []string + silenced []string + acts []link.HandAct + now time.Time +} + +func newAskerRig(t *testing.T) *askerRig { + r := &askerRig{store: memAskedStore{}, now: time.Date(2026, 10, 8, 14, 0, 0, 0, time.UTC)} + r.a = &asker{ + open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil }, + silence: func(_ context.Context, key string, d time.Duration, by, why string) error { + r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why) + return nil + }, + store: r.store, + publish: func(_ context.Context, subject string, body []byte, id string) error { + r.sent = append(r.sent, published{subject, id, body}) + return nil + }, + call: func(_ context.Context, a conditions.Action, args map[string]string) error { + raw, _ := json.Marshal(args) + r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw)) + return nil + }, + record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil }, + now: func() time.Time { return r.now }, + logf: t.Logf, + } + return r +} + +func heldCondition() conditions.Condition { + o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", + Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0] + return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline, + Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} +} + +func unitsCondition() conditions.Condition { + key := "machine.shanks.units" + return conditions.Condition{Key: key, Kind: "machine-units", Severity: conditions.Warning, + Headline: "3 failed services on shanks", Explanation: "Needs you: mend or remove them on shanks, or silence this.", + Needs: "mend or remove them on shanks, or silence this.", Actions: []conditions.Action{conditions.SilenceAction(key)}} +} + +func (r *askerRig) asksSent(t *testing.T) []asks.Ask { + t.Helper() + var out []asks.Ask + for _, p := range r.sent { + if p.subject != asks.AskSubject("mesh-controller") { + continue + } + var q asks.Ask + if err := json.Unmarshal(p.body, &q); err != nil { + t.Fatal(err) + } + out = append(out, q) + } + return out +} + +func TestAnAskIsMadeForEachConditionThatNamesItsAnswers(t *testing.T) { + r := newAskerRig(t) + quiet := conditions.Condition{Key: "machine.ace.silent", Headline: "ace silent", Explanation: "Nothing for you to do. x"} + r.open = []conditions.Condition{heldCondition(), unitsCondition(), quiet} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + sent := r.asksSent(t) + if len(sent) != 2 { + t.Fatalf("asked %d times: %+v", len(sent), sent) + } + byAbout := map[string]asks.Ask{} + for _, q := range sent { + byAbout[q.About] = q + if err := q.Check(r.now); err != nil { + t.Errorf("%s: %v", q.About, err) + } + } + held := byAbout[heldCondition().Key] + if len(held.Options) != 2 || held.Options[0].Label != "Release" || held.Options[0].Level != asks.Approve || + held.Options[1].ID != "stop" || held.Expires != r.now.Add(askApproveFor) || held.Who != asks.Operator || + held.OnExpiry == "" { + t.Errorf("the held delivery is asked %+v", held) + } + units := byAbout["machine.shanks.units"] + if len(units.Options) != 1 || units.Options[0].Level != asks.Acknowledge || units.Expires != r.now.Add(askAcknowledgeFor) { + t.Errorf("the failed units are asked %+v", units) + } + // No second ask while one is open. + r.now = r.now.Add(time.Minute) + _ = r.a.reconcile(context.Background()) + if n := len(r.asksSent(t)); n != 2 { + t.Errorf("asked again while open: %d", n) + } +} + +func TestAnAskIsTakenBackWhenItsConditionEndsAndAskedAgainAfterItExpires(t *testing.T) { + r := newAskerRig(t) + r.open = []conditions.Condition{heldCondition(), unitsCondition()} + _ = r.a.reconcile(context.Background()) + // The units are silenced, the held delivery lasts past its ask's day. + units := unitsCondition() + units.Silenced = &conditions.Silence{Until: r.now.Add(48 * time.Hour)} + r.open = []conditions.Condition{heldCondition(), units} + r.now = r.now.Add(askApproveFor) + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + var cancels int + for _, p := range r.sent { + if p.subject == asks.CancelSubject("mesh-controller") { + cancels++ + } + } + if cancels != 1 { + t.Errorf("cancels %d, want the silenced one's", cancels) + } + if sent := r.asksSent(t); len(sent) != 3 || sent[2].About != heldCondition().Key { + t.Errorf("the expired ask was not asked again: %+v", sent) + } +} + +// warrantFor is the router's warrant for the open ask about a condition, choosing an option by label. +func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warrant { + t.Helper() + for _, a := range r.store { + if a.Condition != condition || a.State != askOpen { + continue + } + for _, o := range a.Ask.Options { + if o.Label == label { + return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen, + Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now, + By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} + } + } + } + t.Fatalf("no open ask about %s offers %s", condition, label) + return asks.Warrant{} +} + +func answerWith(t *testing.T, r *askerRig, w asks.Warrant) { + t.Helper() + body, _ := json.Marshal(w) + if err := r.a.Decided(context.Background(), body); err != nil { + t.Fatal(err) + } +} + +func TestAWarrantIsActedOnOnce(t *testing.T) { + r := newAskerRig(t) + r.open = []conditions.Condition{heldCondition()} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, heldCondition().Key, "Release") + answerWith(t, r, w) + answerWith(t, r, w) // heard again + if len(r.called) != 1 { + t.Fatalf("called %v", r.called) + } + want := `mesh-delivery.release@ {"id":"novox/hq@055550802096","why":"the operator, via telegram (user id verified), chose Release (ask ` + w.Ask + `)"}` + if r.called[0] != want { + t.Errorf("called\n %s\nwant\n %s", r.called[0], want) + } + if len(r.acts) != 1 { + t.Fatalf("hand-acts %+v", r.acts) + } + act := r.acts[0] + if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" || + act.Via != "telegram (telegram), user id verified" || act.Ask != w.Ask || strings.Join(act.Proofs, ",") != "P1" || + act.Cause != conditions.CauseOperatorAnswer || act.Condition != heldCondition().Key || act.Outcome != "done" { + t.Errorf("the hand-act %+v", act) + } + if !personsDecision(act) { + t.Error("an act on a warrant counts as a repair") + } + if got := r.store[w.Ask]; got.State != string(asks.OutcomeChosen) || got.Acted != "done" { + t.Errorf("kept %+v", got) + } +} + +func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) { + for name, change := range map[string]func(*asks.Warrant){ + "another asker": func(w *asks.Warrant) { w.Asker = "mesh-delivery" }, + "an ask not held": func(w *asks.Warrant) { w.Ask = "c0000000000000000" }, + "an option not offered": func(w *asks.Warrant) { w.Option = "delete" }, + "another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge }, + "no person": func(w *asks.Warrant) { w.By = nil }, + } { + t.Run(name, func(t *testing.T) { + r := newAskerRig(t) + r.open = []conditions.Condition{heldCondition()} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, heldCondition().Key, "Stop") + change(&w) + answerWith(t, r, w) + if len(r.called)+len(r.acts)+len(r.silenced) != 0 { + t.Errorf("acted on it: %v %v %v", r.called, r.acts, r.silenced) + } + }) + } +} + +func TestEachAnswerCallsExactlyItsVerb(t *testing.T) { + plan := "plan-1791454185265004861" + waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent, + Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.", + Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)} + module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning, + Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.", + Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart", + Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove, + Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}} + for _, tc := range []struct { + c conditions.Condition + label string + want string + }{ + {waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`}, + {waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`}, + {module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`}, + } { + r := newAskerRig(t) + r.open = []conditions.Condition{tc.c} + _ = r.a.reconcile(context.Background()) + answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label)) + if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) { + t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want) + } + } +} + +func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) { + r := newAskerRig(t) + r.open = []conditions.Condition{unitsCondition()} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, "machine.shanks.units", "Silence for a week") + w.Level, w.Proofs = asks.Acknowledge, nil + w.By = &asks.Person{Who: asks.Operator, Kind: "desktop", Identity: "g14", + Verified: "a desk click: whoever was at the operator's session on g14"} + w.Channel = "desk-channel" + answerWith(t, r, w) + if len(r.called) != 0 || len(r.silenced) != 1 || !strings.HasPrefix(r.silenced[0], "machine.shanks.units for 168h0m0s by the operator, as desktop identity g14") { + t.Fatalf("silenced %v, called %v", r.silenced, r.called) + } + if len(r.acts) != 1 || r.acts[0].Cause != conditions.CauseOperatorAnswer || len(r.acts[0].Proofs) != 0 { + t.Errorf("%+v", r.acts) + } +} + +func TestAnAskThatEndedWithoutAChoiceDoesNothing(t *testing.T) { + r := newAskerRig(t) + r.open = []conditions.Condition{heldCondition()} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, heldCondition().Key, "Release") + w.Outcome, w.Option, w.Label, w.Level, w.By, w.Words = asks.OutcomeExpired, "", "", "", nil, "nobody answered in time" + answerWith(t, r, w) + if len(r.called)+len(r.acts) != 0 || r.store[w.Ask].State != string(asks.OutcomeExpired) || + !strings.HasPrefix(r.store[w.Ask].Acted, "nothing") { + t.Errorf("called %v acts %v kept %+v", r.called, r.acts, r.store[w.Ask]) + } + // And a choice for a condition that ended meanwhile does nothing either. + r2 := newAskerRig(t) + r2.open = []conditions.Condition{heldCondition()} + _ = r2.a.reconcile(context.Background()) + w2 := r2.warrantFor(t, heldCondition().Key, "Release") + r2.open = nil + answerWith(t, r2, w2) + if len(r2.called) != 0 || r2.store[w2.Ask].Acted != "nothing: the condition ended before the answer" { + t.Errorf("%v %+v", r2.called, r2.store[w2.Ask]) + } +} + +func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) { + r := newAskerRig(t) + r.open = []conditions.Condition{heldCondition()} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, heldCondition().Key, "Stop") + r.a.routerRecord = func(_ context.Context, id string) (*asks.Warrant, error) { + if id != w.Ask { + return nil, errors.New("another ask") + } + return &w, nil + } + r.now = r.now.Add(askCatchUpAfter) + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-delivery.stop@") { + t.Errorf("called %v", r.called) + } + if n := len(r.asksSent(t)); n != 1 { + t.Errorf("asked again after the answer: %d", n) + } +} diff --git a/cmd/mesh-controller/asker_wire.go b/cmd/mesh-controller/asker_wire.go new file mode 100644 index 00000000..283bbb44 --- /dev/null +++ b/cmd/mesh-controller/asker_wire.go @@ -0,0 +1,210 @@ +package main + +// The asker on the bus: its asks in the controller's bucket `asked`, its asks and cancels published on the +// seat under the controller's name, the verbs a warrant chooses called with the controller's grant, and the +// router's record of its asks read under its name (novox/hq ADR 0259). + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/asks" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// askerFrom is the serving controller's asker; nil in any other process. +var askerFrom *asker + +// askWithin is how long a verb a warrant chose is given to answer. +const askWithin = time.Minute + +type busAsked struct{ conn *nats.Conn } + +func (b busAsked) kv(ctx context.Context) (jetstream.KeyValue, error) { + js, err := jetstream.New(b.conn) + if err != nil { + return nil, err + } + return js.KeyValue(ctx, broker.AskedBucket) +} + +func (b busAsked) Get(ctx context.Context, id string) (*asked, error) { + kv, err := b.kv(ctx) + if err != nil { + return nil, err + } + e, err := kv.Get(ctx, id) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + var r asked + return &r, json.Unmarshal(e.Value(), &r) +} + +func (b busAsked) Put(ctx context.Context, r asked) error { + kv, err := b.kv(ctx) + if err != nil { + return err + } + body, err := json.Marshal(r) + if err != nil { + return err + } + _, err = kv.Put(ctx, r.ID, body) + return err +} + +func (b busAsked) All(ctx context.Context) ([]asked, error) { + kv, err := b.kv(ctx) + if err != nil { + return nil, err + } + lister, err := kv.ListKeys(ctx) + if err != nil { + return nil, err + } + defer func() { _ = lister.Stop() }() + var out []asked + for k := range lister.Keys() { + e, err := kv.Get(ctx, k) + if err != nil { + continue + } + var r asked + if json.Unmarshal(e.Value(), &r) == nil { + out = append(out, r) + } + } + return out, nil +} + +// callAction performs an action's verb as the controller, through the grant that names it. +func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error { + return func(ctx context.Context, a conditions.Action, args map[string]string) error { + seat, verb, ok := strings.Cut(a.Verb, ".") + if !ok { + return fmt.Errorf("%q names no seat and verb", a.Verb) + } + body := map[string]any{} + for k, v := range args { + body[k] = v + } + if seat == catalogue.DeliverySeat { + _, err := askDeliveryOwner(ctx, conn, verb, body) + return err + } + granted := false + for _, v := range broker.VerbsTheControllerActsOnAWarrant { + granted = granted || (v.Seat == seat && v.Verb == verb) + } + if !granted { + return fmt.Errorf("%s: %w", a.Verb, errNotGranted) + } + var answer link.Answer + var err error + if a.Machine != "" { + answer, err = link.AskSeatTool(ctx, conn, seat, verb, a.Machine, body, askWithin) + } else { + answer, err = link.AskMeshSeatTool(ctx, conn, seat, verb, body, askWithin) + } + if err != nil { + return err + } + if answer.Error != "" { + return fmt.Errorf("%s refused: %s", a.Verb, answer.Error) + } + return nil + } +} + +// routerRecordOf reads the router's record of one of the controller's asks, under its name, and answers +// how it ended when it did: the bucket is the one the asks seat's declarer names as its records. +func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.Context, id string) (*asks.Warrant, error) { + return func(ctx context.Context, id string) (*asks.Warrant, error) { + bucket, err := asksRecords(ctx, inv) + if err != nil || bucket == "" { + return nil, err + } + reply, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+askerName+"."+id, nil) + if err != nil { + return nil, err + } + if reply.Header.Get("Status") != "" { + return nil, nil // none, or not readable: the event says it + } + var rec struct { + State string `json:"state"` + Warrant *asks.Warrant `json:"warrant"` + } + if json.Unmarshal(reply.Data, &rec) != nil || rec.State == "open" || rec.Warrant == nil { + return nil, nil + } + return rec.Warrant, nil + } +} + +// asksRecords is the bucket the asks seat's declarer keeps its record of asks in. +func asksRecords(ctx context.Context, inv *inventory.Inventory) (string, error) { + declared, err := inv.Catalogue(ctx) + if err != nil { + return "", err + } + for _, m := range declared { + for _, s := range m.DefinesSeats { + if s.Name == broker.AsksSeat && len(s.Records) > 0 { + return broker.BucketName(m.Module, s.Records[0]), nil + } + } + } + return "", nil +} + +// startAsking makes the serving controller's asker and hands it the router's words. +func startAsking(ctx context.Context, open *stores, server *link.Server, conn *nats.Conn, keeper *conditions.Keeper) { + js, err := jetstream.New(conn) + if err != nil { + fmt.Printf("the operator cannot be asked: %v\n", err) + return + } + a := &asker{ + open: keeper.Open, + silence: func(ctx context.Context, key string, d time.Duration, by, why string) error { + _, err := keeper.Silence(ctx, key, d, by, why) + return err + }, + store: busAsked{conn: conn}, + publish: func(ctx context.Context, subject string, body []byte, id string) error { + _, err := js.Publish(ctx, subject, body, jetstream.WithMsgID(id)) + return err + }, + call: callAction(conn), + record: func(ctx context.Context, act link.HandAct) error { + _, err := link.RecordHandAct(ctx, conn, act) + return err + }, + routerRecord: routerRecordOf(conn, open.inventory), + now: time.Now, + logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }, + } + if err := server.Decides(a); err != nil { + fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err) + return + } + askerFrom = a + go a.keep(ctx) +} diff --git a/cmd/mesh-controller/conditions.go b/cmd/mesh-controller/conditions.go index eed26b61..0bc24496 100644 --- a/cmd/mesh-controller/conditions.go +++ b/cmd/mesh-controller/conditions.go @@ -47,7 +47,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn) (*conditions.Keeper, error) Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }, // What status leads with changed: composed again soon (a nudge outside the serving controller // does nothing). - Changed: statusFrom.nudge, + Changed: func() { statusFrom.nudge(); askerFrom.nudge() }, // Written under the lease, carrying its epoch (novox/hq to-be 45 §6). Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil } diff --git a/cmd/mesh-controller/delivery_conditions_test.go b/cmd/mesh-controller/delivery_conditions_test.go index 27be9213..916b3ca3 100644 --- a/cmd/mesh-controller/delivery_conditions_test.go +++ b/cmd/mesh-controller/delivery_conditions_test.go @@ -136,12 +136,13 @@ func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) { if err != nil { t.Fatal(err) } - for _, verb := range []string{"stalled", "close"} { + // And release and stop, which the operator's warrant chooses (novox/hq ADR 0259). + for _, verb := range []string{"stalled", "close", "release", "stop"} { if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) { t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb) } } - if _, err := askDeliveryOwner(t.Context(), nil, "stop", nil); err == nil || !strings.Contains(err.Error(), "grant") { + if _, err := askDeliveryOwner(t.Context(), nil, "retire-history", nil); err == nil || !strings.Contains(err.Error(), "grant") { t.Fatalf("a verb the grant does not name was asked: %v", err) } conn, err := nats.Connect(testbus.URL(t)) diff --git a/cmd/mesh-controller/handacts.go b/cmd/mesh-controller/handacts.go index 38aea23b..0f2fae3e 100644 --- a/cmd/mesh-controller/handacts.go +++ b/cmd/mesh-controller/handacts.go @@ -60,11 +60,18 @@ var handActVerbs = []handActVerb{ // a person's word (ADR 0242), which the push itself reads from what it carried (recorded_push.go). {Verb: "push", Decision: "a recorded build moves only by a person's push: that push is the word its " + "upgrade policy asks for (ADR 0242)", DecidedWhen: pushedRecorded}, - {Verb: "plans stop"}, + // Stopping or starting a walk the operator chose on a warrant (novox/hq ADR 0259) is their decision. + {Verb: "plans stop", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)", + DecidedFor: []string{conditions.CauseOperatorAnswer}}, {Verb: "plans close"}, // A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or - // not trusted with it — either is a repair the owner should have made. - {Verb: "plans go"}, + // not trusted with it — either is a repair the owner should have made. Unless the operator chose it on + // a warrant (ADR 0259). + {Verb: "plans go", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)", + DecidedFor: []string{conditions.CauseOperatorAnswer}}, + // An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a + // channel that proved who answered, performed by the controller as itself. + {Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"}, {Verb: "broker consumer-reset"}, // Silencing the same condition twice says the condition, or what it watches, wants mending — unless // it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258). diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index b14689b3..dcd3e13f 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -402,6 +402,7 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node, namesWords(plain, 3)), Needs: needs, + Actions: moduleActions(node, rs), Resolved: fmt.Sprintf("%s works again on %s", module, node)} } diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index c48d9f05..8416b610 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -680,13 +680,29 @@ func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severit } // waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it. +// Start and Stop are asked of the operator (novox/hq ADR 0259), so the words do not say where: the router +// says where each can be answered. func waitingNeeds(severity conditions.Severity) string { if severity == conditions.Urgent { - return "start it, or stop it, " + FromMeshMCPServer + return "start it, or stop it." } return "" } +// waitingActions are the answers to a walk waiting past its urgent bound: start it, or stop it — the plan's +// own verbs, approved by the operator (novox/hq ADR 0259). None before the bound. +func waitingActions(plan string, severity conditions.Severity) []conditions.Action { + if severity != conditions.Urgent || plan == "" { + return nil + } + return []conditions.Action{ + {Label: "Start", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, + Arguments: map[string]string{"go": plan, "why": "", "cause": conditions.CauseOperatorAnswer}}, + {Label: "Stop", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, + Arguments: map[string]string{"stop": plan, "why": "", "cause": conditions.CauseOperatorAnswer}}, + } +} + // moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its // account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it. // No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258). @@ -701,11 +717,35 @@ func moduleNeeds(node string, rs []inventory.ResourceHealth) string { } } if unit != "" { - return fmt.Sprintf("restart its service %s on %s %s", unit, node, FromMeshMCPServer) + // Asked of the operator (moduleActions): the router says where it can be answered. + return fmt.Sprintf("restart its service %s on %s.", unit, node) } return "" } +// moduleActions are the answers to a module unhealthy on a machine: restart its failed service there, +// approved by the operator (novox/hq ADR 0259) — none when the mesh restarts it, or a new login is what it +// waits for. +func moduleActions(node string, rs []inventory.ResourceHealth) []conditions.Action { + for _, r := range rs { + if strings.Contains(r.Reason, "relogin needed") { + return nil + } + } + for _, r := range rs { + if r.Kind != link.KindUnit || r.Target == "" { + continue + } + scope := "system" + if r.Account != "" { + scope = "user" + } + return []conditions.Action{{Label: "Restart", Verb: "node-service-manager.restart", Machine: node, + Level: conditions.LevelApprove, Arguments: map[string]string{"unit": r.Target, "scope": scope}}} + } + return nil +} + // FromMeshMCPServer ends what the operator needs when no notification can do it (ADR 0258), naming the mesh MCP // server (the glossary's word; "console" is retired): the answer is not an // acknowledgement, so it is given where the operator is known to be the one asking, until answers are @@ -776,15 +816,19 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio long = "for " + humanDuration(d) } if o.Resolver == conditions.ResolverOperator { - // Words only: releasing or stopping a delivery is not an acknowledgement, so no desk click - // performs it (ADR 0258). + // Asked of the operator, approved on a channel that proves who answered (novox/hq ADR 0259); the + // router says where each can be answered, so the words do not. + release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove, + Arguments: map[string]string{"id": l.ID, "why": ""}} + stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove, + Arguments: map[string]string{"id": l.ID, "why": ""}} switch held { case "held": - needs = "release it, or stop it, " + FromMeshMCPServer + needs, actions = "release it, or stop it.", []conditions.Action{release, stop} case "ready", "checked": needs = "merge its pull request, or close it." default: - needs = "stop it " + FromMeshMCPServer + needs, actions = "stop it.", []conditions.Action{stop} } } return fmt.Sprintf("Delivery of %s %s %s", name, held, long), diff --git a/cmd/mesh-controller/plain_words_test.go b/cmd/mesh-controller/plain_words_test.go index d045d651..3cfe2dcc 100644 --- a/cmd/mesh-controller/plain_words_test.go +++ b/cmd/mesh-controller/plain_words_test.go @@ -65,13 +65,21 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary) } - // Past four hours it is urgent, and offers the controller's own answers. + // Past four hours it is urgent, and asks the operator to start or stop it (novox/hq ADR 0259): the plan's + // own verbs, approved, which the controller performs on the warrant. The router says where to answer. f.waits[0].since = now.Add(-5 * time.Hour) got = watchWaits(f) plainExample(t, got[0], "openrazer delivery waiting to start", - "Needs you: start it, or stop it, from the mesh MCP server; this notification cannot do it. The change to openrazer is merged and built, and mesh-delivery (the "+ + "Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+ "module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+ - "be stuck.") + "be stuck.", "Start", "Stop") + for i, want := range []string{"go", "stop"} { + a := got[0].Actions[i] + if a.Verb != "mesh-controller.plans" || a.Arguments[want] != "plan-1791454185265004861" || + a.Level != conditions.LevelApprove || a.Arguments["cause"] != conditions.CauseOperatorAnswer { + t.Errorf("%s: %+v", a.Label, a) + } + } // Many modules are counted, not listed in the headline. f.waits[0].modules = []string{"a", "b", "c", "d"} @@ -82,16 +90,20 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test } // **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the -// account's own service manager (exit-code)". Restarting is not an acknowledgement, so it is said in words -// and offered as no answer (ADR 0258). +// account's own service manager (exit-code)". Restarting is not an acknowledgement: it is asked of the +// operator at the approve level (novox/hq ADR 0259), so a desk click never performs it (ADR 0258). func TestAModuleUnhealthyAsksForARestartInWords(t *testing.T) { o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit, - Resource: "openrazer-daemon", Target: "openrazer-daemon.service", + Resource: "openrazer-daemon", Target: "openrazer-daemon.service", Account: "jochen", Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}}) plainExample(t, o, "openrazer not working on g14", - "Needs you: restart its service openrazer-daemon on g14 from the mesh MCP server; this notification cannot do it. "+ + "Needs you: restart its service openrazer-daemon on g14. "+ "openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon "+ - "as it runs again.") + "as it runs again.", "Restart") + if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" || a.Level != conditions.LevelApprove || + a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" { + t.Errorf("restart: %+v", a) + } // An account waiting for a new login (ADR 0252) asks for the login, held to the plain rule. o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account", Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}}) @@ -153,8 +165,13 @@ func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) { got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}}) plainExample(t, got[0], "Delivery of hq held for 36 hours", - "Needs you: release it, or stop it, from the mesh MCP server; this notification cannot do it. A delivery of hq has been held for 36 hours, past its limit.", - ) + "Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.", + "Release", "Stop") + for i, verb := range []string{"mesh-delivery.release", "mesh-delivery.stop"} { + if a := got[0].Actions[i]; a.Verb != verb || a.Arguments["id"] != "novox/hq@055550802096" || a.Level != conditions.LevelApprove { + t.Errorf("%+v", a) + } + } } // **Every kind the controller raises has plain words**, and its words are plain for a subject of every diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 1dcc3f20..3b2e82a2 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -378,6 +378,7 @@ func watchWaits(f *signalFacts) []conditions.Observation { Headline: deliveryName(w.modules, w.repository) + " waiting to start", Explanation: walkWaitingWords(w, in, severity), Needs: waitingNeeds(severity), + Actions: waitingActions(w.id, severity), Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"}) } return out diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 99437153..ddb940ba 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -700,6 +700,9 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li // under the lease and the brake, every act said. healers := newHealing(open, keeper, bus, server.JetStream()) go healers.keep(watching) + // And the asker (novox/hq ADR 0259): what needs the operator and names its answers is asked of them, + // and the answer chosen is performed on its warrant. + startAsking(watching, open, server, bus.Conn, keeper) go forgettingOldHeals(watching, open.inventory) fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+ "and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery, diff --git a/go.mod b/go.mod index 88491951..2c4967d6 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( ) require ( + git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6 // indirect github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1507aa09..198e3aaf 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ git.novox.be/novox/mesh-host v0.0.0-20261009101157-2673e7a2c95e h1:H7eVqDILL6e9c git.novox.be/novox/mesh-host v0.0.0-20261009101157-2673e7a2c95e/go.mod h1:72ZATZjxMLaJfWdvlSDJrygIoBzCmKIjCDMhEXxVzTo= git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d h1:IrmJ+lz21n+eSqKrmXREtR/7raUCBJ+fZvs+BNhuXVI= git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d/go.mod h1:72ZATZjxMLaJfWdvlSDJrygIoBzCmKIjCDMhEXxVzTo= +git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6 h1:JT7xM1bnLNInW7/oImV2OlXTrcQ4/GSM0Y8tAb+AhmY= +git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -42,10 +44,10 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= diff --git a/internal/broker/controller_asks_test.go b/internal/broker/controller_asks_test.go new file mode 100644 index 00000000..777f7c8a --- /dev/null +++ b/internal/broker/controller_asks_test.go @@ -0,0 +1,53 @@ +package broker + +import ( + "slices" + "testing" +) + +// novox/hq ADR 0259 §6: the controller asks the operator through the router's seat as any user of it, under +// its own name, hears its own warrants, reads its own record, and calls the verbs a warrant chooses. +func TestTheControllerAsksUnderItsOwnNameAndCallsTheVerbsAWarrantChooses(t *testing.T) { + records := Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": { + {Module: "messenger", Holds: []Seat{operatorChannel()}}, + }}} + users, err := Users(records) + if err != nil { + t.Fatal(err) + } + got := perms(t, users[0]) + for _, s := range []string{ + "mesh.seat.operator-channel.accept.ask.mesh-controller", + "mesh.seat.operator-channel.accept.cancel.mesh-controller", + "$JS.API.DIRECT.GET.KV_messenger_asks.$KV.messenger_asks.mesh-controller.c1", + "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stop", + "mesh.seat.node-service-manager.tool.restart.g14", "mesh.seat.mesh-controller.tool.plans", + } { + if !allowed(got.Publish, s) { + t.Errorf("the controller may not publish %s", s) + } + } + for _, s := range []string{ + "mesh.seat.operator-channel.accept.ask.mesh-delivery", + "mesh.seat.operator-channel.event.decided.mesh-controller", + // (A direct get of another asker's record is not refused here: the controller holds the whole + // JetStream API, as the only writer of stream definitions.) + "mesh.seat.node-service-manager.tool.stop.g14", + } { + if allowed(got.Publish, s) { + t.Errorf("the controller may publish %s", s) + } + } + if !allowed(got.Subscribe, DecidedSubject) || allowed(got.Subscribe, "mesh.seat.operator-channel.event.decided.mesh-delivery") { + t.Error("the controller does not hear exactly its own warrants") + } + // Its events consumer carries them, so a controller that was away hears what was decided meanwhile. + if !slices.Contains(ControllerFollows, DecidedSubject) { + t.Error("the controller does not follow its warrants") + } + // Without a holder of the seat it is granted no ask at all. + alone, _ := Users(Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{}}) + if allowed(perms(t, alone[0]).Publish, "mesh.seat.operator-channel.accept.ask.mesh-controller") { + t.Error("asked a seat nobody holds") + } +} diff --git a/internal/broker/controller_buckets.go b/internal/broker/controller_buckets.go index 12850d5c..69bc81ad 100644 --- a/internal/broker/controller_buckets.go +++ b/internal/broker/controller_buckets.go @@ -34,8 +34,15 @@ var ( // LeaseBucket holds the controller's lease (to-be 45 §6): one key, `holder`, which the instance // allowed to act writes by compare-and-set and renews; its revision when taken is the epoch. LeaseBucket = BucketName(ControllerSeat, "lease") + // AskedBucket keeps what the controller asked the operator about its conditions (novox/hq ADR 0259): + // each ask by its id, its options and the actions they stand for, how it ended and whether the + // controller acted on its warrant — so a restart neither asks twice nor acts twice. + AskedBucket = BucketName(ControllerSeat, "asked") ) +// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own. +const AskedKeptFor = 30 * 24 * time.Hour + // LeaseTTL is how long the lease's key lives unrenewed (to-be 45 §6): fifteen seconds, renewed // every five. The bucket's age, so the bus forgets a holder that stopped renewing. const LeaseTTL = 15 * time.Second @@ -59,12 +66,12 @@ const ( // IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares. func IsControllerBucket(bucket string) bool { return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket || - bucket == ConditionHistoryBucket || bucket == LeaseBucket + bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket } // ControllerBuckets are the controller's own buckets, in the order they are asserted. func ControllerBuckets() []string { - return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket} + return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket} } // ControllerBucketsAsserter is what raising the controller's buckets needs of a connection. @@ -149,6 +156,18 @@ func (j *JetStream) EnsureControllerBuckets() error { }); err != nil { return fmt.Errorf("asserting bucket %s: %w", ConditionHistoryBucket, err) } + if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: AskedBucket, + Description: "what the controller asked the operator about its conditions, and what came of each (novox/hq " + + "ADR 0259): written by the controller alone; an ask acted on is acted on once", + History: 1, + TTL: AskedKeptFor, + MaxValueSize: 32 << 10, + MaxBytes: 32 << 20, + Storage: jetstream.FileStorage, + }); err != nil { + return fmt.Errorf("asserting bucket %s: %w", AskedBucket, err) + } return nil } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 52d1cfba..3ac3f1b9 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -186,8 +186,17 @@ var VerbsTheBusStepAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}} // VerbsTheControllerAsksTheDeliveryOwner are the mesh-delivery seat's verbs the controller calls (novox/hq // ADR 0239): its self-check reads `stalled`, and healer H2 takes the one transition the table allows // through `close`. A mesh seat's verb is flat: no machine in the subject. +// +// And, since novox/hq ADR 0259, `release` and `stop`: the controller asks the operator for them about a +// delivery held past its bound, and calls them on the operator's warrant, with its why. var VerbsTheControllerAsksTheDeliveryOwner = []SeatVerb{{Seat: "mesh-delivery", Verb: "stalled"}, - {Seat: "mesh-delivery", Verb: "close"}} + {Seat: "mesh-delivery", Verb: "close"}, {Seat: "mesh-delivery", Verb: "release"}, {Seat: "mesh-delivery", Verb: "stop"}} + +// VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant +// chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the +// controller's own `plans`. Named one by one; a node seat's on any machine, a mesh seat's flat. +var VerbsTheControllerActsOnAWarrant = []SeatVerb{{Seat: "node-service-manager", Verb: "restart"}, + {Seat: ControllerSeat, Verb: "plans"}} // perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the // holder's machine (novox/hq ADR 0219): `paused.`, the build agent saying whether it takes work. @@ -387,6 +396,20 @@ func PermissionsFor(p Principal) (Permissions, error) { for _, v := range VerbsTheControllerAsksTheDeliveryOwner { pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb) } + // And the verbs a warrant chooses (novox/hq ADR 0259): a node seat's on any machine, its own flat. + for _, v := range VerbsTheControllerActsOnAWarrant { + if v.Seat == ControllerSeat { + pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb) + continue + } + pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*") + } + // And asking the operator (novox/hq ADR 0259): an ask and its cancel under its own name, its warrants + // heard under its own name, the record of its asks read under its own name — as any user of the seat, + // derived the same way, from the seat its holder declares. + tp, ts := SeatTrafficOf(ControllerSeat, nil, p.Uses, nil).grants() + pub = append(pub, tp...) + sub = append(sub, ts...) // And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by // the same discovery the console reads. The question only; the answers come to its own inbox. pub = append(pub, "$SRV.INFO") diff --git a/internal/broker/streams.go b/internal/broker/streams.go index f1bc66bf..124dfa86 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -363,8 +363,19 @@ var ControllerFollows = []string{ // seat to check before it merges — every machine of the facts snapshot composed with the change. // Appended, because the index is a name. moduleEventSubject("gitea", "pull.updated"), + // **The operator's answers to what the controller asked** (novox/hq ADR 0259): the router's warrant, or + // the end of an ask without one, said to the controller alone under its own name. On the stream, so a + // controller that was away hears what was decided meanwhile. Appended, because the index is a name. + DecidedSubject, } +// AsksSeat is the seat an ask is made on and its warrant heard from (novox/hq ADR 0259): the router's. +const AsksSeat = "operator-channel" + +// DecidedSubject is where the router says the controller's warrants: the seat's event named by the +// controller as its caller. +var DecidedSubject = seatEventSubject(AsksSeat, "decided."+ControllerSeat) + // The provider standing events, by their local names. Written here as well as in the catalogue // (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing. const ( diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 0519c572..2b1aa418 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,8 +24,8 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } - subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-service-manager.tool.restart.*"] } + subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { diff --git a/internal/broker/users.go b/internal/broker/users.go index adddd6e9..86ff3b4e 100644 --- a/internal/broker/users.go +++ b/internal/broker/users.go @@ -82,7 +82,7 @@ type Records struct { // is a mesh that cannot be told anything, and there is no state of the records in which that is // correct. func Users(r Records) ([]Principal, error) { - out := []Principal{{Kind: KindController}} + out := []Principal{{Kind: KindController, Uses: asksSeatOf(r)}} for _, node := range sortedCopy(r.Nodes) { witness := false @@ -199,3 +199,21 @@ func sortedNames(in map[string][]string) []string { // controllerModule is the controller's module: the machine assigned it witnesses its upgrades. const controllerModule = "mesh-controller" + +// asksSeatOf is the seat an ask is made on, as its holder declares it (novox/hq ADR 0259): the controller +// asks the operator through it like any other user, and is granted what its declaration names for a caller. +// None while nothing holds it. +func asksSeatOf(r Records) []Seat { + for _, node := range sortedCopy(r.Nodes) { + for _, d := range r.Assigned[node] { + for _, s := range d.Holds { + if s.Name == AsksSeat && namesVerb(s.ByCaller, "ask") { + seat := s + seat.Kind, seat.Capabilities = "", nil + return []Seat{seat} + } + } + } + } + return nil +} diff --git a/internal/conditions/plain.go b/internal/conditions/plain.go index 1a2dca67..53bad38e 100644 --- a/internal/conditions/plain.go +++ b/internal/conditions/plain.go @@ -53,8 +53,19 @@ type Action struct { Verb string `json:"verb"` Machine string `json:"machine,omitempty"` Arguments map[string]string `json:"arguments,omitempty"` + // Level is how much proof its answer needs (novox/hq ADR 0234 §8, ADR 0259): LevelAcknowledge for what + // any granted principal may already do, LevelApprove for what only the operator's proven word does. + // The controller asks for every action, and performs the one chosen on the warrant the router issues. + Level string `json:"level,omitempty"` } +// The assurance levels an action's answer needs (novox/hq ADR 0234 §8): acknowledge, approve. Destroy is +// not asked for by any condition: nothing carries its second proof yet. +const ( + LevelAcknowledge = "acknowledge" + LevelApprove = "approve" +) + // The two verdicts an explanation opens with. const ( NothingToDo = "Nothing for you to do." @@ -66,7 +77,7 @@ const ( // only kind of answer a desk click performs until answers are authorised (novox/hq ADR 0258). Its cause // marks it as an answer, which the hand-act log does not count as a repair. func SilenceAction(key string) Action { - return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions", + return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions", Level: LevelAcknowledge, Arguments: map[string]string{"silence": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}} } diff --git a/internal/link/contracts.go b/internal/link/contracts.go index b3acb7c7..381fdf4f 100644 --- a/internal/link/contracts.go +++ b/internal/link/contracts.go @@ -47,6 +47,10 @@ var Contracts = map[string]Contract{ KindPullUpdated: {Unordered: "a pull request's head, asked to be checked: each head is its own commit, and its " + "verdict is set on that commit alone, so a head heard late is checked and judged as itself and never " + "stands for a newer one (novox/hq to-be 45 §9)"}, + KindDecided: {Unordered: "the router's word on one ask, by the ask's id: an ask is ended once, by compare-and-set " + + "at the router, and the controller acts on it once, recording that it did under the ask's id — so a word " + + "heard again, or late, does nothing more (novox/hq ADR 0259)", + Tests: []string{"TestAWarrantIsActedOnOnce"}}, KindCatchUp: {Unordered: "a catalogue asking what it missed: answered from the record, whenever asked"}, KindProvisioner: {Unordered: "a provider's newest word about a consumer, said again every fifteen minutes " + "while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop. " + diff --git a/internal/link/handacts.go b/internal/link/handacts.go index e4e916c6..dd42f6e0 100644 --- a/internal/link/handacts.go +++ b/internal/link/handacts.go @@ -49,6 +49,16 @@ type HandAct struct { Kind string `json:"kind,omitempty"` // Carried is what such a push moved, one "module from → to" per module, so the log says it. Carried []string `json:"carried,omitempty"` + // Via, Ask, Proofs and RequestedBy are an act the operator chose on a warrant (novox/hq ADR 0234 §8, ADR + // 0259): the channel it came through (module and kind, and how the sender was known), the ask's id, the + // proofs present (P1, P2, P3), and what asked (a condition's key). By then names the operator as that + // kind's identity. Absent from every other act. + Via string `json:"via,omitempty"` + Ask string `json:"ask,omitempty"` + Proofs []string `json:"proofs,omitempty"` + RequestedBy string `json:"requested-by,omitempty"` + // Outcome is what came of an act recorded after it was done: done, or the verb's refusal. + Outcome string `json:"outcome,omitempty"` } // KindRecordedBuilds is a push that only moved recorded builds: the person's word their `record` policy diff --git a/internal/link/receive.go b/internal/link/receive.go index 3e4cbdb7..0173777a 100644 --- a/internal/link/receive.go +++ b/internal/link/receive.go @@ -44,6 +44,9 @@ const ( // KindPullUpdated is the forge announcing a pull request's new head: checked before it merges // (novox/hq to-be 45 §9). KindPullUpdated = "pull-updated" + // KindDecided is the router's warrant for an ask the controller made, or that ask's end without one + // (novox/hq ADR 0259): said to the controller alone, under its own name. + KindDecided = "decided" ) // Control is one thing a node or a module said, as the controller must act on it. diff --git a/internal/link/receive_nats.go b/internal/link/receive_nats.go index e8ad09f3..e8a6cb42 100644 --- a/internal/link/receive_nats.go +++ b/internal/link/receive_nats.go @@ -62,7 +62,7 @@ func Nats(js *broker.JetStream) Inbound { // whatever was asked for — and not at all when nothing was. func (n *natsInbound) Also(kind string) error { switch kind { - case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner, KindPullUpdated: + case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner, KindPullUpdated, KindDecided: n.follows[kind] = true return nil default: @@ -280,6 +280,8 @@ func kindOfSubject(subject string) (string, bool) { return KindSourceMoved, true case PullUpdatedSubject: return KindPullUpdated, true + case broker.DecidedSubject: + return KindDecided, true case BuildOutcome(), BuildOutcomeOf(TheBuildMachineBefore): // A build's outcome is the role's event now, so it arrives on the events stream rather than // the control branch — and is acted on by the same handler, because what the controller does diff --git a/internal/link/serve.go b/internal/link/serve.go index f07627e7..0565e6f5 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -70,7 +70,7 @@ type Checker interface { } // PullUpdatedSubject is where the forge's pull requests land: the controller's own follow of them. -var PullUpdatedSubject = broker.ControllerFollows[len(broker.ControllerFollows)-1] +var PullUpdatedSubject = "mesh.mod.gitea.event.pull.updated" // Server acts on what nodes and modules say. // @@ -96,6 +96,8 @@ type Server struct { checker Checker // healths keeps what machines say of their long-running resources (novox/hq ADR 0240). healths Healths + // decider acts on the operator's warrants for what the controller asked (novox/hq ADR 0259). + decider Decider log *log.Logger // giveUp is how long one message is held for the store; zero means GiveUpAfter. @@ -138,6 +140,21 @@ func (s *Server) Checks(c Checker) error { return nil } +// Decider is what the controller does with the router's word on an ask it made (novox/hq ADR 0259): act +// on a warrant once, or record how the ask ended without one. +type Decider interface { + Decided(ctx context.Context, body []byte) error +} + +// Decides says what to do about the router's word on the controller's asks, and asks for it delivered. +func (s *Server) Decides(d Decider) error { + if err := s.inbound.Also(KindDecided); err != nil { + return err + } + s.decider = d + return nil +} + // Answers says what to do about a catalogue's catch-up request, and asks for them to be delivered. func (s *Server) Answers(r Replayer) error { if err := s.inbound.Also(KindCatchUp); err != nil { @@ -210,6 +227,8 @@ func (s *Server) act(ctx context.Context, m Control) { s.provisioner(ctx, m) case KindPullUpdated: s.pullUpdated(ctx, m) + case KindDecided: + s.decided(ctx, m) default: // Dropped: a message nothing understands will not be understood on the next attempt // either, and asking for it again would spin. @@ -656,6 +675,24 @@ func (s *Server) pullUpdated(ctx context.Context, m Control) { _ = m.Took() } +// decided hands the router's word on an ask to the decider; a failure to keep what it did is held for the +// store, like any word that must not be lost. +func (s *Server) decided(ctx context.Context, m Control) { + if s.decider == nil { + _ = m.Took() + return + } + err := s.decider.Decided(ctx, m.Body()) + switch s.decide(ctx, m, "the operator's word on an ask", "", "", err) { + case Hold: + return + } + if err != nil { + s.log.Printf("the operator's word on an ask could not be kept: %v", err) + } + _ = m.Took() +} + // saysWhatItDid states what a machine now runs, or what it would not take, as a fact on the bus // (novox/hq ADR 0134). // diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go new file mode 100644 index 00000000..52dfe165 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go @@ -0,0 +1,377 @@ +// Package asks is the contract of asking a person and answering on a channel (novox/hq ADR 0259): the +// shapes an asker, the router and a channel exchange on the bus, and the subjects they travel on. No +// transport and no channel's service: an asker publishes an Ask under its own name and acts on the Warrant +// it hears; the router holds the ask, sends channels a Message, and judges the Choice a channel says; a +// channel shows a Message and says what was chosen and by whom, as its service authenticated it. +package asks + +import ( + "errors" + "fmt" + "regexp" + "strings" + "time" +) + +// The seats (novox/hq ADR 0259 §3). +const ( + // Seat is held by the router: an ask and its cancel are its accepts, a warrant its event, each named by + // the asker. + Seat = "operator-channel" + // ChannelSeat is the kinded bench a channel holds to show and say: its accepts carry the kind. + ChannelSeat = "channel" + // IntakeSeat is the kinded bench a channel holds to say what was chosen: its events and proofs carry + // the kind. + IntakeSeat = "intake" +) + +// AskSubject is where an asker publishes an ask, CancelSubject its cancel, and DecidedSubject where it +// hears the warrant, or the ask's end without one. +func AskSubject(asker string) string { return "mesh.seat." + Seat + ".accept.ask." + asker } +func CancelSubject(asker string) string { return "mesh.seat." + Seat + ".accept.cancel." + asker } +func DecidedSubject(asker string) string { return "mesh.seat." + Seat + ".event.decided." + asker } + +// The work a channel takes, on ChannelSubject. +const ( + Show = "show" // a message offering answers + Edit = "edit" // a message shown before, replaced + Send = "send" // a message offering nothing +) + +// ChannelSubject is where the router sends a channel of a kind its work. +func ChannelSubject(verb, kind string) string { + return "mesh.seat." + ChannelSeat + ".accept." + verb + "." + kind +} + +// What a channel says, on IntakeSubject. +const ( + Chosen = "choice" // a button tapped + Link = "link" // somebody asked to be linked as the operator +) + +// IntakeSubject is where a channel of a kind says what arrived. +func IntakeSubject(what, kind string) string { + return "mesh.seat." + IntakeSeat + ".event." + what + "." + kind +} + +// CodeProof is the one proof verb: a code the operator typed, carried by request and reply, never kept. +const CodeProof = "code" + +// ProofSubject is where a channel of a kind asks a proof. +func ProofSubject(verb, kind string) string { + return "mesh.seat." + IntakeSeat + ".proof." + verb + "." + kind +} + +// A Level is how much proof an option's answer needs (novox/hq ADR 0234 §8, the glossary's assurance level). +type Level string + +const ( + // Acknowledge performs only what any granted principal may already do: silencing, details. No proof. + Acknowledge Level = "acknowledge" + // Approve needs one proof: a verified sender (a linked account, linked an hour or more) or a code. + Approve Level = "approve" + // Destroy needs two proofs, one of them a code. + Destroy Level = "destroy" +) + +// Rank orders the levels; an unknown level ranks above every known one, so it is never taken as less. +func (l Level) Rank() int { + switch l { + case Acknowledge: + return 0 + case Approve: + return 1 + case Destroy: + return 2 + } + return 3 +} + +// Operator is the one role an ask may be answered by today. +const Operator = "operator" + +// Option is one answer an ask offers: a label for the button, what it does in plain words, its level. +type Option struct { + ID string `json:"id"` + Label string `json:"label"` + Does string `json:"does"` + Level Level `json:"level"` +} + +// Ask is a request for a person's word (novox/hq ADR 0259 §4). +type Ask struct { + // ID is the asker's own, unique to it. + ID string `json:"id"` + // Headline names the thing and what is wrong, in a few plain words; Explanation is what happened and + // what it means. Both are held to the plain rule and the content rule by the router. + Headline string `json:"headline"` + Explanation string `json:"explanation"` + Options []Option `json:"options"` + // Who may answer: Operator. + Who string `json:"who"` + // Expires is when the ask ends unanswered; OnExpiry is what the asker then does, in words the person + // is shown ("the delivery stays held"). An ask that authorises never defaults. + Expires time.Time `json:"expires"` + OnExpiry string `json:"on-expiry"` + // About is what the ask is about (a condition's key): a newer ask about it replaces the older. + About string `json:"about,omitempty"` + Urgent bool `json:"urgent,omitempty"` +} + +// The bounds of an ask (novox/hq ADR 0234 §8, ADR 0259 §4). +const ( + MostOptions = 4 + MostOpen = 3 + ApproveLasts = 24 * time.Hour + DestroyLasts = 10 * time.Minute + HeadlineLength = 60 + LabelLength = 24 +) + +var usableID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$`) + +// UsableID says whether a name can be an ask's or an option's id: a key and a subject token both. +func UsableID(id string) bool { return usableID.MatchString(id) } + +// Highest is the highest level among the ask's options. +func (a Ask) Highest() Level { + high := Acknowledge + for _, o := range a.Options { + if o.Level.Rank() > high.Rank() { + high = o.Level + } + } + return high +} + +// Option is the option of this id, or false. +func (a Ask) Option(id string) (Option, bool) { + for _, o := range a.Options { + if o.ID == id { + return o, true + } + } + return Option{}, false +} + +// Check is what an ask is held to before anything is shown: every refusal, in words its asker can act on. +func (a Ask) Check(now time.Time) error { + var problems []string + say := func(format string, args ...any) { problems = append(problems, fmt.Sprintf(format, args...)) } + if !UsableID(a.ID) { + say("its id %q is not letters, digits, - and _, at most 64", a.ID) + } + if strings.TrimSpace(a.Headline) == "" || len([]rune(a.Headline)) > HeadlineLength { + say("its headline is empty or longer than %d characters", HeadlineLength) + } + if strings.TrimSpace(a.Explanation) == "" { + say("it explains nothing") + } + if a.Who != Operator { + say("it is answered by %q, and only the operator answers today", a.Who) + } + if len(a.Options) == 0 || len(a.Options) > MostOptions { + say("it offers %d options, and an ask offers one to %d", len(a.Options), MostOptions) + } + seen := map[string]bool{} + for _, o := range a.Options { + switch { + case !UsableID(o.ID): + say("an option's id %q is not letters, digits, - and _", o.ID) + case seen[o.ID]: + say("the option %s is offered twice", o.ID) + } + seen[o.ID] = true + if strings.TrimSpace(o.Label) == "" || len([]rune(o.Label)) > LabelLength { + say("the option %s's label is empty or longer than %d characters", o.ID, LabelLength) + } + if strings.TrimSpace(o.Does) == "" { + say("the option %s does not say what it does", o.ID) + } + if o.Level.Rank() > Destroy.Rank() { + say("the option %s has the level %q, which is none of acknowledge, approve, destroy", o.ID, o.Level) + } + } + if !a.Expires.After(now) { + say("it expires before it is asked") + } + switch a.Highest() { + case Approve: + if a.Expires.After(now.Add(ApproveLasts)) { + say("an ask that approves lasts at most %s", ApproveLasts) + } + case Destroy: + if a.Expires.After(now.Add(DestroyLasts)) { + say("an ask that destroys lasts at most %s", DestroyLasts) + } + } + if a.Highest() != Acknowledge && strings.TrimSpace(a.OnExpiry) == "" { + say("it does not say what happens when nobody answers, and an ask that authorises never defaults") + } + if a.About != "" && strings.ContainsAny(a.About, " \n") { + say("what it is about is a key, without spaces") + } + if len(problems) > 0 { + return errors.New("the ask is refused: " + strings.Join(problems, "; ")) + } + return nil +} + +// Outcome is how an ask ended. +type Outcome string + +const ( + OutcomeChosen Outcome = "chosen" // a person chose an option: a warrant + OutcomeExpired Outcome = "expired" // nobody answered in time + OutcomeCancelled Outcome = "cancelled" // its asker took it back + OutcomeReplaced Outcome = "replaced" // a newer ask about the same thing replaced it + OutcomeRefused Outcome = "refused" // it was never shown: Words says why +) + +// Person is who chose, as the router verified them. +type Person struct { + // Who is the role: Operator. + Who string `json:"who"` + // Kind is the channel kind they answered on, Identity their account on that service, Display the name + // the service shows, and Verified how the router knew it was them. + Kind string `json:"kind"` + Identity string `json:"identity"` + Display string `json:"display,omitempty"` + Verified string `json:"verified"` +} + +// Warrant is the router's record that a person chose one option of one ask, or the ask's end without one +// (novox/hq ADR 0259 §6). It carries no secret. +type Warrant struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + About string `json:"about,omitempty"` + Outcome Outcome `json:"outcome"` + // Option, Label and Level are the option chosen; By who chose it, Channel the module it came through, + // Proofs which proofs were present (P1, P2, P3). + Option string `json:"option,omitempty"` + Label string `json:"label,omitempty"` + Level Level `json:"level,omitempty"` + By *Person `json:"by,omitempty"` + Channel string `json:"channel,omitempty"` + Proofs []string `json:"proofs,omitempty"` + At time.Time `json:"at"` + // Words are why an ask ended without a choice, or what refused it. + Words string `json:"words,omitempty"` +} + +// Says is the warrant in the words an asker records with its act: "the operator, via telegram (user id +// verified), chose Release". +func (w Warrant) Says() string { + if w.Outcome != OutcomeChosen || w.By == nil { + return fmt.Sprintf("no person chose: the ask %s %s", w.Ask, w.Outcome) + } + via := w.By.Kind + if w.By.Verified != "" { + via += " (" + w.By.Verified + ")" + } + return fmt.Sprintf("the %s, via %s, chose %s", w.By.Who, via, w.Label) +} + +// For checks a warrant against the ask its asker made: the same asker and ask, a choice, an option the ask +// offered, at that option's level. An asker acts on nothing else. +func (w Warrant) For(asker string, a Ask) (Option, error) { + if w.Asker != asker || w.Ask != a.ID { + return Option{}, fmt.Errorf("the warrant is for %s's ask %s, not %s's %s", w.Asker, w.Ask, asker, a.ID) + } + if w.Outcome != OutcomeChosen || w.By == nil || w.By.Who != a.Who { + return Option{}, fmt.Errorf("the ask %s ended %s; no person chose", a.ID, w.Outcome) + } + o, offered := a.Option(w.Option) + if !offered { + return Option{}, fmt.Errorf("the ask %s offered no option %s", a.ID, w.Option) + } + if w.Level != o.Level { + return Option{}, fmt.Errorf("the option %s is %s, and the warrant says %s", o.ID, o.Level, w.Level) + } + return o, nil +} + +// Button is one answer a channel offers: its label and the router's one-time ticket for it. +type Button struct { + Label string `json:"label"` + Ticket string `json:"ticket"` +} + +// Message is the work a channel takes: shown with buttons (Show), shown again in place (Edit), or said +// (Send). Handle is the router's name for it, the same across a show and its edits; the channel keeps +// which of its own messages that is. The words are the router's, shown as given. +type Message struct { + Handle string `json:"handle"` + Title string `json:"title"` + Body string `json:"body"` + Buttons []Button `json:"buttons,omitempty"` + Urgent bool `json:"urgent,omitempty"` + Silent bool `json:"silent,omitempty"` + // Reply is the Choice or LinkAsked this answers, by its ID: the channel shows it where that was made. + Reply string `json:"reply,omitempty"` + // To is the account linked as the operator on this kind, for a channel that verifies its sender: where + // the channel sends what is not a reply. The router's word, from its list; empty when none is linked. + To string `json:"to,omitempty"` +} + +// Sender is who a channel's service says sent something: the account, the name it shows, and whether the +// service authenticated it. The router alone judges whether that is the operator. +type Sender struct { + Identity string `json:"identity"` + Display string `json:"display,omitempty"` + Authenticated bool `json:"authenticated"` +} + +// Failed is a message a channel could not deliver: its handle, why, and whether trying again could help. +type Failed struct { + Handle string `json:"handle"` + Why string `json:"why"` + Permanent bool `json:"permanent,omitempty"` + At time.Time `json:"at"` +} + +// Standing is what a channel says of itself, at least every five minutes and whenever it changes: +// whether it can send now, why not, and whether its edits notify nobody. +type Standing struct { + Ready bool `json:"ready"` + Why string `json:"why,omitempty"` + EditsSilently bool `json:"edits-silently,omitempty"` + At time.Time `json:"at"` +} + +// What a channel says of its own delivery, on IntakeSubject. +const ( + FailedWhat = "failed" + StandingWhat = "standing" +) + +// Choice is a button chosen on a channel. +type Choice struct { + // ID is the channel's own for this arrival, unique, so the router acts on it once. + ID string `json:"id"` + Ticket string `json:"ticket"` + Handle string `json:"handle,omitempty"` + Sender Sender `json:"sender"` + At time.Time `json:"at"` +} + +// LinkAsked is somebody on a channel asking to be linked as the operator. +type LinkAsked struct { + ID string `json:"id"` + Sender Sender `json:"sender"` + At time.Time `json:"at"` +} + +// Code is a code a person typed on a channel, asked as a proof: never in an event, never kept. +type Code struct { + Sender Sender `json:"sender"` + Code string `json:"code"` + Purpose string `json:"purpose"` +} + +// ProofAnswer is the router's answer to a proof: whether it was taken, and words to say to the person. +type ProofAnswer struct { + Accepted bool `json:"accepted"` + Words string `json:"words"` +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b8440549..66bec659 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,3 +1,6 @@ +# git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6 +## explicit; go 1.22 +git.novox.be/novox/mesh-sdk/go/asks # github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op ## explicit; go 1.24.0 github.com/antithesishq/antithesis-sdk-go/assert From 744b0b9162735e4ee2368a9dddf16ee6a7205139 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 18:37:30 +0200 Subject: [PATCH 02/15] Ask at most three at a time, wait out a refusal, need a router, and act only on a claimed open ask, as the review asked (hq ADR 0259) --- cmd/mesh-controller/asker.go | 125 +++++++++++++--- cmd/mesh-controller/asker_test.go | 134 ++++++++++++++++++ cmd/mesh-controller/asker_wire.go | 85 +++++++++++ cmd/mesh-controller/plain_words.go | 14 +- cmd/mesh-controller/plain_words_test.go | 6 +- go.mod | 5 +- go.sum | 2 + internal/broker/controller_buckets_test.go | 37 +++++ .../novox/mesh-sdk/go/asks/asks.go | 5 + vendor/modules.txt | 4 +- 10 files changed, 384 insertions(+), 33 deletions(-) diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index 5896226e..b83e439e 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -43,7 +43,9 @@ const askerName = broker.ControllerSeat // How long an ask lasts: a day when an answer approves, a week when every answer only acknowledges. const ( - askApproveFor = 24 * time.Hour + // askApproveFor is a day less a margin, so an ask is never refused at the router for lasting a day and + // a moment (the SDK's bound is a day). + askApproveFor = 24*time.Hour - 10*time.Minute askAcknowledgeFor = 7 * 24 * time.Hour // askEvery is how often what is open is asked about again, beside every change. askEvery = time.Minute @@ -53,6 +55,9 @@ const ( // askAgainAfterAnswer is how long a condition the operator answered is not asked about again with the // same answers: what was chosen takes a while to clear it, and asking again at once would ask twice. askAgainAfterAnswer = time.Hour + // askMostOpen is how many asks the controller holds open at once (the router refuses a fourth): the + // most urgent conditions first, then the oldest. + askMostOpen = asks.MostOpen ) // What became of an ask, as the controller keeps it. @@ -63,10 +68,13 @@ const ( // asked is one ask the controller made, as it keeps it. type asked struct { - ID string `json:"id"` - Condition string `json:"condition"` - Ask asks.Ask `json:"ask"` - Actions []conditions.Action `json:"actions"` + ID string `json:"id"` + Condition string `json:"condition"` + // Channels is what the channels were when it was asked (asker.channels): an ask the router refused is not + // asked again until the condition's answers or the channels change. + Channels string `json:"channels,omitempty"` + Ask asks.Ask `json:"ask"` + Actions []conditions.Action `json:"actions"` // Options are the actions by option id. Options map[string]int `json:"options"` State string `json:"state"` @@ -83,6 +91,9 @@ type askedStore interface { Get(ctx context.Context, id string) (*asked, error) Put(ctx context.Context, a asked) error All(ctx context.Context) ([]asked, error) + // Claim marks an open ask acting, by compare-and-set, and says whether this write stood: of two + // deliveries of one warrant, or two controllers, only the one whose write stands acts. + Claim(ctx context.Context, id string, w asks.Warrant) (bool, error) } // asker is the controller asking the operator and acting on the answer. @@ -98,8 +109,14 @@ type asker struct { record func(ctx context.Context, act link.HandAct) error // routerRecord reads the router's record of an ask for a warrant missed; nil reads nothing. routerRecord func(ctx context.Context, id string) (*asks.Warrant, error) - now func() time.Time - logf func(string, ...any) + // routerHere says whether a router holds the seat and takes asks under the asker's name; nil is yes. + routerHere func(ctx context.Context) (bool, error) + // channels is what the channels are now, as a fingerprint: who holds which kind, promising what. + channels func(ctx context.Context) string + now func() time.Time + logf func(string, ...any) + + saidNoRouter bool mu sync.Mutex nudged chan struct{} @@ -156,6 +173,25 @@ func sameAsked(a []conditions.Action, b []conditions.Action) bool { // reconcile brings what is asked in line with what is open. func (a *asker) reconcile(ctx context.Context) error { now := a.now() + if a.routerHere != nil { + here, err := a.routerHere(ctx) + if err != nil { + return err + } + if !here { + if !a.saidNoRouter { + a.logf("no router takes asks under the controller's name (a module declaring %s with its ask "+ + "named by its caller, assigned): the operator is asked nothing until one is", broker.AsksSeat) + a.saidNoRouter = true + } + return nil + } + a.saidNoRouter = false + } + channels := "" + if a.channels != nil { + channels = a.channels(ctx) + } open, err := a.open(ctx) if err != nil { return err @@ -195,20 +231,47 @@ func (a *asker) reconcile(ctx context.Context) error { } } } - // What the operator answered lately, by condition: not asked again at once. - answered := map[string]asked{} + // What the operator answered lately, by condition: not asked again at once; and what the router refused, + // newest first: not asked again until the answers or the channels change. + answered, refused := map[string]asked{}, map[string]asked{} for _, r := range all { if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer { answered[r.Condition] = r } + if r.State == string(asks.OutcomeRefused) { + if prior, has := refused[r.Condition]; !has || r.Opened.After(prior.Opened) { + refused[r.Condition] = r + } + } } wanted := map[string]bool{} - sort.Slice(open, func(i, j int) bool { return open[i].Key < open[j].Key }) + // The most urgent first, then the oldest: those are asked when no more than askMostOpen may be. + sort.SliceStable(open, func(i, j int) bool { + ui, uj := open[i].Severity == conditions.Urgent, open[j].Severity == conditions.Urgent + if ui != uj { + return ui + } + if !open[i].Raised.Equal(open[j].Raised) { + return open[i].Raised.Before(open[j].Raised) + } + return open[i].Key < open[j].Key + }) + openNow := 0 + for _, c := range open { + if r, held := byCondition[c.Key]; held && wants(c, now) && sameAsked(r.Actions, c.Actions) && now.Before(r.Ask.Expires) { + openNow++ + } + } for _, c := range open { if !wants(c, now) { continue } wanted[c.Key] = true + if r, was := refused[c.Key]; was && sameAsked(r.Actions, c.Actions) && r.Channels == channels { + if _, held := byCondition[c.Key]; !held { + continue // refused, and nothing it was refused for has changed + } + } if r, done := answered[c.Key]; done && sameAsked(r.Actions, c.Actions) { if _, held := byCondition[c.Key]; !held { continue @@ -226,13 +289,19 @@ func (a *asker) reconcile(ctx context.Context) error { if err := a.store.Put(ctx, r); err != nil { return err } + openNow-- default: continue } } - if err := a.ask(ctx, c); err != nil { - a.logf("the operator could not be asked about %s: %v", c.Key, err) + if openNow >= askMostOpen { + continue // asked when one of the open ones ends, most urgent first } + if err := a.ask(ctx, c, channels); err != nil { + a.logf("the operator could not be asked about %s: %v", c.Key, err) + continue + } + openNow++ } for key, r := range byCondition { if !wanted[key] { @@ -280,9 +349,18 @@ func doesWords(act conditions.Action) string { return strings.ToLower(act.Label) } +// askText is a condition's words as an ask says them: without where an answer is given when no channel can +// give it (FromMeshMCPServer), since the ask is answered on a channel and the router says where else. +func askText(s string) string { + for _, with := range []string{", " + FromMeshMCPServer, " " + FromMeshMCPServer} { + s = strings.ReplaceAll(s, with, ".") + } + return strings.ReplaceAll(s, "..", ".") +} + // askOf is the ask a condition is asked with. func askOf(id string, c conditions.Condition, now time.Time) (asks.Ask, map[string]int) { - q := asks.Ask{ID: id, Headline: c.Headline, Explanation: c.Explanation, Who: asks.Operator, + q := asks.Ask{ID: id, Headline: c.Headline, Explanation: askText(c.Explanation), Who: asks.Operator, OnExpiry: "nothing is done, and you are asked again while it lasts", About: c.Key, Urgent: c.Severity == conditions.Urgent} options := map[string]int{} @@ -311,7 +389,7 @@ func newAskID() string { } // ask publishes one ask about a condition, and keeps it. -func (a *asker) ask(ctx context.Context, c conditions.Condition) error { +func (a *asker) ask(ctx context.Context, c conditions.Condition, channels string) error { now := a.now() id := newAskID() q, options := askOf(id, c, now) @@ -327,7 +405,7 @@ func (a *asker) ask(ctx context.Context, c conditions.Condition) error { } a.logf("asked the operator about %s (%s): %d answer(s)", c.Key, id, len(q.Options)) return a.store.Put(ctx, asked{ID: id, Condition: c.Key, Ask: q, Actions: c.Actions, Options: options, - State: askOpen, Opened: now}) + State: askOpen, Opened: now, Channels: channels}) } // cancel takes an ask back. @@ -375,6 +453,11 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { a.logf("the ask %s about %s ended %s; nothing is done", r.ID, r.Condition, w.Outcome) return a.store.Put(ctx, *r) } + if r.State != askOpen { + // Cancelled, replaced or expired in the controller's own record: no answer to it is acted on. + a.logf("REFUSED a warrant for the ask %s, which is %s in the controller's own record", r.ID, r.State) + return nil + } option, err := w.For(askerName, r.Ask) if err != nil { a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err) @@ -401,10 +484,18 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { a.logf("%s, for %s, which ended meanwhile: nothing is done", w.Says(), r.Condition) return a.store.Put(ctx, *r) } - r.Acted = "acting" - if err := a.store.Put(ctx, *r); err != nil { + // Claimed before acting, by compare-and-set: only the delivery whose write stands acts (security review + // of 2026-10-08, finding 9). + claimed, err := a.store.Claim(ctx, r.ID, w) + if err != nil { return err } + if !claimed { + a.logf("the warrant for the ask %s was already taken by another delivery; nothing more is done", r.ID) + return nil + } + r.Acted = "acting" + why := fmt.Sprintf("%s (ask %s)", w.Says(), r.ID) args := map[string]string{} for k, v := range act.Arguments { diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index e3100eb4..1bc43ae1 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -27,6 +27,15 @@ func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) { return &r, nil } func (m memAskedStore) Put(_ context.Context, r asked) error { m[r.ID] = r; return nil } +func (m memAskedStore) Claim(_ context.Context, id string, w asks.Warrant) (bool, error) { + r, ok := m[id] + if !ok || r.State != askOpen || r.Acted != "" { + return false, nil + } + r.State, r.Warrant, r.Acted = string(asks.OutcomeChosen), &w, "acting" + m[id] = r + return true, nil +} func (m memAskedStore) All(context.Context) ([]asked, error) { var out []asked for _, r := range m { @@ -340,3 +349,128 @@ func TestAWarrantMissedWhileAwayIsReadFromTheRoutersRecord(t *testing.T) { t.Errorf("asked again after the answer: %d", n) } } + +// After review (2026-10-08): a refused ask is not asked again until its answers or the channels change. +func TestAnAskTheRouterRefusedWaitsUntilSomethingChanges(t *testing.T) { + r := newAskerRig(t) + channels := "channel/telegram=telegram@anchor[choice]own:true" + r.a.channels = func(context.Context) string { return channels } + r.open = []conditions.Condition{heldCondition()} + _ = r.a.reconcile(context.Background()) + first := r.asksSent(t)[0] + refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused, + Words: "no channel can carry any of its answers now", At: r.now}) + if err := r.a.Decided(context.Background(), refusal); err != nil { + t.Fatal(err) + } + if got := r.store[first.ID]; got.State != string(asks.OutcomeRefused) || !strings.Contains(got.Acted, "nothing") { + t.Fatalf("the refusal was kept as %+v", got) + } + for i := 0; i < 3; i++ { + r.now = r.now.Add(askEvery) + _ = r.a.reconcile(context.Background()) + } + if n := len(r.asksSent(t)); n != 1 { + t.Fatalf("asked again %d time(s) though nothing changed", n-1) + } + channels = "channel/telegram=telegram@anchor[choice,verified-sender]own:true" + _ = r.a.reconcile(context.Background()) + if n := len(r.asksSent(t)); n != 2 { + t.Errorf("not asked again once the channels changed: %d", n) + } +} + +// After review: at most three asks open at once, the most urgent first, then the oldest. +func TestAtMostThreeAsksAreOpenTheMostUrgentFirst(t *testing.T) { + r := newAskerRig(t) + var open []conditions.Condition + for i := 0; i < 4; i++ { + c := unitsCondition() + c.Key = "machine.m" + string(rune('a'+i)) + ".units" + c.Actions = []conditions.Action{conditions.SilenceAction(c.Key)} + c.Raised = r.now.Add(-time.Duration(10-i) * time.Hour) + open = append(open, c) + } + urgent := heldCondition() + urgent.Severity, urgent.Raised = conditions.Urgent, r.now.Add(-time.Minute) + r.open = append(open, urgent) + _ = r.a.reconcile(context.Background()) + sent := r.asksSent(t) + if len(sent) != askMostOpen || sent[0].About != urgent.Key || sent[1].About != "machine.ma.units" || sent[2].About != "machine.mb.units" { + var about []string + for _, q := range sent { + about = append(about, q.About) + } + t.Fatalf("asked %v", about) + } +} + +// After review: nothing is asked while no router takes asks under the controller's name, and that is said once. +func TestNothingIsAskedWithoutARouter(t *testing.T) { + r := newAskerRig(t) + var said []string + r.a.logf = func(f string, a ...any) { said = append(said, f) } + r.a.routerHere = func(context.Context) (bool, error) { return false, nil } + r.open = []conditions.Condition{heldCondition()} + _ = r.a.reconcile(context.Background()) + _ = r.a.reconcile(context.Background()) + if len(r.asksSent(t)) != 0 { + t.Error("asked with no router") + } + n := 0 + for _, s := range said { + if strings.Contains(s, "no router takes asks") { + n++ + } + } + if n != 1 { + t.Errorf("said %d times", n) + } +} + +// After review: the condition's words keep where an answer is given without a channel; the ask's text does not. +func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) { + c := heldCondition() + if !strings.Contains(c.Explanation, FromMeshMCPServer) { + t.Fatalf("the condition lost where it is answered: %q", c.Explanation) + } + q, _ := askOf("x", c, time.Now()) + if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") { + t.Errorf("the ask says %q", q.Explanation) + } + if askApproveFor >= 24*time.Hour { + t.Errorf("an approving ask lasts %s, which the SDK may refuse at its bound", askApproveFor) + } +} + +// After review (security finding 9): a warrant is acted on only for an ask open in the controller's own record, +// once the claim stands, and never when given after the ask expired. +func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) { + r := newAskerRig(t) + r.open = []conditions.Condition{heldCondition()} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, heldCondition().Key, "Release") + late := w + late.At = r.store[w.Ask].Ask.Expires.Add(time.Minute) + body, _ := json.Marshal(late) + _ = r.a.Decided(context.Background(), body) + if len(r.called) != 0 { + t.Fatalf("acted on a warrant given after the ask expired: %v", r.called) + } + // Claimed already by another delivery: nothing done here. + kept := r.store[w.Ask] + kept.Acted = "acting" + r.store[w.Ask] = kept + body, _ = json.Marshal(w) + _ = r.a.Decided(context.Background(), body) + if len(r.called) != 0 { + t.Fatalf("acted though the claim was another's: %v", r.called) + } + // Cancelled in its own record: refused. + kept.Acted, kept.State = "", askCancelled + r.store[w.Ask] = kept + _ = r.a.Decided(context.Background(), body) + if len(r.called) != 0 { + t.Errorf("acted on a cancelled ask: %v", r.called) + } +} diff --git a/cmd/mesh-controller/asker_wire.go b/cmd/mesh-controller/asker_wire.go index 283bbb44..9f0ae5f5 100644 --- a/cmd/mesh-controller/asker_wire.go +++ b/cmd/mesh-controller/asker_wire.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strings" "time" @@ -93,6 +94,41 @@ func (b busAsked) All(ctx context.Context) ([]asked, error) { return out, nil } +// Claim marks an open ask acting, by compare-and-set on its key's revision: only the write that stands acts. +func (b busAsked) Claim(ctx context.Context, id string, w asks.Warrant) (bool, error) { + kv, err := b.kv(ctx) + if err != nil { + return false, err + } + e, err := kv.Get(ctx, id) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil + } + if err != nil { + return false, err + } + var r asked + if err := json.Unmarshal(e.Value(), &r); err != nil { + return false, err + } + if r.State != askOpen || r.Acted != "" { + return false, nil + } + r.State, r.Warrant, r.Acted = string(asks.OutcomeChosen), &w, "acting" + body, err := json.Marshal(r) + if err != nil { + return false, err + } + if _, err := kv.Update(ctx, id, body, e.Revision()); err != nil { + var api *jetstream.APIError + if errors.Is(err, jetstream.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence) { + return false, nil + } + return false, err + } + return true, nil +} + // callAction performs an action's verb as the controller, through the grant that names it. func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error { return func(ctx context.Context, a conditions.Action, args map[string]string) error { @@ -174,6 +210,53 @@ func asksRecords(ctx context.Context, inv *inventory.Inventory) (string, error) return "", nil } +// routerHereIn says whether a module declaring the asks seat, with its ask named by its caller, is assigned: +// without it nothing takes an ask, and asking would only fill a queue nobody reads. +func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, error) { + return func(ctx context.Context) (bool, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return false, err + } + for _, e := range entries { + for _, s := range e.Manifest.DefinesSeats { + if s.Name == broker.AsksSeat && s.NamedByCaller("ask") && len(e.On) > 0 { + return true, nil + } + } + } + return false, nil + } +} + +// channelsIn is what the channels are now, as a fingerprint: each module claiming a kind of the channel +// bench, where, promising what, and whether of its own account. An ask the router refused is asked again +// once this changes. +func channelsIn(inv *inventory.Inventory) func(ctx context.Context) string { + return func(ctx context.Context) string { + entries, err := inv.Catalogued(ctx) + if err != nil { + return "" + } + var parts []string + for _, e := range entries { + for _, c := range e.Manifest.Claims { + if c.Kind == "" || !catalogue.KindedBenches[c.Name] { + continue + } + on := append([]string(nil), e.On...) + sort.Strings(on) + caps := append([]string(nil), c.Capabilities...) + sort.Strings(caps) + parts = append(parts, fmt.Sprintf("%s/%s=%s@%s[%s]own:%t", c.Name, c.Kind, e.Manifest.Module, + strings.Join(on, ","), strings.Join(caps, ","), e.Manifest.RunsAs != "")) + } + } + sort.Strings(parts) + return strings.Join(parts, ";") + } +} + // startAsking makes the serving controller's asker and hands it the router's words. func startAsking(ctx context.Context, open *stores, server *link.Server, conn *nats.Conn, keeper *conditions.Keeper) { js, err := jetstream.New(conn) @@ -198,6 +281,8 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n return err }, routerRecord: routerRecordOf(conn, open.inventory), + routerHere: routerHereIn(open.inventory), + channels: channelsIn(open.inventory), now: time.Now, logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }, } diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 8416b610..4146bab3 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -680,11 +680,11 @@ func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severit } // waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it. -// Start and Stop are asked of the operator (novox/hq ADR 0259), so the words do not say where: the router -// says where each can be answered. +// Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying +// where they are given without a channel, and the ask's text drops that (askText). func waitingNeeds(severity conditions.Severity) string { if severity == conditions.Urgent { - return "start it, or stop it." + return "start it, or stop it, " + FromMeshMCPServer } return "" } @@ -717,8 +717,8 @@ func moduleNeeds(node string, rs []inventory.ResourceHealth) string { } } if unit != "" { - // Asked of the operator (moduleActions): the router says where it can be answered. - return fmt.Sprintf("restart its service %s on %s.", unit, node) + // Also asked of the operator (moduleActions); the ask's text drops where (askText). + return fmt.Sprintf("restart its service %s on %s %s", unit, node, FromMeshMCPServer) } return "" } @@ -824,11 +824,11 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio Arguments: map[string]string{"id": l.ID, "why": ""}} switch held { case "held": - needs, actions = "release it, or stop it.", []conditions.Action{release, stop} + needs, actions = "release it, or stop it, "+FromMeshMCPServer, []conditions.Action{release, stop} case "ready", "checked": needs = "merge its pull request, or close it." default: - needs, actions = "stop it.", []conditions.Action{stop} + needs, actions = "stop it "+FromMeshMCPServer, []conditions.Action{stop} } } return fmt.Sprintf("Delivery of %s %s %s", name, held, long), diff --git a/cmd/mesh-controller/plain_words_test.go b/cmd/mesh-controller/plain_words_test.go index 3cfe2dcc..cd180050 100644 --- a/cmd/mesh-controller/plain_words_test.go +++ b/cmd/mesh-controller/plain_words_test.go @@ -70,7 +70,7 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test f.waits[0].since = now.Add(-5 * time.Hour) got = watchWaits(f) plainExample(t, got[0], "openrazer delivery waiting to start", - "Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+ + "Needs you: start it, or stop it, from the mesh MCP server; this notification cannot do it. The change to openrazer is merged and built, and mesh-delivery (the "+ "module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+ "be stuck.", "Start", "Stop") for i, want := range []string{"go", "stop"} { @@ -97,7 +97,7 @@ func TestAModuleUnhealthyAsksForARestartInWords(t *testing.T) { Resource: "openrazer-daemon", Target: "openrazer-daemon.service", Account: "jochen", Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}}) plainExample(t, o, "openrazer not working on g14", - "Needs you: restart its service openrazer-daemon on g14. "+ + "Needs you: restart its service openrazer-daemon on g14 from the mesh MCP server; this notification cannot do it. "+ "openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon "+ "as it runs again.", "Restart") if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" || a.Level != conditions.LevelApprove || @@ -165,7 +165,7 @@ func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) { got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}}) plainExample(t, got[0], "Delivery of hq held for 36 hours", - "Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.", + "Needs you: release it, or stop it, from the mesh MCP server; this notification cannot do it. A delivery of hq has been held for 36 hours, past its limit.", "Release", "Stop") for i, verb := range []string{"mesh-delivery.release", "mesh-delivery.stop"} { if a := got[0].Actions[i]; a.Verb != verb || a.Arguments["id"] != "novox/hq@055550802096" || a.Level != conditions.LevelApprove { diff --git a/go.mod b/go.mod index 2c4967d6..5eaafc6b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,9 @@ module github.com/novox/mesh-controller go 1.26.0 require ( + git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f github.com/jackc/pgx/v5 v5.10.0 + github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 github.com/novox/mesh-host v0.0.0 golang.org/x/crypto v0.57.0 @@ -11,7 +13,6 @@ require ( ) require ( - git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6 // indirect github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -20,10 +21,8 @@ require ( github.com/klauspost/compress v1.20.0 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nats-server/v2 v2.11.17 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - go.uber.org/automaxprocs v1.6.0 // indirect golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect diff --git a/go.sum b/go.sum index 198e3aaf..4c87bdf9 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d h1:IrmJ+lz21n+eS git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d/go.mod h1:72ZATZjxMLaJfWdvlSDJrygIoBzCmKIjCDMhEXxVzTo= git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6 h1:JT7xM1bnLNInW7/oImV2OlXTrcQ4/GSM0Y8tAb+AhmY= git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f h1:BNvyWq899GwP7F3sY4ACieB5a5fnFAq+sJ9lP6HQ5qI= +git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/broker/controller_buckets_test.go b/internal/broker/controller_buckets_test.go index 96965471..804f1c5e 100644 --- a/internal/broker/controller_buckets_test.go +++ b/internal/broker/controller_buckets_test.go @@ -1,9 +1,15 @@ package broker import ( + "context" "slices" "strings" "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/testbus" ) // **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a @@ -59,3 +65,34 @@ func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) { t.Error("the controller may not ask who answers, or hears every API call") } } + +// The controller's record of what it asked the operator is bounded (correctness review of 2026-10-08): one +// value a key, a month's age, and a size it cannot outgrow. +func TestWhatTheControllerAskedIsBounded(t *testing.T) { + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + kv, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + bucket, err := kv.KeyValue(ctx, AskedBucket) + if err != nil { + t.Fatal(err) + } + status, err := bucket.Status(ctx) + if err != nil { + t.Fatal(err) + } + info := status.(*jetstream.KeyValueBucketStatus).StreamInfo() + if status.History() != 1 || status.TTL() != AskedKeptFor || info.Config.MaxBytes <= 0 || info.Config.MaxBytes > 64<<20 { + t.Errorf("history %d, age %s, bytes %d", status.History(), status.TTL(), info.Config.MaxBytes) + } +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go index 52dfe165..491d2ba2 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go @@ -289,6 +289,11 @@ func (w Warrant) For(asker string, a Ask) (Option, error) { if w.Level != o.Level { return Option{}, fmt.Errorf("the option %s is %s, and the warrant says %s", o.ID, o.Level, w.Level) } + // A choice made after the ask expired is no answer to it, whatever the router said. + if !w.At.IsZero() && w.At.After(a.Expires) { + return Option{}, fmt.Errorf("the warrant was given at %s, after the ask %s expired at %s", + w.At.UTC().Format(time.RFC3339), a.ID, a.Expires.UTC().Format(time.RFC3339)) + } return o, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 66bec659..365367ae 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6 +# git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f ## explicit; go 1.22 git.novox.be/novox/mesh-sdk/go/asks # github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op @@ -83,8 +83,6 @@ github.com/nats-io/nuid github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch github.com/novox/mesh-host/validate -# go.uber.org/automaxprocs v1.6.0 -## explicit; go 1.20 # golang.org/x/crypto v0.57.0 ## explicit; go 1.26.0 golang.org/x/crypto/acme From 0909d7b125a23327da6beefee786bcc9b1bd40b0 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 10:16:12 +0200 Subject: [PATCH 03/15] =?UTF-8?q?Bind=20each=20asked=20option=20to=20the?= =?UTF-8?q?=20exact=20act,=20and=20perform=20only=20that=20act=20on=20its?= =?UTF-8?q?=20warrant=20(hq=20ADR=200259=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every option the controller asks with carries the digest of its verb, machine, arguments and level (the SDK's Option.Binds). A warrant must name the digest of the ask the controller keeps, and before acting the controller checks that the act it is about to perform is the one the option bound: a record changed after the ask is refused, never performed. mesh-sdk moves to d4077b4. --- cmd/mesh-controller/asker.go | 18 ++++- cmd/mesh-controller/asker_test.go | 42 +++++++++++- go.mod | 2 +- go.sum | 2 + .../novox/mesh-sdk/go/asks/asks.go | 65 ++++++++++++++++++- vendor/modules.txt | 2 +- 6 files changed, 125 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index b83e439e..0c8f977d 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -373,7 +373,11 @@ func askOf(id string, c conditions.Condition, now time.Time) (asks.Ask, map[stri approves = approves || level != asks.Acknowledge oid := optionID(act.Label) options[oid] = i - q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesWords(act), Level: level}) + // Every option binds the exact act it stands for (novox/hq ADR 0259 §6): the verb, the machine and + // every argument. The warrant then authorises that act and no other. + binds, _ := asks.ActDigest(boundAct(act)) + q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesWords(act), Level: level, + Binds: binds}) } q.Expires = now.Add(askAcknowledgeFor) if approves { @@ -382,6 +386,12 @@ func askOf(id string, c conditions.Condition, now time.Time) (asks.Ask, map[stri return q, options } +// boundAct is what an option's Binds digests: the act exactly as the controller will perform it, with its +// level, and never its label or words. +func boundAct(act conditions.Action) map[string]any { + return map[string]any{"verb": act.Verb, "machine": act.Machine, "arguments": act.Arguments, "level": act.Level} +} + func newAskID() string { var b [8]byte _, _ = rand.Read(b[:]) @@ -469,6 +479,12 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { return nil } act := r.Actions[index] + // The act about to be performed is the one the option bound when the controller asked: a record changed + // since is refused, never performed. + if err := option.Performs(boundAct(act)); err != nil { + a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err) + return nil + } r.State, r.Warrant = string(asks.OutcomeChosen), &w open, err := a.open(ctx) if err != nil { diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index 1bc43ae1..0edcb308 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -188,7 +188,8 @@ func (r *askerRig) warrantFor(t *testing.T, condition, label string) asks.Warran if o.Label == label { return asks.Warrant{Ask: a.ID, Asker: "mesh-controller", About: condition, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now, - By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} + AskDigest: a.Ask.Digest(), + By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} } } } @@ -242,6 +243,8 @@ func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) { "an option not offered": func(w *asks.Warrant) { w.Option = "delete" }, "another level": func(w *asks.Warrant) { w.Level = asks.Acknowledge }, "no person": func(w *asks.Warrant) { w.By = nil }, + "another ask's digest": func(w *asks.Warrant) { w.AskDigest = "sha256:0000" }, + "no ask's digest": func(w *asks.Warrant) { w.AskDigest = "" }, } { t.Run(name, func(t *testing.T) { r := newAskerRig(t) @@ -474,3 +477,40 @@ func TestAWarrantIsActedOnlyForAnOpenAskItClaimsBeforeItExpired(t *testing.T) { t.Errorf("acted on a cancelled ask: %v", r.called) } } + +// novox/hq ADR 0259 §6: a warrant authorises the act its option bound when the controller asked, and no +// other. A record of the act changed after the ask — another delivery, another machine, another argument — +// is refused and nothing is performed. +func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) { + for name, change := range map[string]func(*conditions.Action){ + "another argument": func(a *conditions.Action) { + a.Arguments = map[string]string{"id": "novox/mesh-controller@000000000000"} + }, + "another verb": func(a *conditions.Action) { a.Verb = "mesh-delivery.stop" }, + "another machine": func(a *conditions.Action) { a.Machine = "anchor" }, + } { + t.Run(name, func(t *testing.T) { + r := newAskerRig(t) + r.open = []conditions.Condition{heldCondition()} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, heldCondition().Key, "Release") + kept := r.store[w.Ask] + acts := append([]conditions.Action(nil), kept.Actions...) + i := kept.Options[w.Option] + change(&acts[i]) + kept.Actions = acts + r.store[w.Ask] = kept + answerWith(t, r, w) + if len(r.called)+len(r.acts) != 0 { + t.Errorf("performed an act the option did not bind: %v %v", r.called, r.acts) + } + }) + } + // Every option of an ask binds its act. + q, _ := askOf("x", heldCondition(), time.Now()) + for _, o := range q.Options { + if o.Binds == "" { + t.Errorf("the option %s binds nothing", o.ID) + } + } +} diff --git a/go.mod b/go.mod index 5eaafc6b..af4c63ee 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f + git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8 github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 4c87bdf9..46fc965e 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6 h1:JT7xM1bnL git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008145004-62367ce15ad6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f h1:BNvyWq899GwP7F3sY4ACieB5a5fnFAq+sJ9lP6HQ5qI= git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8 h1:soqhLNpEXThdq6PdiPy6ExxjJ+yjhh1N1n9E3j1CtrM= +git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go index 491d2ba2..ca2e8006 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go @@ -6,6 +6,9 @@ package asks import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" "fmt" "regexp" @@ -96,6 +99,34 @@ type Option struct { Label string `json:"label"` Does string `json:"does"` Level Level `json:"level"` + // Binds is the digest of exactly what the asker performs when this option is chosen — the verb, the + // machine and every argument — as ActDigest gives it. It travels in the ask, so the router's warrant, + // which names the ask's digest, names it too: a warrant then authorises that act and no other, and an + // asker whose record of the act changed after it asked finds the digests differ and does nothing. + // Required on an option above acknowledge. + Binds string `json:"binds,omitempty"` +} + +// ActDigest is the digest an asker puts in Option.Binds: SHA-256 over the act's JSON (Go's encoding sorts a +// map's keys, so the same act gives the same digest), written "sha256:". +func ActDigest(act any) (string, error) { + raw, err := json.Marshal(act) + if err != nil { + return "", fmt.Errorf("the act cannot be digested: %w", err) + } + sum := sha256.Sum256(raw) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +// Digest is the digest of the ask exactly as its asker published it: its id, words, options with what each +// binds, who answers, and its expiry. The router puts it in the warrant (Warrant.AskDigest), and an asker +// acts only on a warrant whose digest is that of the ask it keeps — so a warrant answers one ask, as the +// person was shown it, and nothing published under the same id before or after. +func (a Ask) Digest() string { + a.Expires = a.Expires.UTC() + raw, _ := json.Marshal(a) + sum := sha256.Sum256(raw) + return "sha256:" + hex.EncodeToString(sum[:]) } // Ask is a request for a person's word (novox/hq ADR 0259 §4). @@ -191,6 +222,9 @@ func (a Ask) Check(now time.Time) error { if o.Level.Rank() > Destroy.Rank() { say("the option %s has the level %q, which is none of acknowledge, approve, destroy", o.ID, o.Level) } + if o.Level != Acknowledge && !strings.HasPrefix(o.Binds, "sha256:") { + say("the option %s authorises and does not bind what it performs (its binds is not an ActDigest)", o.ID) + } } if !a.Expires.After(now) { say("it expires before it is asked") @@ -256,6 +290,9 @@ type Warrant struct { Channel string `json:"channel,omitempty"` Proofs []string `json:"proofs,omitempty"` At time.Time `json:"at"` + // AskDigest is the digest of the ask as the router took it (Ask.Digest): the warrant answers that ask + // alone, with the options it bound. + AskDigest string `json:"ask-digest,omitempty"` // Words are why an ask ended without a choice, or what refused it. Words string `json:"words,omitempty"` } @@ -273,8 +310,9 @@ func (w Warrant) Says() string { return fmt.Sprintf("the %s, via %s, chose %s", w.By.Who, via, w.Label) } -// For checks a warrant against the ask its asker made: the same asker and ask, a choice, an option the ask -// offered, at that option's level. An asker acts on nothing else. +// For checks a warrant against the ask its asker made: the same asker and ask, the same ask's digest (so the +// same words, options and binds), a choice, an option the ask offered, at that option's level, before the +// ask expired. An asker acts on nothing else, and then performs only what the option's Binds names. func (w Warrant) For(asker string, a Ask) (Option, error) { if w.Asker != asker || w.Ask != a.ID { return Option{}, fmt.Errorf("the warrant is for %s's ask %s, not %s's %s", w.Asker, w.Ask, asker, a.ID) @@ -282,6 +320,10 @@ func (w Warrant) For(asker string, a Ask) (Option, error) { if w.Outcome != OutcomeChosen || w.By == nil || w.By.Who != a.Who { return Option{}, fmt.Errorf("the ask %s ended %s; no person chose", a.ID, w.Outcome) } + if d := a.Digest(); w.AskDigest != d { + return Option{}, fmt.Errorf("the warrant answers an ask whose digest is %q, and the ask %s kept here is %s: "+ + "it was not the ask the person was shown", w.AskDigest, a.ID, d) + } o, offered := a.Option(w.Option) if !offered { return Option{}, fmt.Errorf("the ask %s offered no option %s", a.ID, w.Option) @@ -297,6 +339,22 @@ func (w Warrant) For(asker string, a Ask) (Option, error) { return o, nil } +// Performs checks that the act an asker is about to perform is the one the chosen option bound when it +// asked: the act's digest equals the option's Binds. An acknowledge option that bound nothing passes. +func (o Option) Performs(act any) error { + if o.Binds == "" && o.Level == Acknowledge { + return nil + } + d, err := ActDigest(act) + if err != nil { + return err + } + if d != o.Binds { + return fmt.Errorf("the option %s bound %s, and the act about to be performed is %s: nothing is done", o.ID, o.Binds, d) + } + return nil +} + // Button is one answer a channel offers: its label and the router's one-time ticket for it. type Button struct { Label string `json:"label"` @@ -318,6 +376,9 @@ type Message struct { // To is the account linked as the operator on this kind, for a channel that verifies its sender: where // the channel sends what is not a reply. The router's word, from its list; empty when none is linked. To string `json:"to,omitempty"` + // Secret says the words carry something shown once (a link's code): the channel shows it and keeps no + // copy of it — no state, no history of its own. + Secret bool `json:"secret,omitempty"` } // Sender is who a channel's service says sent something: the account, the name it shows, and whether the diff --git a/vendor/modules.txt b/vendor/modules.txt index 365367ae..f7f6eb66 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f +# git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8 ## explicit; go 1.22 git.novox.be/novox/mesh-sdk/go/asks # github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op From 2190e2c664d4e8f6937ab4c308a233d5b99962fc Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 11:03:53 +0200 Subject: [PATCH 04/15] Compose and raise the bus of the lab's proof of the operator's answers, as this controller would (hq ADR 0259) mesh-lab's asks proof runs the router, the Telegram channel and an asker on a real bus. Its accounts, streams, workers, buckets and memberships come from this test at the controller's commit, so the lab proves the composition and not a copy of it. Skipped unless the lab asks. --- internal/inventory/asks_lab_test.go | 243 ++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 internal/inventory/asks_lab_test.go diff --git a/internal/inventory/asks_lab_test.go b/internal/inventory/asks_lab_test.go new file mode 100644 index 00000000..62593d54 --- /dev/null +++ b/internal/inventory/asks_lab_test.go @@ -0,0 +1,243 @@ +package inventory + +// The bus of the lab's proof of the operator's answers (mesh-lab `asks/`, novox/hq ADR 0259). +// +// The proof runs the router, the Telegram channel and an asker against a real bus, and the bus must be the +// one this controller would compose — not a copy of its rules written again in the lab, which would prove +// the copy. So the lab asks this test, at the controller's commit, for both halves: +// +// 1. **Composed** (MESH_LAB_ASKS_OUT and MESH_LAB_ASKS_CATALOGUE set): one machine, `anchor`, running the +// router (messenger), the Telegram channel, the desk channel and the machine's runtime as the catalogue +// declares them, beside two modules of the lab's own — `lab-asker`, which uses `operator-channel`, and +// `lab-bystander`, which does not. Written to the directory: the accounts block exactly as Users and +// ComposeAccounts make it, each user's credential, and every membership as MembershipFor makes it. +// 2. **Raised** (MESH_LAB_ASKS_BUS set as well): on the lab's running bus, as the controller, what a send +// asserts — the mesh's streams and consumers, the seats' work queues and workers, the modules' buckets — +// and every membership published where the runtime reads it. +// +// Without those words it skips: the controller's own suite has nothing to raise. + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/nats-io/nats.go" + "golang.org/x/crypto/bcrypt" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" +) + +// labMachine is the one machine of the lab's bus. +const labMachine = "anchor" + +// The lab's own modules: one that asks, one that may not. +var labManifests = []string{ + `{"module": "lab-asker", "version": "1", "uses": ["operator-channel"], "state": ["acted"], + "own-secrets": {"broker": "${dir:state}/broker"}, + "resources": [{"id": "state", "type": "directory", "mode": "0700", "place": "."}]}`, + `{"module": "lab-bystander", "version": "1", "own-secrets": {"broker": "${dir:state}/broker"}, + "resources": [{"id": "state", "type": "directory", "mode": "0700", "place": "."}]}`, +} + +// labCredential is what a lab process connects as: the runtime's credential shape (mesh-tools bus.Credential). +type labCredential struct { + URL string `json:"url"` + Node string `json:"node,omitempty"` + Module string `json:"module,omitempty"` + User string `json:"user"` + Password string `json:"password"` +} + +func TestTheAsksLabBus(t *testing.T) { + out, modules := os.Getenv("MESH_LAB_ASKS_OUT"), os.Getenv("MESH_LAB_ASKS_CATALOGUE") + if out == "" || modules == "" { + t.Skip("the lab did not ask for its bus (MESH_LAB_ASKS_OUT, MESH_LAB_ASKS_CATALOGUE)") + } + var manifests []catalogue.Manifest + read := func(raw []byte, from string) { + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatalf("%s: %v", from, err) + } + manifests = append(manifests, m) + } + for _, name := range []string{"messenger", "telegram", "desk-channel"} { + path := filepath.Join(modules, name, "module.json") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + read(raw, path) + } + if path := os.Getenv("MESH_LAB_ASKS_RUNTIME"); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + read(raw, path) + } + for i, raw := range labManifests { + read([]byte(raw), "the lab's module "+string(rune('1'+i))) + } + + // As BusRecords reads the store: every seat any module declares, the mesh's own beside them. + seats := map[string]catalogue.SeatDeclaration{} + declarers := map[string]string{} + for _, m := range manifests { + for _, s := range m.DefinesSeats { + seats[s.Name], declarers[s.Name] = s, m.Module + } + } + for _, own := range catalogue.SeatsWithAProtocol() { + seats[own.Name] = catalogue.SeatDeclaration{Name: own.Name, Scope: own.Scope, Accepts: own.Accepts, + Emits: own.Emits, Serves: own.Serves} + } + records := broker.Records{Nodes: []string{labMachine}, Assigned: map[string][]broker.Declared{}, + People: map[string][]string{}, Interchangeable: map[string]bool{}} + var buckets []broker.Bucket + var trafficSeats []broker.Seat + for _, m := range manifests { + records.Assigned[labMachine] = append(records.Assigned[labMachine], declaredFor(m, seats, declarers)) + buckets = append(buckets, bucketsOf(m)...) + for _, s := range m.DefinesSeats { + if seat := asSeat(s, m.Module); seat.Kinded || len(seat.ByCaller) > 0 { + trafficSeats = append(trafficSeats, seat) + } + } + } + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + + // Each user a password of the lab's, the hash in the composition. + passwords := map[string]string{} + if raw, err := os.ReadFile(filepath.Join(out, "passwords.json")); err == nil { + _ = json.Unmarshal(raw, &passwords) + } + for i, u := range users { + name := u.Username() + if passwords[name] == "" { + passwords[name] = "lab-" + name + "-" + time.Now().Format("150405.000000") + } + hash, err := bcrypt.GenerateFromPassword([]byte(passwords[name]), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + users[i].PasswordHash = string(hash) + } + + bus := os.Getenv("MESH_LAB_ASKS_BUS") + if bus == "" { + accounts, err := broker.ComposeAccounts(users) + if err != nil { + t.Fatal(err) + } + creds := map[string]labCredential{} + for _, u := range users { + creds[u.Username()] = labCredential{Node: u.Node, Module: u.Module, User: u.Username(), + Password: passwords[u.Username()]} + } + where := broker.PlacementsOf(records, records.Interchangeable) + memberships := map[string]broker.Membership{} + for _, d := range records.Assigned[labMachine] { + memberships[d.Module] = broker.MembershipFor(labMachine, d, where) + } + write(t, filepath.Join(out, "accounts.conf"), []byte(accounts)) + writeJSON(t, filepath.Join(out, "passwords.json"), passwords) + writeJSON(t, filepath.Join(out, "credentials.json"), creds) + writeJSON(t, filepath.Join(out, "memberships.json"), memberships) + return + } + + // Raised on the lab's bus, as the controller, as a send asserts it (cmd/mesh-controller busobjects.go). + js, err := broker.Dial(bus, nats.UserInfo("controller", passwords["controller"]), nats.CustomInboxPrefix("_INBOX.controller")) + if err != nil { + t.Fatalf("the lab's bus, as the controller: %v", err) + } + defer js.Close() + if err := broker.Raise(js, records.Nodes); err != nil { + t.Fatal(err) + } + holders := map[string]broker.Holder{} + for _, d := range records.Assigned[labMachine] { + for _, s := range d.Holds { + if _, taken := holders[s.Name]; !taken { + holders[s.Name] = broker.Holder{Node: labMachine, Module: d.Module} + } + } + } + if err := broker.RaiseSeats(js, MeshSeats(), holders); err != nil { + t.Fatal(err) + } + streams, workers := broker.SeatTrafficObjects(users) + have := map[string]bool{} + for _, s := range streams { + have[s.Name] = true + } + for _, s := range broker.TrafficQueues(trafficSeats) { + if !have[s.Name] { + streams, have[s.Name] = append(streams, s), true + } + } + for _, s := range streams { + if err := js.EnsureStream(s); err != nil { + t.Fatalf("the work queue %s: %v", s.Name, err) + } + } + for _, c := range workers { + if err := js.EnsureConsumer(c); err != nil { + t.Fatalf("the worker %s: %v", c.Name, err) + } + } + for _, c := range broker.ConsumersOf(users) { + if err := js.EnsureConsumer(c.Consumer); err != nil { + t.Fatalf("how %s hears what it consumes: %v", c.Module, err) + } + } + if _, err := broker.RaiseBuckets(js, buckets); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + where := broker.PlacementsOf(records, records.Interchangeable) + names := make([]string, 0) + for _, d := range records.Assigned[labMachine] { + body, err := json.Marshal(broker.MembershipFor(labMachine, d, where)) + if err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(broker.MembershipSubject(labMachine, d.Module), body); err != nil { + t.Fatalf("issuing %s its membership: %v", d.Module, err) + } + names = append(names, d.Module) + } + sort.Strings(names) + t.Logf("raised on %s: %d streams of seats, %d workers, %d buckets, memberships for %v", bus, len(streams), + len(workers), len(buckets), names) +} + +func write(t *testing.T, path string, body []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } +} + +func writeJSON(t *testing.T, path string, v any) { + t.Helper() + body, err := json.MarshalIndent(v, "", " ") + if err != nil { + t.Fatal(err) + } + write(t, path, body) +} From 646c5e53db4215bc021fef7f19b694c108c1cdc6 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 11:07:13 +0200 Subject: [PATCH 05/15] Add drill: an ask the operator starts at the controller's terminal, whose approval performs nothing and is recorded (hq ADR 0259) The live acceptance needs an approval the operator can ask for at will and that changes nothing. A drill is asked like any condition's ask, bound to its own act, claimed once on its warrant and recorded as a warrant hand-act with who answered, through which channel and the proofs. A verb's process may not start one, so no agent asks the operator a question they did not start. --- cmd/mesh-controller/asker.go | 16 +++-- cmd/mesh-controller/drill.go | 110 ++++++++++++++++++++++++++++++ cmd/mesh-controller/drill_test.go | 60 ++++++++++++++++ cmd/mesh-controller/main.go | 2 + 4 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 cmd/mesh-controller/drill.go create mode 100644 cmd/mesh-controller/drill_test.go diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index 0c8f977d..3c21e574 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -84,6 +84,9 @@ type asked struct { // Acted is what the controller did on the warrant: empty before it did anything, "acting" while it acts, // then "done", "failed: …" or "nothing: …". Anything but empty is never acted on again. Acted string `json:"acted,omitempty"` + // Drill is an ask started at the controller's terminal (drill.go): about no condition, its answers + // perform nothing, and the reconciling of conditions leaves it alone. + Drill bool `json:"drill,omitempty"` } // askedStore keeps the asks (broker.AskedBucket). @@ -202,7 +205,7 @@ func (a *asker) reconcile(ctx context.Context) error { } byCondition := map[string]asked{} for _, r := range all { - if r.State == askOpen { + if r.State == askOpen && !r.Drill { if prior, held := byCondition[r.Condition]; !held || r.Opened.After(prior.Opened) { byCondition[r.Condition] = r } @@ -226,7 +229,7 @@ func (a *asker) reconcile(ctx context.Context) error { } byCondition = map[string]asked{} for _, r := range all { - if r.State == askOpen { + if r.State == askOpen && !r.Drill { byCondition[r.Condition] = r } } @@ -490,7 +493,7 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { if err != nil { return err } - stillOpen := false + stillOpen := r.Drill // a drill is about no condition for _, c := range open { stillOpen = stillOpen || c.Key == r.Condition } @@ -521,9 +524,12 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { args["why"] = why } var acted error - if act.Arguments["silence"] != "" { + switch { + case r.Drill && act.Verb == drillVerb: + // A drill's answer performs nothing: it is recorded below as the operator's decision. + case act.Arguments["silence"] != "": acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why) - } else { + default: acted = a.call(ctx, act, args) } r.Ended = a.now() diff --git a/cmd/mesh-controller/drill.go b/cmd/mesh-controller/drill.go new file mode 100644 index 00000000..1db79cf8 --- /dev/null +++ b/cmd/mesh-controller/drill.go @@ -0,0 +1,110 @@ +package main + +// The drill of the operator's answers (novox/hq ADR 0259, the live acceptance after rollout): an ask the +// operator starts at the controller's terminal, answered on the phone, whose approval changes nothing and is +// recorded as a person's decision like any other. +// +// mesh-controller drill [--for 15m] +// +// It asks with two answers — Approve (an approval, so only a channel that proves who answered carries it) and +// Decline (an acknowledgement) — each bound to the drill's own act. The serving controller acts on the warrant +// as on any other: it claims the ask once, checks the act is the one bound, performs nothing, and records the +// hand-act `warrant` with who answered, through which channel, and the proofs. `hand-acts` then shows it. +// +// **The terminal's alone**: a command a verb runs (MESH_VERB set) is refused, so no agent starts a drill — a +// drill is a question the operator expects, and one an agent could start would teach them to approve what they +// did not ask for. + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "time" + + "github.com/nats-io/nats.go" + + "git.novox.be/novox/mesh-sdk/go/asks" + + "github.com/novox/mesh-controller/internal/conditions" +) + +// drillVerb is the act a drill's answers bind: nothing is called. +const drillVerb = "drill" + +// drillActions are the drill's two answers. +func drillActions() []conditions.Action { + return []conditions.Action{ + {Label: "Approve", Verb: drillVerb, Level: conditions.LevelApprove, Arguments: map[string]string{"drill": "approve"}}, + {Label: "Decline", Verb: drillVerb, Level: conditions.LevelAcknowledge, Arguments: map[string]string{"drill": "decline"}}, + } +} + +// drillAsk is the drill's ask, as the router is sent it. +func drillAsk(id string, now time.Time, lasts time.Duration) (asks.Ask, map[string]int) { + q := asks.Ask{ID: id, Headline: "Drill: approve this test question?", Who: asks.Operator, + Explanation: "Needs you: approve or decline. You started this drill at the controller's terminal. Approving " + + "changes nothing on the mesh; it is recorded as your decision, so you can check the record.", + OnExpiry: "nothing is done", Expires: now.Add(lasts), About: "drill." + id} + options := map[string]int{} + for i, act := range drillActions() { + binds, _ := asks.ActDigest(boundAct(act)) + oid := optionID(act.Label) + options[oid] = i + q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesDrill(act), Level: asks.Level(act.Level), + Binds: binds}) + } + return q, options +} + +func doesDrill(act conditions.Action) string { + if act.Arguments["drill"] == "approve" { + return "nothing changes; your approval is recorded" + } + return "nothing changes; your answer is recorded" +} + +func drillCommand(ctx context.Context, args []string) error { + if os.Getenv(verbVar) != "" { + return errors.New("drill is the controller's terminal's alone: a verb may not start one, so no agent asks " + + "the operator a question they did not start (novox/hq ADR 0259)") + } + set := flag.NewFlagSet("drill", flag.ContinueOnError) + lasts := set.Duration("for", 15*time.Minute, "how long the question waits for an answer") + if err := set.Parse(args); err != nil { + return err + } + if *lasts < time.Minute || *lasts > askApproveFor { + return fmt.Errorf("a drill waits between a minute and %s", askApproveFor) + } + js, err := aBus() + if err != nil { + return err + } + defer js.Close() + now := time.Now() + id := newAskID() + q, options := drillAsk(id, now, *lasts) + if err := q.Check(now); err != nil { + return err + } + store := busAsked{conn: js.Conn()} + // Kept before it is published, as the asker keeps every ask, so a warrant always finds it. + if err := store.Put(ctx, asked{ID: id, Condition: q.About, Ask: q, Actions: drillActions(), Options: options, + State: askOpen, Opened: now, Drill: true}); err != nil { + return fmt.Errorf("the drill could not be kept in the controller's asks: %w", err) + } + body, err := json.Marshal(q) + if err != nil { + return err + } + if _, err := js.Context().Publish(asks.AskSubject(askerName), body, nats.MsgId("ask."+id), nats.Context(ctx)); err != nil { + return fmt.Errorf("the drill could not be asked: %w", err) + } + fmt.Printf("drill %s asked: answer it on your phone before %s. Then `mesh-controller hand-acts` shows the "+ + "answer as a warrant, with who answered, through which channel and the proofs; nothing else changes.\n", + id, q.Expires.Local().Format("15:04")) + return nil +} diff --git a/cmd/mesh-controller/drill_test.go b/cmd/mesh-controller/drill_test.go new file mode 100644 index 00000000..074abd4b --- /dev/null +++ b/cmd/mesh-controller/drill_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// A drill (the live acceptance of novox/hq ADR 0259): its approval is a warrant like any other — claimed once, +// its act checked against what the option bound, recorded as the operator's decision with who, how and the +// proofs — and it performs nothing. The reconciling of conditions leaves it open. +func TestADrillsApprovalIsRecordedAndPerformsNothing(t *testing.T) { + r := newAskerRig(t) + q, options := drillAsk("cdrill", r.now, askerDrillFor) + if err := q.Check(r.now); err != nil { + t.Fatalf("the drill's ask is refused: %v", err) + } + r.store["cdrill"] = asked{ID: "cdrill", Condition: q.About, Ask: q, Actions: drillActions(), Options: options, + State: askOpen, Opened: r.now, Drill: true} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + if got := r.store["cdrill"]; got.State != askOpen { + t.Fatalf("the reconciling of conditions ended the drill: %+v", got) + } + approve, _ := q.Option("approve") + w := asks.Warrant{Ask: "cdrill", Asker: "mesh-controller", Outcome: asks.OutcomeChosen, Option: approve.ID, + Label: approve.Label, Level: approve.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now, + AskDigest: q.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} + answerWith(t, r, w) + answerWith(t, r, w) // heard again + if len(r.called)+len(r.silenced) != 0 { + t.Errorf("a drill performed something: %v %v", r.called, r.silenced) + } + if len(r.acts) != 1 { + t.Fatalf("hand-acts %+v", r.acts) + } + act := r.acts[0] + if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" || act.Ask != "cdrill" || + strings.Join(act.Args, " ") != "drill drill=approve" || act.Outcome != "done" || strings.Join(act.Proofs, ",") != "P1" { + t.Errorf("the drill's record: %+v", act) + } + if !personsDecision(act) { + t.Error("a drill's answer counts as a repair") + } +} + +// Only the terminal starts a drill: a verb's process is refused before anything is asked. +func TestADrillIsTheTerminalsAlone(t *testing.T) { + t.Setenv(verbVar, "mesh-controller.command") + if err := drillCommand(context.Background(), nil); err == nil || !strings.Contains(err.Error(), "terminal") { + t.Fatalf("a verb started a drill: %v", err) + } +} + +// askerDrillFor is how long the test's drill waits. +const askerDrillFor = 15 * time.Minute diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 2c03372a..7ea824ba 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -78,6 +78,8 @@ func run() error { return rotateCommand(ctx, args[1:]) case "ask": return askCommand(ctx, args[1:]) + case "drill": + return drillCommand(ctx, args[1:]) case "builds": return buildsCommand(ctx, args[1:]) // The build queue, controlled by hand (novox/hq ADR 0219). From ad406e81b87f68f136c717c98c1f28e4c22d9d3f Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 11:11:54 +0200 Subject: [PATCH 06/15] Say loudly when a condition that needs the operator could not be asked on any channel (hq ADR 0259) With no router, or an ask the router refused and nothing changed since, the controller asked nothing and said it only in its own log. It now keeps a condition of its own, asks-undelivered, naming the conditions not asked and why, cleared once each can be asked again. --- cmd/mesh-controller/asker.go | 65 +++++++++++++++++++++++++++++-- cmd/mesh-controller/asker_test.go | 49 +++++++++++++++++++++++ cmd/mesh-controller/asker_wire.go | 7 +++- 3 files changed, 116 insertions(+), 5 deletions(-) diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index 3c21e574..5202fc19 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -116,8 +116,11 @@ type asker struct { routerHere func(ctx context.Context) (bool, error) // channels is what the channels are now, as a fingerprint: who holds which kind, promising what. channels func(ctx context.Context) string - now func() time.Time - logf func(string, ...any) + // raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be + // asked, and why. Nil raises nothing (a test that does not look). + raise func(ctx context.Context, obs []conditions.Observation) error + now func() time.Time + logf func(string, ...any) saidNoRouter bool @@ -187,7 +190,18 @@ func (a *asker) reconcile(ctx context.Context) error { "named by its caller, assigned): the operator is asked nothing until one is", broker.AsksSeat) a.saidNoRouter = true } - return nil + open, err := a.open(ctx) + if err != nil { + return err + } + var unasked []conditions.Condition + for _, c := range open { + if wants(c, now) { + unasked = append(unasked, c) + } + } + return a.sayUnasked(ctx, unasked, "no router takes the controller's asks: no module holding "+ + broker.AsksSeat+" that takes an ask under its asker's name is assigned") } a.saidNoRouter = false } @@ -248,6 +262,8 @@ func (a *asker) reconcile(ctx context.Context) error { } } wanted := map[string]bool{} + var unasked []conditions.Condition // refused by the router, and nothing it was refused for changed + var refusedWords []string // The most urgent first, then the oldest: those are asked when no more than askMostOpen may be. sort.SliceStable(open, func(i, j int) bool { ui, uj := open[i].Severity == conditions.Urgent, open[j].Severity == conditions.Urgent @@ -272,6 +288,10 @@ func (a *asker) reconcile(ctx context.Context) error { wanted[c.Key] = true if r, was := refused[c.Key]; was && sameAsked(r.Actions, c.Actions) && r.Channels == channels { if _, held := byCondition[c.Key]; !held { + unasked = append(unasked, c) + if r.Warrant != nil && r.Warrant.Words != "" { + refusedWords = append(refusedWords, r.Warrant.Words) + } continue // refused, and nothing it was refused for has changed } } @@ -313,6 +333,45 @@ func (a *asker) reconcile(ctx context.Context) error { } } } + why := "the router refused the ask" + if len(refusedWords) > 0 { + why += ": " + refusedWords[0] + } + return a.sayUnasked(ctx, unasked, why) +} + +// sourceAsker raises the asker's own condition. +const sourceAsker = "asker" + +// sayUnasked keeps the asker's one condition: while a condition that needs the operator could not be asked +// on any channel, said loudly (failure must be loud), cleared when every one could be. +func (a *asker) sayUnasked(ctx context.Context, unasked []conditions.Condition, why string) error { + if a.raise == nil { + return nil + } + var obs []conditions.Observation + if len(unasked) > 0 { + keys := make([]string, 0, len(unasked)) + severity := conditions.Warning + for _, c := range unasked { + keys = append(keys, c.Key) + if c.Severity == conditions.Urgent { + severity = conditions.Urgent + } + } + sort.Strings(keys) + obs = append(obs, conditions.Observation{Scope: conditions.ScopeSeat, ID: broker.AsksSeat, Token: "unasked", + Kind: "asks-undelivered", Severity: severity, Source: sourceAsker, + Summary: fmt.Sprintf("%d condition(s) that need the operator could not be asked on any channel: %s; %s", + len(keys), strings.Join(keys, ", "), why), + Headline: "Questions for you not delivered", + Explanation: "Needs you: answer them from the mesh MCP server. The mesh could not send you its questions on any channel.", + Needs: "answer them from the mesh MCP server, and check why no channel carries them.", + Resolved: "The mesh can ask you again"}) + } + if err := a.raise(ctx, obs); err != nil { + a.logf("whether the operator could be asked could not be kept as a condition: %v", err) + } return nil } diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index 0edcb308..5e40c9c4 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -514,3 +514,52 @@ func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) { } } } + +// Failure is loud (novox/hq ADR 0259, the self-review of 2026-10-09): a condition that needs the operator and +// could not be asked on any channel — no router, or the router refused the ask — is a condition of its own, +// cleared once it can be asked again. +func TestAnAskThatCannotBeDeliveredIsSaid(t *testing.T) { + r := newAskerRig(t) + var raised [][]conditions.Observation + r.a.raise = func(_ context.Context, obs []conditions.Observation) error { + raised = append(raised, obs) + return nil + } + last := func() []conditions.Observation { return raised[len(raised)-1] } + routerHere := false + r.a.routerHere = func(context.Context) (bool, error) { return routerHere, nil } + channels := "channel/telegram=telegram@anchor[choice]own:true" + r.a.channels = func(context.Context) string { return channels } + r.open = []conditions.Condition{heldCondition()} + + _ = r.a.reconcile(context.Background()) + if got := last(); len(got) != 1 || got[0].Kind != "asks-undelivered" || + !strings.Contains(got[0].Summary, heldCondition().Key) || !strings.Contains(got[0].Summary, "no router") { + t.Fatalf("no router, said as %+v", got) + } + + routerHere = true + _ = r.a.reconcile(context.Background()) + if got := last(); len(got) != 0 { + t.Fatalf("asked, and still said undelivered: %+v", got) + } + first := r.asksSent(t)[0] + refusal, _ := json.Marshal(asks.Warrant{Ask: first.ID, Asker: "mesh-controller", Outcome: asks.OutcomeRefused, + Words: "no channel can carry any of its answers now", At: r.now}) + if err := r.a.Decided(context.Background(), refusal); err != nil { + t.Fatal(err) + } + _ = r.a.reconcile(context.Background()) + if got := last(); len(got) != 1 || !strings.Contains(got[0].Summary, "no channel can carry") { + t.Fatalf("the router's refusal, said as %+v", got) + } + if why, ok := conditions.PlainWords(conditions.Words{Headline: last()[0].Headline, Explanation: last()[0].Explanation, + Needs: last()[0].Needs, Resolved: last()[0].Resolved}, ""); !ok { + t.Errorf("not plain: %s", why) + } + r.open = nil + _ = r.a.reconcile(context.Background()) + if got := last(); len(got) != 0 { + t.Errorf("nothing needs asking, and still said: %+v", got) + } +} diff --git a/cmd/mesh-controller/asker_wire.go b/cmd/mesh-controller/asker_wire.go index 9f0ae5f5..dd7ba87a 100644 --- a/cmd/mesh-controller/asker_wire.go +++ b/cmd/mesh-controller/asker_wire.go @@ -283,8 +283,11 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n routerRecord: routerRecordOf(conn, open.inventory), routerHere: routerHereIn(open.inventory), channels: channelsIn(open.inventory), - now: time.Now, - logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }, + raise: func(ctx context.Context, obs []conditions.Observation) error { + return keeper.Reconcile(ctx, sourceAsker, obs) + }, + now: time.Now, + logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }, } if err := server.Decides(a); err != nil { fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err) From 799eec0a5a0132c4ac82970a14d59188e1dcffb5 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 12:24:52 +0200 Subject: [PATCH 07/15] Ask an acknowledgement apart from an approval, change every kept ask by compare-and-set, and rehearse rather than drill (hq ADR 0259, review M1/L2/L3/L7) - M1: a condition offering both kinds of answer is asked twice: its authorising answers about the condition, its acknowledging ones (Silence) apart, so an answer from a channel that only acknowledges never ends an approval. - L2: the asked store creates once and changes only over the revision it read, deciding again on what it reads; a stale cancel no longer writes over an act. - L3: every ask is kept before it is published, one whose publishing failed is marked unsent and asked again, and a cancel is kept before it is said. The terminal's test question is now `rehearse`, so it is not called what the glossary calls a drill; its two answers are both approve-level. - L7: two deliveries of one warrant to two controllers at once act exactly once, on a real bus. - Re-vendored onto mesh-sdk 76902998 (canonical digests): an option binds an asks.Act with each argument as arg.. - The lab's bus fixture composes verified-sender only where the lab says its machine is root-free (MESH_LAB_ASKS_ROOT_FREE=true). --- cmd/mesh-controller/asker.go | 310 +++++++++++++----- cmd/mesh-controller/asker_bus_test.go | 151 +++++++++ cmd/mesh-controller/asker_test.go | 91 ++++- cmd/mesh-controller/asker_wire.go | 79 ++--- cmd/mesh-controller/drill.go | 110 ------- cmd/mesh-controller/drill_test.go | 60 ---- cmd/mesh-controller/main.go | 4 +- cmd/mesh-controller/rehearse.go | 111 +++++++ cmd/mesh-controller/rehearse_test.go | 60 ++++ go.mod | 2 +- go.sum | 2 + internal/inventory/asks_lab_test.go | 8 +- .../novox/mesh-sdk/go/asks/asks.go | 67 +++- vendor/modules.txt | 2 +- 14 files changed, 736 insertions(+), 321 deletions(-) create mode 100644 cmd/mesh-controller/asker_bus_test.go delete mode 100644 cmd/mesh-controller/drill.go delete mode 100644 cmd/mesh-controller/drill_test.go create mode 100644 cmd/mesh-controller/rehearse.go create mode 100644 cmd/mesh-controller/rehearse_test.go diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index 5202fc19..3b28ba5e 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -84,21 +84,74 @@ type asked struct { // Acted is what the controller did on the warrant: empty before it did anything, "acting" while it acts, // then "done", "failed: …" or "nothing: …". Anything but empty is never acted on again. Acted string `json:"acted,omitempty"` - // Drill is an ask started at the controller's terminal (drill.go): about no condition, its answers + // Part is which ask of its condition this is (askPart): empty for the one that carries the condition's + // answers, or the authorising ones where it has both; "acknowledge" for its acknowledging answers asked + // apart (the review of 2026-10-09, M1). + Part string `json:"part,omitempty"` + // Rehearsal is an ask started at the controller's terminal (rehearse.go): about no condition, its answers // perform nothing, and the reconciling of conditions leaves it alone. - Drill bool `json:"drill,omitempty"` + Rehearsal bool `json:"rehearsal,omitempty"` } -// askedStore keeps the asks (broker.AskedBucket). +// partKey is an ask's place among what is asked: its condition and its part. +func partKey(condition, part string) string { return condition + "#" + part } + +// partAcknowledge is the part of a condition asked apart for its acknowledging answers. +const partAcknowledge = "acknowledge" + +// askPart is one ask a condition is asked with: its part, what it is about, and its answers. +type askPart struct { + name string + about string + actions []conditions.Action +} + +// levelOf is an action's level as an option offers it: one that says none is never taken for less than +// approve. +func levelOf(act conditions.Action) asks.Level { + if act.Level == "" { + return asks.Approve + } + return asks.Level(act.Level) +} + +// partsOf is the asks a condition is asked with (the review of 2026-10-09, M1): one, when its answers are all +// of one kind; else its authorising answers (Release, Stop, Restart) in one ask, about the condition, and its +// acknowledging ones (Silence) in another. **An acknowledgement never shares an ask with an approval**: a +// channel that only acknowledges would otherwise answer the ask, and end the approval with it. +func partsOf(c conditions.Condition) []askPart { + var ack, auth []conditions.Action + for _, act := range c.Actions { + if levelOf(act) == asks.Acknowledge { + ack = append(ack, act) + } else { + auth = append(auth, act) + } + } + if len(ack) == 0 || len(auth) == 0 { + return []askPart{{about: c.Key, actions: c.Actions}} + } + return []askPart{{about: c.Key, actions: auth}, + {name: partAcknowledge, about: c.Key + "." + partAcknowledge, actions: ack}} +} + +// askedStore keeps the asks (broker.AskedBucket). **Every write after the first is a compare-and-set** (the +// review of 2026-10-09, L2): an ask is created once, and changed only over the revision it was read at, the +// change decided again on what is read — so two controllers, or two deliveries of one warrant, never write +// over each other, and of two that would act only the one whose write stands does. type askedStore interface { Get(ctx context.Context, id string) (*asked, error) - Put(ctx context.Context, a asked) error + // Create keeps a new ask, and refuses one already kept under its id. + Create(ctx context.Context, a asked) error + // Change applies change to the ask kept under id, by compare-and-set, and says whether its write stood. + // change says whether to write at all; on a write that came between, it is asked again on what is read. + Change(ctx context.Context, id string, change func(*asked) bool) (bool, error) All(ctx context.Context) ([]asked, error) - // Claim marks an open ask acting, by compare-and-set, and says whether this write stood: of two - // deliveries of one warrant, or two controllers, only the one whose write stands acts. - Claim(ctx context.Context, id string, w asks.Warrant) (bool, error) } +// askChangeTries is how often a change is read and tried again when another write came between. +const askChangeTries = 5 + // asker is the controller asking the operator and acting on the answer. type asker struct { open func(ctx context.Context) ([]conditions.Condition, error) @@ -217,11 +270,12 @@ func (a *asker) reconcile(ctx context.Context) error { if err != nil { return err } - byCondition := map[string]asked{} + byCondition := map[string]asked{} // by partKey for _, r := range all { - if r.State == askOpen && !r.Drill { - if prior, held := byCondition[r.Condition]; !held || r.Opened.After(prior.Opened) { - byCondition[r.Condition] = r + if r.State == askOpen && !r.Rehearsal { + k := partKey(r.Condition, r.Part) + if prior, held := byCondition[k]; !held || r.Opened.After(prior.Opened) { + byCondition[k] = r } } } @@ -243,8 +297,8 @@ func (a *asker) reconcile(ctx context.Context) error { } byCondition = map[string]asked{} for _, r := range all { - if r.State == askOpen && !r.Drill { - byCondition[r.Condition] = r + if r.State == askOpen && !r.Rehearsal { + byCondition[partKey(r.Condition, r.Part)] = r } } } @@ -252,12 +306,13 @@ func (a *asker) reconcile(ctx context.Context) error { // newest first: not asked again until the answers or the channels change. answered, refused := map[string]asked{}, map[string]asked{} for _, r := range all { + k := partKey(r.Condition, r.Part) if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer { - answered[r.Condition] = r + answered[k] = r } if r.State == string(asks.OutcomeRefused) { - if prior, has := refused[r.Condition]; !has || r.Opened.After(prior.Opened) { - refused[r.Condition] = r + if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) { + refused[k] = r } } } @@ -277,54 +332,70 @@ func (a *asker) reconcile(ctx context.Context) error { }) openNow := 0 for _, c := range open { - if r, held := byCondition[c.Key]; held && wants(c, now) && sameAsked(r.Actions, c.Actions) && now.Before(r.Ask.Expires) { - openNow++ + if !wants(c, now) { + continue + } + for _, p := range partsOf(c) { + if r, held := byCondition[partKey(c.Key, p.name)]; held && sameAsked(r.Actions, p.actions) && now.Before(r.Ask.Expires) { + openNow++ + } } } for _, c := range open { if !wants(c, now) { continue } - wanted[c.Key] = true - if r, was := refused[c.Key]; was && sameAsked(r.Actions, c.Actions) && r.Channels == channels { - if _, held := byCondition[c.Key]; !held { - unasked = append(unasked, c) - if r.Warrant != nil && r.Warrant.Words != "" { - refusedWords = append(refusedWords, r.Warrant.Words) + saidUnasked := false + for _, p := range partsOf(c) { + key := partKey(c.Key, p.name) + wanted[key] = true + if r, was := refused[key]; was && sameAsked(r.Actions, p.actions) && r.Channels == channels { + if _, held := byCondition[key]; !held { + if !saidUnasked { + unasked, saidUnasked = append(unasked, c), true + } + if r.Warrant != nil && r.Warrant.Words != "" { + refusedWords = append(refusedWords, r.Warrant.Words) + } + continue // refused, and nothing it was refused for has changed } - continue // refused, and nothing it was refused for has changed } - } - if r, done := answered[c.Key]; done && sameAsked(r.Actions, c.Actions) { - if _, held := byCondition[c.Key]; !held { + if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) { + if _, held := byCondition[key]; !held { + continue + } + } + if r, held := byCondition[key]; held { + switch { + case !sameAsked(r.Actions, p.actions): + if err := a.cancel(ctx, r, "its answers changed"); err != nil { + return err + } + case !now.Before(r.Ask.Expires): + // Expired unanswered: the router says so too; asked again below while it lasts. + if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { + if x.State != askOpen { + return false + } + x.State, x.Ended = string(asks.OutcomeExpired), now + return true + }); err != nil { + return err + } + openNow-- + default: + continue + } + } + if openNow >= askMostOpen { + continue // asked when one of the open ones ends, most urgent first + } + if err := a.ask(ctx, c, p, channels); err != nil { + a.logf("the operator could not be asked about %s: %v", c.Key, err) continue } + openNow++ } - if r, held := byCondition[c.Key]; held { - switch { - case !sameAsked(r.Actions, c.Actions): - if err := a.cancel(ctx, r, "its answers changed"); err != nil { - return err - } - case !now.Before(r.Ask.Expires): - // Expired unanswered: the router says so too; asked again below while it lasts. - r.State, r.Ended = string(asks.OutcomeExpired), now - if err := a.store.Put(ctx, r); err != nil { - return err - } - openNow-- - default: - continue - } - } - if openNow >= askMostOpen { - continue // asked when one of the open ones ends, most urgent first - } - if err := a.ask(ctx, c, channels); err != nil { - a.logf("the operator could not be asked about %s: %v", c.Key, err) - continue - } - openNow++ } for key, r := range byCondition { if !wanted[key] { @@ -420,18 +491,15 @@ func askText(s string) string { return strings.ReplaceAll(s, "..", ".") } -// askOf is the ask a condition is asked with. -func askOf(id string, c conditions.Condition, now time.Time) (asks.Ask, map[string]int) { +// askOf is the ask one part of a condition is asked with. +func askOf(id string, c conditions.Condition, p askPart, now time.Time) (asks.Ask, map[string]int) { q := asks.Ask{ID: id, Headline: c.Headline, Explanation: askText(c.Explanation), Who: asks.Operator, - OnExpiry: "nothing is done, and you are asked again while it lasts", About: c.Key, + OnExpiry: "nothing is done, and you are asked again while it lasts", About: p.about, Urgent: c.Severity == conditions.Urgent} options := map[string]int{} approves := false - for i, act := range c.Actions { - level := asks.Level(act.Level) - if level == "" { - level = asks.Approve // an action that says nothing of its level is never taken for less - } + for i, act := range p.actions { + level := levelOf(act) // an action that says nothing of its level is never taken for less than approve approves = approves || level != asks.Acknowledge oid := optionID(act.Label) options[oid] = i @@ -448,10 +516,14 @@ func askOf(id string, c conditions.Condition, now time.Time) (asks.Ask, map[stri return q, options } -// boundAct is what an option's Binds digests: the act exactly as the controller will perform it, with its -// level, and never its label or words. -func boundAct(act conditions.Action) map[string]any { - return map[string]any{"verb": act.Verb, "machine": act.Machine, "arguments": act.Arguments, "level": act.Level} +// boundAct is what an option's Binds digests: the act exactly as the controller will perform it — its verb, +// machine, level, and each argument as "arg." — and never its label or words. +func boundAct(act conditions.Action) asks.Act { + out := asks.Act{"verb": act.Verb, "machine": act.Machine, "level": act.Level} + for k, v := range act.Arguments { + out["arg."+k] = v + } + return out } func newAskID() string { @@ -460,11 +532,15 @@ func newAskID() string { return "c" + hex.EncodeToString(b[:]) } -// ask publishes one ask about a condition, and keeps it. -func (a *asker) ask(ctx context.Context, c conditions.Condition, channels string) error { +// askUnsent is an ask kept and never published: asked again at the next look. +const askUnsent = "unsent" + +// ask publishes one ask about a part of a condition, kept before it is published (the review of 2026-10-09, +// L3): a warrant for it then always finds it, and one whose publishing failed is marked so and asked again. +func (a *asker) ask(ctx context.Context, c conditions.Condition, p askPart, channels string) error { now := a.now() id := newAskID() - q, options := askOf(id, c, now) + q, options := askOf(id, c, p, now) if err := q.Check(now); err != nil { return err } @@ -472,24 +548,47 @@ func (a *asker) ask(ctx context.Context, c conditions.Condition, channels string if err != nil { return err } + if err := a.store.Create(ctx, asked{ID: id, Condition: c.Key, Part: p.name, Ask: q, Actions: p.actions, + Options: options, State: askOpen, Opened: now, Channels: channels}); err != nil { + return fmt.Errorf("the ask could not be kept, so it was not asked: %w", err) + } if err := a.publish(ctx, asks.AskSubject(askerName), body, "ask."+id); err != nil { + if _, cerr := a.store.Change(ctx, id, func(x *asked) bool { + if x.State != askOpen || x.Acted != "" { + return false + } + x.State, x.Ended, x.Acted = askUnsent, a.now(), "nothing: it could not be published: "+err.Error() + return true + }); cerr != nil { + a.logf("the ask %s could not be published, and could not be marked so: %v", id, cerr) + } return err } a.logf("asked the operator about %s (%s): %d answer(s)", c.Key, id, len(q.Options)) - return a.store.Put(ctx, asked{ID: id, Condition: c.Key, Ask: q, Actions: c.Actions, Options: options, - State: askOpen, Opened: now, Channels: channels}) + return nil } -// cancel takes an ask back. +// cancel takes an ask back: kept cancelled first, so a warrant that comes after is refused, then said to the +// router; a cancel the router did not hear leaves the ask to expire there, and nothing is done on it here. func (a *asker) cancel(ctx context.Context, r asked, why string) error { + stood, err := a.store.Change(ctx, r.ID, func(x *asked) bool { + if x.State != askOpen { + return false + } + x.State, x.Ended = askCancelled, a.now() + return true + }) + if err != nil || !stood { + return err + } body, _ := json.Marshal(map[string]string{"id": r.ID}) if err := a.publish(ctx, asks.CancelSubject(askerName), body, "cancel."+r.ID); err != nil { - a.logf("the ask %s about %s could not be taken back (%v); taken back at the next look", r.ID, r.Condition, err) + a.logf("the ask %s about %s is taken back here, and the router could not be told (%v): it expires there, "+ + "and no answer to it is acted on", r.ID, r.Condition, err) return nil } a.logf("took back the ask %s about %s: %s", r.ID, r.Condition, why) - r.State, r.Ended = askCancelled, a.now() - return a.store.Put(ctx, r) + return nil } // Decided takes the router's word on one of the controller's asks (link.Decider). An error is returned only @@ -517,13 +616,21 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { } now := a.now() if w.Outcome != asks.OutcomeChosen { - r.State, r.Ended, r.Warrant = string(w.Outcome), now, &w - r.Acted = "nothing: the ask " + string(w.Outcome) + acted := "nothing: the ask " + string(w.Outcome) if w.Words != "" { - r.Acted += ": " + w.Words + acted += ": " + w.Words + } + if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { + if x.Acted != "" { + return false + } + x.State, x.Ended, x.Warrant, x.Acted = string(w.Outcome), now, &w, acted + return true + }); err != nil { + return err } a.logf("the ask %s about %s ended %s; nothing is done", r.ID, r.Condition, w.Outcome) - return a.store.Put(ctx, *r) + return nil } if r.State != askOpen { // Cancelled, replaced or expired in the controller's own record: no answer to it is acted on. @@ -547,24 +654,39 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { a.logf("REFUSED a warrant for the ask %s: %v", r.ID, err) return nil } - r.State, r.Warrant = string(asks.OutcomeChosen), &w open, err := a.open(ctx) if err != nil { return err } - stillOpen := r.Drill // a drill is about no condition + stillOpen := r.Rehearsal // a rehearsal is about no condition for _, c := range open { stillOpen = stillOpen || c.Key == r.Condition } if !stillOpen { // The asker checks the state is still what it asked about before it acts (to-be 46 §10, step 7). - r.Ended, r.Acted = now, "nothing: the condition ended before the answer" + if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { + if x.State != askOpen || x.Acted != "" { + return false + } + x.State, x.Warrant, x.Ended, x.Acted = string(asks.OutcomeChosen), &w, now, + "nothing: the condition ended before the answer" + return true + }); err != nil { + return err + } a.logf("%s, for %s, which ended meanwhile: nothing is done", w.Says(), r.Condition) - return a.store.Put(ctx, *r) + return nil } // Claimed before acting, by compare-and-set: only the delivery whose write stands acts (security review - // of 2026-10-08, finding 9). - claimed, err := a.store.Claim(ctx, r.ID, w) + // of 2026-10-08, finding 9). Not by the warrant's message id, which another publisher could take first: + // the controller's own record decides. + claimed, err := a.store.Change(ctx, r.ID, func(x *asked) bool { + if x.State != askOpen || x.Acted != "" { + return false + } + x.State, x.Warrant, x.Acted = string(asks.OutcomeChosen), &w, "acting" + return true + }) if err != nil { return err } @@ -584,20 +706,26 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { } var acted error switch { - case r.Drill && act.Verb == drillVerb: - // A drill's answer performs nothing: it is recorded below as the operator's decision. + case r.Rehearsal && act.Verb == rehearsalVerb: + // A rehearsal's answer performs nothing: it is recorded below as the operator's decision. case act.Arguments["silence"] != "": acted = a.silence(ctx, act.Arguments["silence"], conditions.MaxSilence, byWords(w), why) default: acted = a.call(ctx, act, args) } - r.Ended = a.now() - r.Acted = "done" + ended, outcome := a.now(), "done" if acted != nil { - r.Acted = "failed: " + acted.Error() + outcome = "failed: " + acted.Error() } - if err := a.store.Put(ctx, *r); err != nil { - return err + r.Ended, r.Acted = ended, outcome + if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { + if x.Acted != "acting" { + return false + } + x.Ended, x.Acted = ended, outcome + return true + }); err != nil { + a.logf("%s was acted on (%s), and how it ended could NOT be kept: %v", r.ID, outcome, err) } verbArgs := []string{act.Verb} if act.Machine != "" { diff --git a/cmd/mesh-controller/asker_bus_test.go b/cmd/mesh-controller/asker_bus_test.go new file mode 100644 index 00000000..0e22c949 --- /dev/null +++ b/cmd/mesh-controller/asker_bus_test.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/asks" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +// busAsker is an asker on a real bus's `asked` bucket, counting what it performs: two of them are two +// controllers sharing one record. +type busAskerRig struct { + mu sync.Mutex + called int + acts int + open []conditions.Condition + sent [][]byte +} + +func (rig *busAskerRig) asker(t *testing.T, conn *nats.Conn, now time.Time) *asker { + return &asker{ + open: func(context.Context) ([]conditions.Condition, error) { + rig.mu.Lock() + defer rig.mu.Unlock() + return rig.open, nil + }, + silence: func(context.Context, string, time.Duration, string, string) error { return nil }, + store: busAsked{conn: conn}, + publish: func(_ context.Context, subject string, body []byte, _ string) error { + rig.mu.Lock() + defer rig.mu.Unlock() + if subject == asks.AskSubject(askerName) { + rig.sent = append(rig.sent, body) + } + return nil + }, + call: func(context.Context, conditions.Action, map[string]string) error { + time.Sleep(20 * time.Millisecond) // long enough for the other delivery to arrive meanwhile + rig.mu.Lock() + defer rig.mu.Unlock() + rig.called++ + return nil + }, + record: func(context.Context, link.HandAct) error { + rig.mu.Lock() + defer rig.mu.Unlock() + rig.acts++ + return nil + }, + now: func() time.Time { return now }, + logf: t.Logf, + } +} + +func askedBus(t *testing.T) *nats.Conn { + t.Helper() + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + if _, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: broker.AskedBucket}); err != nil { + t.Fatal(err) + } + return conn +} + +// The review of 2026-10-09 (L7): two deliveries of one warrant, to two controllers at once, perform its act +// exactly once and record it once — the record's compare-and-set decides, never the warrant's message id. +func TestTwoAnswersAtOnceActOnce(t *testing.T) { + conn := askedBus(t) + now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC) + rig := &busAskerRig{open: []conditions.Condition{heldCondition()}} + first, second := rig.asker(t, conn, now), rig.asker(t, conn, now) + if err := first.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + if len(rig.sent) != 1 { + t.Fatalf("asked %d times", len(rig.sent)) + } + var q asks.Ask + _ = json.Unmarshal(rig.sent[0], &q) + release, _ := q.Option("release") + w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID, + Label: release.Label, Level: release.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now, + AskDigest: q.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} + body, _ := json.Marshal(w) + var wg sync.WaitGroup + for _, a := range []*asker{first, second, first, second} { + wg.Add(1) + go func(a *asker) { + defer wg.Done() + if err := a.Decided(context.Background(), body); err != nil { + t.Error(err) + } + }(a) + } + wg.Wait() + if rig.called != 1 || rig.acts != 1 { + t.Fatalf("performed %d time(s), recorded %d time(s)", rig.called, rig.acts) + } + got, err := busAsked{conn: conn}.Get(context.Background(), q.ID) + if err != nil || got == nil || got.Acted != "done" { + t.Fatalf("kept as %+v (%v)", got, err) + } +} + +// The review of 2026-10-09 (L2): a write decided on a record read earlier never lands over one made since. A +// cancel read before the answer was acted on leaves the act's record as it is. +func TestAStaleCancelDoesNotWriteOverAnAct(t *testing.T) { + conn := askedBus(t) + now := time.Date(2026, 10, 9, 14, 0, 0, 0, time.UTC) + rig := &busAskerRig{open: []conditions.Condition{heldCondition()}} + a := rig.asker(t, conn, now) + if err := a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + var q asks.Ask + _ = json.Unmarshal(rig.sent[0], &q) + stale, _ := busAsked{conn: conn}.Get(context.Background(), q.ID) + release, _ := q.Option("release") + w := asks.Warrant{Ask: q.ID, Asker: askerName, About: q.About, Outcome: asks.OutcomeChosen, Option: release.ID, + Label: release.Label, Level: release.Level, Channel: "telegram", At: now, AskDigest: q.Digest(), + By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} + body, _ := json.Marshal(w) + if err := a.Decided(context.Background(), body); err != nil { + t.Fatal(err) + } + if err := a.cancel(context.Background(), *stale, "the condition ended"); err != nil { + t.Fatal(err) + } + got, _ := busAsked{conn: conn}.Get(context.Background(), q.ID) + if got.State != string(asks.OutcomeChosen) || got.Acted != "done" { + t.Errorf("a stale cancel wrote over the act: %+v", got) + } +} diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index 5e40c9c4..024fd1fd 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "strings" + "sync" "testing" "time" @@ -19,24 +20,40 @@ import ( type memAskedStore map[string]asked +// memAskedMu guards every memAskedStore: Change is a compare-and-set as the bus's is. +var memAskedMu sync.Mutex + func (m memAskedStore) Get(_ context.Context, id string) (*asked, error) { + memAskedMu.Lock() + defer memAskedMu.Unlock() r, ok := m[id] if !ok { return nil, nil } return &r, nil } -func (m memAskedStore) Put(_ context.Context, r asked) error { m[r.ID] = r; return nil } -func (m memAskedStore) Claim(_ context.Context, id string, w asks.Warrant) (bool, error) { +func (m memAskedStore) Create(_ context.Context, r asked) error { + memAskedMu.Lock() + defer memAskedMu.Unlock() + if _, kept := m[r.ID]; kept { + return errors.New("an ask is kept under that id") + } + m[r.ID] = r + return nil +} +func (m memAskedStore) Change(_ context.Context, id string, change func(*asked) bool) (bool, error) { + memAskedMu.Lock() + defer memAskedMu.Unlock() r, ok := m[id] - if !ok || r.State != askOpen || r.Acted != "" { + if !ok || !change(&r) { return false, nil } - r.State, r.Warrant, r.Acted = string(asks.OutcomeChosen), &w, "acting" m[id] = r return true, nil } func (m memAskedStore) All(context.Context) ([]asked, error) { + memAskedMu.Lock() + defer memAskedMu.Unlock() var out []asked for _, r := range m { out = append(out, r) @@ -437,7 +454,7 @@ func TestTheAskDropsWhereItIsAnsweredAndTheConditionKeepsIt(t *testing.T) { if !strings.Contains(c.Explanation, FromMeshMCPServer) { t.Fatalf("the condition lost where it is answered: %q", c.Explanation) } - q, _ := askOf("x", c, time.Now()) + q, _ := askOf("x", c, partsOf(c)[0], time.Now()) if strings.Contains(q.Explanation, "mesh MCP server") || !strings.HasPrefix(q.Explanation, "Needs you: release it, or stop it.") { t.Errorf("the ask says %q", q.Explanation) } @@ -507,7 +524,7 @@ func TestAWarrantPerformsOnlyTheActItsOptionBound(t *testing.T) { }) } // Every option of an ask binds its act. - q, _ := askOf("x", heldCondition(), time.Now()) + q, _ := askOf("x", heldCondition(), partsOf(heldCondition())[0], time.Now()) for _, o := range q.Options { if o.Binds == "" { t.Errorf("the option %s binds nothing", o.ID) @@ -563,3 +580,65 @@ func TestAnAskThatCannotBeDeliveredIsSaid(t *testing.T) { t.Errorf("nothing needs asking, and still said: %+v", got) } } + +// The review of 2026-10-09 (M1): an acknowledging answer never shares an ask with an authorising one. A +// condition offering Restart and Silence is asked twice — Restart alone, about the condition, and Silence +// alone, apart — so Silence chosen on a channel that only acknowledges leaves the Restart ask open. +func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) { + r := newAskerRig(t) + key := "module.shanks.plex.down" + c := conditions.Condition{Key: key, Kind: "module-down", Severity: conditions.Urgent, Headline: "Plex down on shanks", + Explanation: "Needs you: restart it, or silence this.", Needs: "restart it, or silence this.", + Actions: []conditions.Action{ + {Label: "Restart", Verb: "node-service-manager.restart", Machine: "shanks", Level: conditions.LevelApprove, + Arguments: map[string]string{"unit": "plex"}}, + conditions.SilenceAction(key)}} + r.open = []conditions.Condition{c} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + sent := r.asksSent(t) + if len(sent) != 2 { + t.Fatalf("asked %d time(s): %+v", len(sent), sent) + } + for _, q := range sent { + if err := q.Check(r.now); err != nil { + t.Errorf("%s: %v", q.About, err) + } + levels := map[asks.Level]bool{} + for _, o := range q.Options { + levels[o.Level] = true + } + if len(levels) != 1 { + t.Errorf("the ask about %s mixes levels: %+v", q.About, q.Options) + } + } + byAbout := map[string]asks.Ask{} + for _, q := range sent { + byAbout[q.About] = q + } + if q := byAbout[key]; len(q.Options) != 1 || q.Options[0].Label != "Restart" { + t.Errorf("the condition's own ask: %+v", q) + } + if q := byAbout[key+".acknowledge"]; len(q.Options) != 1 || q.Options[0].Level != asks.Acknowledge { + t.Errorf("the acknowledging ask: %+v", q) + } + // Silence chosen: performed, and the Restart ask stays open, never asked twice. + answerWith(t, r, r.warrantFor(t, key, "Silence for a week")) + if len(r.silenced) != 1 || len(r.called) != 0 { + t.Fatalf("silenced %v called %v", r.silenced, r.called) + } + _ = r.a.reconcile(context.Background()) + open := 0 + for _, a := range r.store { + if a.State == askOpen && a.Condition == key { + open++ + if a.Part != "" || a.Ask.Options[0].Label != "Restart" { + t.Errorf("the open ask is %+v", a) + } + } + } + if open != 1 || len(r.asksSent(t)) != 2 { + t.Errorf("after the silence: %d open, %d asked", open, len(r.asksSent(t))) + } +} diff --git a/cmd/mesh-controller/asker_wire.go b/cmd/mesh-controller/asker_wire.go index dd7ba87a..454986c8 100644 --- a/cmd/mesh-controller/asker_wire.go +++ b/cmd/mesh-controller/asker_wire.go @@ -57,7 +57,8 @@ func (b busAsked) Get(ctx context.Context, id string) (*asked, error) { return &r, json.Unmarshal(e.Value(), &r) } -func (b busAsked) Put(ctx context.Context, r asked) error { +// Create keeps a new ask under its id, and only where none is kept: never over another. +func (b busAsked) Create(ctx context.Context, r asked) error { kv, err := b.kv(ctx) if err != nil { return err @@ -66,10 +67,49 @@ func (b busAsked) Put(ctx context.Context, r asked) error { if err != nil { return err } - _, err = kv.Put(ctx, r.ID, body) + _, err = kv.Create(ctx, r.ID, body) return err } +// Change applies change to the ask kept under id by compare-and-set on its key's revision (the review of +// 2026-10-09, L2): read, changed, and written only over the revision read; when another write came between, +// read again and asked again, at most askChangeTries times. change says whether to write at all. +func (b busAsked) Change(ctx context.Context, id string, change func(*asked) bool) (bool, error) { + kv, err := b.kv(ctx) + if err != nil { + return false, err + } + for try := 0; try < askChangeTries; try++ { + e, err := kv.Get(ctx, id) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil + } + if err != nil { + return false, err + } + var r asked + if err := json.Unmarshal(e.Value(), &r); err != nil { + return false, err + } + if !change(&r) { + return false, nil + } + body, err := json.Marshal(r) + if err != nil { + return false, err + } + if _, err := kv.Update(ctx, id, body, e.Revision()); err != nil { + var api *jetstream.APIError + if errors.Is(err, jetstream.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence) { + continue + } + return false, err + } + return true, nil + } + return false, fmt.Errorf("the ask %s changed under every one of %d tries", id, askChangeTries) +} + func (b busAsked) All(ctx context.Context) ([]asked, error) { kv, err := b.kv(ctx) if err != nil { @@ -94,41 +134,6 @@ func (b busAsked) All(ctx context.Context) ([]asked, error) { return out, nil } -// Claim marks an open ask acting, by compare-and-set on its key's revision: only the write that stands acts. -func (b busAsked) Claim(ctx context.Context, id string, w asks.Warrant) (bool, error) { - kv, err := b.kv(ctx) - if err != nil { - return false, err - } - e, err := kv.Get(ctx, id) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return false, nil - } - if err != nil { - return false, err - } - var r asked - if err := json.Unmarshal(e.Value(), &r); err != nil { - return false, err - } - if r.State != askOpen || r.Acted != "" { - return false, nil - } - r.State, r.Warrant, r.Acted = string(asks.OutcomeChosen), &w, "acting" - body, err := json.Marshal(r) - if err != nil { - return false, err - } - if _, err := kv.Update(ctx, id, body, e.Revision()); err != nil { - var api *jetstream.APIError - if errors.Is(err, jetstream.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence) { - return false, nil - } - return false, err - } - return true, nil -} - // callAction performs an action's verb as the controller, through the grant that names it. func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error { return func(ctx context.Context, a conditions.Action, args map[string]string) error { diff --git a/cmd/mesh-controller/drill.go b/cmd/mesh-controller/drill.go deleted file mode 100644 index 1db79cf8..00000000 --- a/cmd/mesh-controller/drill.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -// The drill of the operator's answers (novox/hq ADR 0259, the live acceptance after rollout): an ask the -// operator starts at the controller's terminal, answered on the phone, whose approval changes nothing and is -// recorded as a person's decision like any other. -// -// mesh-controller drill [--for 15m] -// -// It asks with two answers — Approve (an approval, so only a channel that proves who answered carries it) and -// Decline (an acknowledgement) — each bound to the drill's own act. The serving controller acts on the warrant -// as on any other: it claims the ask once, checks the act is the one bound, performs nothing, and records the -// hand-act `warrant` with who answered, through which channel, and the proofs. `hand-acts` then shows it. -// -// **The terminal's alone**: a command a verb runs (MESH_VERB set) is refused, so no agent starts a drill — a -// drill is a question the operator expects, and one an agent could start would teach them to approve what they -// did not ask for. - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "os" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/asks" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// drillVerb is the act a drill's answers bind: nothing is called. -const drillVerb = "drill" - -// drillActions are the drill's two answers. -func drillActions() []conditions.Action { - return []conditions.Action{ - {Label: "Approve", Verb: drillVerb, Level: conditions.LevelApprove, Arguments: map[string]string{"drill": "approve"}}, - {Label: "Decline", Verb: drillVerb, Level: conditions.LevelAcknowledge, Arguments: map[string]string{"drill": "decline"}}, - } -} - -// drillAsk is the drill's ask, as the router is sent it. -func drillAsk(id string, now time.Time, lasts time.Duration) (asks.Ask, map[string]int) { - q := asks.Ask{ID: id, Headline: "Drill: approve this test question?", Who: asks.Operator, - Explanation: "Needs you: approve or decline. You started this drill at the controller's terminal. Approving " + - "changes nothing on the mesh; it is recorded as your decision, so you can check the record.", - OnExpiry: "nothing is done", Expires: now.Add(lasts), About: "drill." + id} - options := map[string]int{} - for i, act := range drillActions() { - binds, _ := asks.ActDigest(boundAct(act)) - oid := optionID(act.Label) - options[oid] = i - q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesDrill(act), Level: asks.Level(act.Level), - Binds: binds}) - } - return q, options -} - -func doesDrill(act conditions.Action) string { - if act.Arguments["drill"] == "approve" { - return "nothing changes; your approval is recorded" - } - return "nothing changes; your answer is recorded" -} - -func drillCommand(ctx context.Context, args []string) error { - if os.Getenv(verbVar) != "" { - return errors.New("drill is the controller's terminal's alone: a verb may not start one, so no agent asks " + - "the operator a question they did not start (novox/hq ADR 0259)") - } - set := flag.NewFlagSet("drill", flag.ContinueOnError) - lasts := set.Duration("for", 15*time.Minute, "how long the question waits for an answer") - if err := set.Parse(args); err != nil { - return err - } - if *lasts < time.Minute || *lasts > askApproveFor { - return fmt.Errorf("a drill waits between a minute and %s", askApproveFor) - } - js, err := aBus() - if err != nil { - return err - } - defer js.Close() - now := time.Now() - id := newAskID() - q, options := drillAsk(id, now, *lasts) - if err := q.Check(now); err != nil { - return err - } - store := busAsked{conn: js.Conn()} - // Kept before it is published, as the asker keeps every ask, so a warrant always finds it. - if err := store.Put(ctx, asked{ID: id, Condition: q.About, Ask: q, Actions: drillActions(), Options: options, - State: askOpen, Opened: now, Drill: true}); err != nil { - return fmt.Errorf("the drill could not be kept in the controller's asks: %w", err) - } - body, err := json.Marshal(q) - if err != nil { - return err - } - if _, err := js.Context().Publish(asks.AskSubject(askerName), body, nats.MsgId("ask."+id), nats.Context(ctx)); err != nil { - return fmt.Errorf("the drill could not be asked: %w", err) - } - fmt.Printf("drill %s asked: answer it on your phone before %s. Then `mesh-controller hand-acts` shows the "+ - "answer as a warrant, with who answered, through which channel and the proofs; nothing else changes.\n", - id, q.Expires.Local().Format("15:04")) - return nil -} diff --git a/cmd/mesh-controller/drill_test.go b/cmd/mesh-controller/drill_test.go deleted file mode 100644 index 074abd4b..00000000 --- a/cmd/mesh-controller/drill_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" -) - -// A drill (the live acceptance of novox/hq ADR 0259): its approval is a warrant like any other — claimed once, -// its act checked against what the option bound, recorded as the operator's decision with who, how and the -// proofs — and it performs nothing. The reconciling of conditions leaves it open. -func TestADrillsApprovalIsRecordedAndPerformsNothing(t *testing.T) { - r := newAskerRig(t) - q, options := drillAsk("cdrill", r.now, askerDrillFor) - if err := q.Check(r.now); err != nil { - t.Fatalf("the drill's ask is refused: %v", err) - } - r.store["cdrill"] = asked{ID: "cdrill", Condition: q.About, Ask: q, Actions: drillActions(), Options: options, - State: askOpen, Opened: r.now, Drill: true} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - if got := r.store["cdrill"]; got.State != askOpen { - t.Fatalf("the reconciling of conditions ended the drill: %+v", got) - } - approve, _ := q.Option("approve") - w := asks.Warrant{Ask: "cdrill", Asker: "mesh-controller", Outcome: asks.OutcomeChosen, Option: approve.ID, - Label: approve.Label, Level: approve.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now, - AskDigest: q.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} - answerWith(t, r, w) - answerWith(t, r, w) // heard again - if len(r.called)+len(r.silenced) != 0 { - t.Errorf("a drill performed something: %v %v", r.called, r.silenced) - } - if len(r.acts) != 1 { - t.Fatalf("hand-acts %+v", r.acts) - } - act := r.acts[0] - if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" || act.Ask != "cdrill" || - strings.Join(act.Args, " ") != "drill drill=approve" || act.Outcome != "done" || strings.Join(act.Proofs, ",") != "P1" { - t.Errorf("the drill's record: %+v", act) - } - if !personsDecision(act) { - t.Error("a drill's answer counts as a repair") - } -} - -// Only the terminal starts a drill: a verb's process is refused before anything is asked. -func TestADrillIsTheTerminalsAlone(t *testing.T) { - t.Setenv(verbVar, "mesh-controller.command") - if err := drillCommand(context.Background(), nil); err == nil || !strings.Contains(err.Error(), "terminal") { - t.Fatalf("a verb started a drill: %v", err) - } -} - -// askerDrillFor is how long the test's drill waits. -const askerDrillFor = 15 * time.Minute diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 7ea824ba..15674cd0 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -78,8 +78,8 @@ func run() error { return rotateCommand(ctx, args[1:]) case "ask": return askCommand(ctx, args[1:]) - case "drill": - return drillCommand(ctx, args[1:]) + case "rehearse": + return rehearseCommand(ctx, args[1:]) case "builds": return buildsCommand(ctx, args[1:]) // The build queue, controlled by hand (novox/hq ADR 0219). diff --git a/cmd/mesh-controller/rehearse.go b/cmd/mesh-controller/rehearse.go new file mode 100644 index 00000000..5ec46044 --- /dev/null +++ b/cmd/mesh-controller/rehearse.go @@ -0,0 +1,111 @@ +package main + +// The rehearsal of the operator's answers — not a drill, which in the glossary is something broken on purpose (novox/hq ADR 0259, the live acceptance after rollout): an ask the +// operator starts at the controller's terminal, answered on the phone, whose approval changes nothing and is +// recorded as a person's decision like any other. +// +// mesh-controller rehearse [--for 15m] +// +// It asks with two answers, Approve and Decline, each bound to the rehearsal's own act and **both at the level +// approve** (the review of 2026-10-09, M1: an acknowledgement never shares an ask with an approval), so only a +// channel that proves who answered carries either — the rehearsal is of exactly that. The serving controller acts on the warrant +// as on any other: it claims the ask once, checks the act is the one bound, performs nothing, and records the +// hand-act `warrant` with who answered, through which channel, and the proofs. `hand-acts` then shows it. +// +// **The terminal's alone**: a command a verb runs (MESH_VERB set) is refused, so no agent starts a rehearsal — a +// rehearsal is a question the operator expects, and one an agent could start would teach them to approve what they +// did not ask for. + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "time" + + "github.com/nats-io/nats.go" + + "git.novox.be/novox/mesh-sdk/go/asks" + + "github.com/novox/mesh-controller/internal/conditions" +) + +// rehearsalVerb is the act a rehearsal's answers bind: nothing is called. +const rehearsalVerb = "rehearsal" + +// rehearsalActions are the rehearsal's two answers. +func rehearsalActions() []conditions.Action { + return []conditions.Action{ + {Label: "Approve", Verb: rehearsalVerb, Level: conditions.LevelApprove, Arguments: map[string]string{"rehearsal": "approve"}}, + {Label: "Decline", Verb: rehearsalVerb, Level: conditions.LevelApprove, Arguments: map[string]string{"rehearsal": "decline"}}, + } +} + +// rehearsalAsk is the rehearsal's ask, as the router is sent it. +func rehearsalAsk(id string, now time.Time, lasts time.Duration) (asks.Ask, map[string]int) { + q := asks.Ask{ID: id, Headline: "Rehearsal: approve this test question?", Who: asks.Operator, + Explanation: "Needs you: approve or decline. You started this rehearsal at the controller's terminal. Approving " + + "changes nothing on the mesh; it is recorded as your decision, so you can check the record.", + OnExpiry: "nothing is done", Expires: now.Add(lasts), About: "rehearsal." + id} + options := map[string]int{} + for i, act := range rehearsalActions() { + binds, _ := asks.ActDigest(boundAct(act)) + oid := optionID(act.Label) + options[oid] = i + q.Options = append(q.Options, asks.Option{ID: oid, Label: act.Label, Does: doesRehearsal(act), Level: asks.Level(act.Level), + Binds: binds}) + } + return q, options +} + +func doesRehearsal(act conditions.Action) string { + if act.Arguments["rehearsal"] == "approve" { + return "nothing changes; your approval is recorded" + } + return "nothing changes; your answer is recorded" +} + +func rehearseCommand(ctx context.Context, args []string) error { + if os.Getenv(verbVar) != "" { + return errors.New("rehearse is the controller's terminal's alone: a verb may not start one, so no agent asks " + + "the operator a question they did not start (novox/hq ADR 0259)") + } + set := flag.NewFlagSet("rehearse", flag.ContinueOnError) + lasts := set.Duration("for", 15*time.Minute, "how long the question waits for an answer") + if err := set.Parse(args); err != nil { + return err + } + if *lasts < time.Minute || *lasts > askApproveFor { + return fmt.Errorf("a rehearsal waits between a minute and %s", askApproveFor) + } + js, err := aBus() + if err != nil { + return err + } + defer js.Close() + now := time.Now() + id := newAskID() + q, options := rehearsalAsk(id, now, *lasts) + if err := q.Check(now); err != nil { + return err + } + store := busAsked{conn: js.Conn()} + // Kept before it is published, as the asker keeps every ask, so a warrant always finds it. + if err := store.Create(ctx, asked{ID: id, Condition: q.About, Ask: q, Actions: rehearsalActions(), Options: options, + State: askOpen, Opened: now, Rehearsal: true}); err != nil { + return fmt.Errorf("the rehearsal could not be kept in the controller's asks: %w", err) + } + body, err := json.Marshal(q) + if err != nil { + return err + } + if _, err := js.Context().Publish(asks.AskSubject(askerName), body, nats.MsgId("ask."+id), nats.Context(ctx)); err != nil { + return fmt.Errorf("the rehearsal could not be asked: %w", err) + } + fmt.Printf("rehearsal %s asked: answer it on your phone before %s. Then `mesh-controller hand-acts` shows the "+ + "answer as a warrant, with who answered, through which channel and the proofs; nothing else changes.\n", + id, q.Expires.Local().Format("15:04")) + return nil +} diff --git a/cmd/mesh-controller/rehearse_test.go b/cmd/mesh-controller/rehearse_test.go new file mode 100644 index 00000000..4cfdeb20 --- /dev/null +++ b/cmd/mesh-controller/rehearse_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// A rehearsal (the live acceptance of novox/hq ADR 0259): its approval is a warrant like any other — claimed once, +// its act checked against what the option bound, recorded as the operator's decision with who, how and the +// proofs — and it performs nothing. The reconciling of conditions leaves it open. +func TestARehearsalsApprovalIsRecordedAndPerformsNothing(t *testing.T) { + r := newAskerRig(t) + q, options := rehearsalAsk("crehearsal", r.now, askerRehearsalFor) + if err := q.Check(r.now); err != nil { + t.Fatalf("the rehearsal's ask is refused: %v", err) + } + r.store["crehearsal"] = asked{ID: "crehearsal", Condition: q.About, Ask: q, Actions: rehearsalActions(), Options: options, + State: askOpen, Opened: r.now, Rehearsal: true} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + if got := r.store["crehearsal"]; got.State != askOpen { + t.Fatalf("the reconciling of conditions ended the rehearsal: %+v", got) + } + approve, _ := q.Option("approve") + w := asks.Warrant{Ask: "crehearsal", Asker: "mesh-controller", Outcome: asks.OutcomeChosen, Option: approve.ID, + Label: approve.Label, Level: approve.Level, Channel: "telegram", Proofs: []string{"P1"}, At: r.now, + AskDigest: q.Digest(), By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} + answerWith(t, r, w) + answerWith(t, r, w) // heard again + if len(r.called)+len(r.silenced) != 0 { + t.Errorf("a rehearsal performed something: %v %v", r.called, r.silenced) + } + if len(r.acts) != 1 { + t.Fatalf("hand-acts %+v", r.acts) + } + act := r.acts[0] + if act.Verb != handActWarrant || act.By != "the operator, as telegram identity 42" || act.Ask != "crehearsal" || + strings.Join(act.Args, " ") != "rehearsal rehearsal=approve" || act.Outcome != "done" || strings.Join(act.Proofs, ",") != "P1" { + t.Errorf("the rehearsal's record: %+v", act) + } + if !personsDecision(act) { + t.Error("a rehearsal's answer counts as a repair") + } +} + +// Only the terminal starts a rehearsal: a verb's process is refused before anything is asked. +func TestARehearsalIsTheTerminalsAlone(t *testing.T) { + t.Setenv(verbVar, "mesh-controller.command") + if err := rehearseCommand(context.Background(), nil); err == nil || !strings.Contains(err.Error(), "terminal") { + t.Fatalf("a verb started a rehearsal: %v", err) + } +} + +// askerRehearsalFor is how long the test's rehearsal waits. +const askerRehearsalFor = 15 * time.Minute diff --git a/go.mod b/go.mod index af4c63ee..84b6bb66 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8 + git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39 github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 46fc965e..52ecaa40 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f h1:BNvyWq899 git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261008162031-55090da7e08f/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8 h1:soqhLNpEXThdq6PdiPy6ExxjJ+yjhh1N1n9E3j1CtrM= git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39 h1:WHW6CgbuTxP7M+qRBOgzsiG9vT49xdkZ/rarc9/vKMA= +git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/inventory/asks_lab_test.go b/internal/inventory/asks_lab_test.go index 62593d54..a96a9a73 100644 --- a/internal/inventory/asks_lab_test.go +++ b/internal/inventory/asks_lab_test.go @@ -98,7 +98,13 @@ func TestTheAsksLabBus(t *testing.T) { Emits: own.Emits, Serves: own.Serves} } records := broker.Records{Nodes: []string{labMachine}, Assigned: map[string][]broker.Declared{}, - People: map[string][]string{}, Interchangeable: map[string]bool{}} + People: map[string][]string{}, Interchangeable: map[string]bool{}, RootFree: map[string]bool{}} + // Whether the lab's machine is root-free is the lab's to say (MESH_LAB_ASKS_ROOT_FREE=true): it has no + // node-engine to judge it. Unsaid, it is not, and no kind is composed with verified-sender — as a push + // composes on a machine that is not (novox/hq ADR 0259 §8). + if os.Getenv("MESH_LAB_ASKS_ROOT_FREE") == "true" { + records.RootFree[labMachine] = true + } var buckets []broker.Bucket var trafficSeats []broker.Seat for _, m := range manifests { diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go index ca2e8006..efe2a661 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go @@ -8,10 +8,11 @@ package asks import ( "crypto/sha256" "encoding/hex" - "encoding/json" "errors" "fmt" "regexp" + "sort" + "strconv" "strings" "time" ) @@ -107,25 +108,67 @@ type Option struct { Binds string `json:"binds,omitempty"` } -// ActDigest is the digest an asker puts in Option.Binds: SHA-256 over the act's JSON (Go's encoding sorts a -// map's keys, so the same act gives the same digest), written "sha256:". -func ActDigest(act any) (string, error) { - raw, err := json.Marshal(act) - if err != nil { - return "", fmt.Errorf("the act cannot be digested: %w", err) +// An Act is what an option binds: named fields, each a string — the verb, the machine, the level, and each +// argument under a name of its own ("arg.delivery"). Flat on purpose: its digest is over these names and +// values alone, never over how a language happens to encode a struct. +type Act map[string]string + +// ActDigest is the digest an asker puts in Option.Binds: SHA-256 over the act's canonical encoding (canonical), +// written "sha256:". The same names and values give the same digest in any language, whatever order +// they were set in; a field renamed, added or emptied gives another. +func ActDigest(act Act) (string, error) { + if len(act) == 0 { + return "", fmt.Errorf("the act cannot be digested: it names nothing") } - sum := sha256.Sum256(raw) + keys := make([]string, 0, len(act)) + for k := range act { + if k == "" { + return "", fmt.Errorf("the act cannot be digested: a field has no name") + } + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + b.WriteString("novox.act.v1\n") + for _, k := range keys { + canonical(&b, k) + canonical(&b, act[k]) + } + sum := sha256.Sum256([]byte(b.String())) return "sha256:" + hex.EncodeToString(sum[:]), nil } +// canonical writes one value as its length in bytes, a colon, the bytes and a newline: no value can be read +// as another's end or start, so two different sequences of values never encode the same. +func canonical(b *strings.Builder, v string) { + b.WriteString(strconv.Itoa(len(v))) + b.WriteByte(':') + b.WriteString(v) + b.WriteByte('\n') +} + // Digest is the digest of the ask exactly as its asker published it: its id, words, options with what each // binds, who answers, and its expiry. The router puts it in the warrant (Warrant.AskDigest), and an asker // acts only on a warrant whose digest is that of the ask it keeps — so a warrant answers one ask, as the // person was shown it, and nothing published under the same id before or after. +// +// It is over the ask's named fields in a fixed order, each written canonically, and the expiry as UTC +// RFC 3339 to the nanosecond — never over a language's encoding of the struct, so a field added to Ask +// later changes no digest until it is added here, on purpose. func (a Ask) Digest() string { - a.Expires = a.Expires.UTC() - raw, _ := json.Marshal(a) - sum := sha256.Sum256(raw) + var b strings.Builder + b.WriteString("novox.ask.v1\n") + for _, v := range []string{a.ID, a.Headline, a.Explanation, a.Who, + a.Expires.UTC().Format(time.RFC3339Nano), a.OnExpiry, a.About, strconv.FormatBool(a.Urgent), + strconv.Itoa(len(a.Options))} { + canonical(&b, v) + } + for _, o := range a.Options { + for _, v := range []string{o.ID, o.Label, o.Does, string(o.Level), o.Binds} { + canonical(&b, v) + } + } + sum := sha256.Sum256([]byte(b.String())) return "sha256:" + hex.EncodeToString(sum[:]) } @@ -341,7 +384,7 @@ func (w Warrant) For(asker string, a Ask) (Option, error) { // Performs checks that the act an asker is about to perform is the one the chosen option bound when it // asked: the act's digest equals the option's Binds. An acknowledge option that bound nothing passes. -func (o Option) Performs(act any) error { +func (o Option) Performs(act Act) error { if o.Binds == "" && o.Level == Acknowledge { return nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index f7f6eb66..dcaf690d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8 +# git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39 ## explicit; go 1.22 git.novox.be/novox/mesh-sdk/go/asks # github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op From d19c9ed5b784fcbeb8f8e542e49f599be4e1c1b2 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:53:28 +0200 Subject: [PATCH 08/15] Start a rehearsal only at the controller's terminal as main now judges it (hq ADR 0259, ADR 0272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rehearse refused only a verb's process. Since mesh-cli (ADR 0272 §4) the serving controller runs an ordinary mesh-cli line without a verb, naming its caller: such a line, from an agent's account, read as the terminal and could start a question the operator did not ask. rehearse now asks startedAtTheTerminal. And main's mesh-cli test helper asked is cliAsked, beside the asker's asked. --- cmd/mesh-controller/meshcli_test.go | 38 ++++++++++++++-------------- cmd/mesh-controller/rehearse.go | 13 ++++++---- cmd/mesh-controller/rehearse_test.go | 16 ++++++++++++ 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/cmd/mesh-controller/meshcli_test.go b/cmd/mesh-controller/meshcli_test.go index 8dcc5522..7968f0c8 100644 --- a/cmd/mesh-controller/meshcli_test.go +++ b/cmd/mesh-controller/meshcli_test.go @@ -24,7 +24,7 @@ var cliNodes = []inventory.Node{ {Name: "unnamed"}, } -func asked(account string, uid uint32, line ...string) link.CLIAsked { +func cliAsked(account string, uid uint32, line ...string) link.CLIAsked { return link.CLIAsked{Line: line, Account: account, UID: uid, Session: "session-1.scope"} } @@ -39,13 +39,13 @@ func TestMeshCLIIsTheTerminalOnlyForTheControlNodesOperator(t *testing.T) { refused string why string }{ - {"the control-node's operator", "control", asked("operator", 1000, "status"), control, true, "", "the controller's terminal"}, - {"another node's operator", "laptop", asked("operator", 1000, "status"), control, false, "", "agents on laptop may run as operator"}, - {"another account", "control", asked("agent", 1001, "status"), control, false, "operator account (operator) only", ""}, - {"root", "control", asked("root", 0, "status"), control, false, "never root", ""}, - {"a node with no operator account", "unnamed", asked("operator", 1000, "status"), control, false, "does not know unnamed's operator account", ""}, - {"a node the mesh does not know", "elsewhere", asked("operator", 1000, "status"), control, false, "not a node this mesh knows", ""}, - {"two control-nodes", "control", asked("operator", 1000, "status"), []string{"control", "laptop"}, false, "", "2 control-nodes"}, + {"the control-node's operator", "control", cliAsked("operator", 1000, "status"), control, true, "", "the controller's terminal"}, + {"another node's operator", "laptop", cliAsked("operator", 1000, "status"), control, false, "", "agents on laptop may run as operator"}, + {"another account", "control", cliAsked("agent", 1001, "status"), control, false, "operator account (operator) only", ""}, + {"root", "control", cliAsked("root", 0, "status"), control, false, "never root", ""}, + {"a node with no operator account", "unnamed", cliAsked("operator", 1000, "status"), control, false, "does not know unnamed's operator account", ""}, + {"a node the mesh does not know", "elsewhere", cliAsked("operator", 1000, "status"), control, false, "not a node this mesh knows", ""}, + {"two control-nodes", "control", cliAsked("operator", 1000, "status"), []string{"control", "laptop"}, false, "", "2 control-nodes"}, } for _, c := range cases { v := judgeCLI(c.node, c.asked, cliNodes, c.control) @@ -70,7 +70,7 @@ func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T) t.Setenv(servedVar, "1") ctx := context.Background() - a := runForMeshCLI(ctx, "control", asked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"}) + a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"}) if a.Exit != 0 || a.Refused != "" || !a.Terminal { t.Fatalf("the terminal's line did not run: %+v", a) } @@ -79,7 +79,7 @@ func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T) t.Fatalf("the terminal's line ran with %s", got) } - a = runForMeshCLI(ctx, "laptop", asked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) + a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) if a.Exit != 0 || a.Terminal || a.Why != "not the terminal" { t.Fatalf("an ordinary line did not run as one: %+v", a) } @@ -93,21 +93,21 @@ func TestAnOrdinaryCallMeetsTheCommandVerbsRefusals(t *testing.T) { t.Setenv(echoEnvironment, "1") ctx := context.Background() ordinary := cliVerdict{why: "not the terminal"} - a := runForMeshCLI(ctx, "laptop", asked("operator", 1000, "cleanup", "delete", "x"), ordinary) + a := runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "cleanup", "delete", "x"), ordinary) if a.Refused == "" || len(a.Stdout) != 0 || a.Exit != 1 || a.Why != "not the terminal" { t.Fatalf("a repair without --why ran as an ordinary call: %+v", a) } - a = runForMeshCLI(ctx, "laptop", asked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary) + a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary) if a.Refused != "" || !strings.Contains(string(a.Stdout), `verb="mesh-cli"`) { t.Fatalf("an ordinary settings set did not run through the settings verb's path with MESH_VERB set: %+v", a) } for _, server := range []string{"serve", "api", "board"} { - a := runForMeshCLI(ctx, "control", asked("operator", 1000, server), cliVerdict{terminal: true}) + a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, server), cliVerdict{terminal: true}) if a.Refused == "" || len(a.Stdout) != 0 { t.Fatalf("%s was run for mesh-cli: %+v", server, a) } } - a = runForMeshCLI(ctx, "control", asked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) + a = runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) if a.Refused != "agent is not answered" || len(a.Stdout) != 0 { t.Fatalf("a refused line ran: %+v", a) } @@ -184,9 +184,9 @@ func TestEveryMeshCLILineIsSaidInTheJournal(t *testing.T) { cliJournal = func(line string) { said = append(said, line) } t.Cleanup(func() { cliJournal = was }) ctx := link.WithCallID(context.Background(), "call-1") - runForMeshCLI(ctx, "control", asked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`), + runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`), cliVerdict{terminal: true, why: "the terminal"}) - runForMeshCLI(ctx, "control", asked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) + runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) all := strings.Join(said, "\n") if len(said) != 2 || !strings.Contains(all, "call-1") || !strings.Contains(all, "operator on control") || !strings.Contains(all, "as the controller's terminal") || !strings.Contains(all, "refused") { @@ -213,7 +213,7 @@ func TestAnOrdinaryLineRunsNothingTheCommandVerbWouldRefuse(t *testing.T) { if _, err := ordinaryLine(line); err == nil { t.Errorf("%q composed as an ordinary line", line) } - a := runForMeshCLI(context.Background(), "laptop", asked("operator", 1000, line...), cliVerdict{why: "not the terminal"}) + a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, line...), cliVerdict{why: "not the terminal"}) if a.Refused == "" || len(a.Stdout) != 0 { t.Errorf("%q ran as an ordinary line: %+v", line, a) } @@ -279,11 +279,11 @@ func TestTheTerminalsMarkIsStrippedFromEveryOtherLine(t *testing.T) { } } } - a := runForMeshCLI(context.Background(), "laptop", asked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) + a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) if got := string(a.Stdout); !strings.Contains(got, "terminal=false") || !strings.Contains(got, `verb="mesh-cli"`) { t.Fatalf("an ordinary line with the mark in the serving environment ran as %s", got) } - a = runForMeshCLI(context.Background(), "control", asked("operator", 1000, "status"), cliVerdict{terminal: true}) + a = runForMeshCLI(context.Background(), "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true}) if got := string(a.Stdout); !strings.Contains(got, "terminal=true") { t.Fatalf("the terminal's line ran as %s", got) } diff --git a/cmd/mesh-controller/rehearse.go b/cmd/mesh-controller/rehearse.go index 5ec46044..15836946 100644 --- a/cmd/mesh-controller/rehearse.go +++ b/cmd/mesh-controller/rehearse.go @@ -12,7 +12,8 @@ package main // as on any other: it claims the ask once, checks the act is the one bound, performs nothing, and records the // hand-act `warrant` with who answered, through which channel, and the proofs. `hand-acts` then shows it. // -// **The terminal's alone**: a command a verb runs (MESH_VERB set) is refused, so no agent starts a rehearsal — a +// **The terminal's alone** (startedAtTheTerminal): a command a verb runs, an ordinary mesh-cli line and anything the +// serving controller started are refused, so no agent starts a rehearsal — a // rehearsal is a question the operator expects, and one an agent could start would teach them to approve what they // did not ask for. @@ -22,7 +23,6 @@ import ( "errors" "flag" "fmt" - "os" "time" "github.com/nats-io/nats.go" @@ -68,9 +68,12 @@ func doesRehearsal(act conditions.Action) string { } func rehearseCommand(ctx context.Context, args []string) error { - if os.Getenv(verbVar) != "" { - return errors.New("rehearse is the controller's terminal's alone: a verb may not start one, so no agent asks " + - "the operator a question they did not start (novox/hq ADR 0259)") + // The terminal as main judges it (startedAtTheTerminal): not a verb, not the serving controller or anything it + // started, and a mesh-cli line only when it is the control-node's operator's (novox/hq ADR 0272 §4). + if !startedAtTheTerminal() { + return errors.New("rehearse is the controller's terminal's alone: a verb, a mesh-cli line from anybody but " + + "the control-node's operator, or a process the serving controller started may not start one, so no agent " + + "asks the operator a question they did not start (novox/hq ADR 0259)") } set := flag.NewFlagSet("rehearse", flag.ContinueOnError) lasts := set.Duration("for", 15*time.Minute, "how long the question waits for an answer") diff --git a/cmd/mesh-controller/rehearse_test.go b/cmd/mesh-controller/rehearse_test.go index 4cfdeb20..df738810 100644 --- a/cmd/mesh-controller/rehearse_test.go +++ b/cmd/mesh-controller/rehearse_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "github.com/novox/mesh-controller/internal/link" "strings" "testing" "time" @@ -54,6 +55,21 @@ func TestARehearsalIsTheTerminalsAlone(t *testing.T) { if err := rehearseCommand(context.Background(), nil); err == nil || !strings.Contains(err.Error(), "terminal") { t.Fatalf("a verb started a rehearsal: %v", err) } + // Nor a mesh-cli line from anybody but the control-node's operator (hq ADR 0272 §4): run without a verb, + // naming its caller, and without the terminal's mark — and nor anything the serving controller started. + for name, env := range map[string]map[string]string{ + "an ordinary mesh-cli line": {verbVar: "", link.CallerVar: "laptop/agent"}, + "a process the serving controller ran": {verbVar: "", servedVar: "1"}, + } { + t.Run(name, func(t *testing.T) { + for k, v := range env { + t.Setenv(k, v) + } + if err := rehearseCommand(context.Background(), nil); err == nil || !strings.Contains(err.Error(), "terminal") { + t.Fatalf("%s started a rehearsal: %v", name, err) + } + }) + } } // askerRehearsalFor is how long the test's rehearsal waits. From 74d35600a69596fc849927ac0566f9dbb0f9ad1a Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:56:09 +0200 Subject: [PATCH 09/15] Keep an approval asked through a silence of its condition (hq ADR 0259, the confirmation review's M1) Choosing Silence silenced the condition, the condition was no longer wanted, and the next reconcile cancelled the Restart or Release ask beside it: an acknowledgement, which any desk click may give, took an approval back. An open approval ask now stays until it is answered or expires while its condition is open and silenced with the same answers. The test silences as the controller does; it failed before (0 open) and passes, and the kept Restart is performed on its warrant. --- cmd/mesh-controller/asker.go | 31 +++++++++++++++++++++++++++---- cmd/mesh-controller/asker_test.go | 12 ++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index 3b28ba5e..e589d2a8 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -397,11 +397,24 @@ func (a *asker) reconcile(ctx context.Context) error { openNow++ } } + stillOpen := map[string]conditions.Condition{} + for _, c := range open { + stillOpen[c.Key] = c + } for key, r := range byCondition { - if !wanted[key] { - if err := a.cancel(ctx, r, "the condition ended, was silenced or needs nothing now"); err != nil { - return err - } + if wanted[key] { + continue + } + // **A silence never takes an approval back** (the confirmation review of 2026-10-09, M1). Silence is an + // acknowledgement — anyone at the desk may give it — so a condition silenced while its approval is asked + // keeps that ask open, unchanged, until it is answered on a channel that proves who answered, or expires. + // It is not asked again once it ends, while the silence lasts. + if c, open := stillOpen[r.Condition]; open && c.SilencedAt(now) && r.Ask.Highest() != asks.Acknowledge && + now.Before(r.Ask.Expires) && keepsItsAnswers(c, r) { + continue + } + if err := a.cancel(ctx, r, "the condition ended, was silenced or needs nothing now"); err != nil { + return err } } why := "the router refused the ask" @@ -411,6 +424,16 @@ func (a *asker) reconcile(ctx context.Context) error { return a.sayUnasked(ctx, unasked, why) } +// keepsItsAnswers says a condition still offers the answers an ask kept was asked with. +func keepsItsAnswers(c conditions.Condition, r asked) bool { + for _, p := range partsOf(c) { + if partKey(c.Key, p.name) == partKey(r.Condition, r.Part) { + return sameAsked(r.Actions, p.actions) + } + } + return false +} + // sourceAsker raises the asker's own condition. const sourceAsker = "asker" diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index 024fd1fd..225c38e3 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -83,6 +83,13 @@ func newAskerRig(t *testing.T) *askerRig { open: func(context.Context) ([]conditions.Condition, error) { return r.open, nil }, silence: func(_ context.Context, key string, d time.Duration, by, why string) error { r.silenced = append(r.silenced, key+" for "+d.String()+" by "+by+" because "+why) + // As the controller's conditions do (the confirmation review of 2026-10-09, M1): the condition is + // silenced from now on, so what is asked next sees it silenced. + for i := range r.open { + if r.open[i].Key == key { + r.open[i].Silenced = &conditions.Silence{Until: r.now.Add(d), By: by, Why: why, Since: r.now} + } + } return nil }, store: r.store, @@ -641,4 +648,9 @@ func TestAnAcknowledgementNeverSharesAnAskWithAnApproval(t *testing.T) { if open != 1 || len(r.asksSent(t)) != 2 { t.Errorf("after the silence: %d open, %d asked", open, len(r.asksSent(t))) } + // And the approval still answers: Restart chosen on a channel that proves who answered is performed. + answerWith(t, r, r.warrantFor(t, key, "Restart")) + if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "node-service-manager.restart@shanks") { + t.Errorf("the approval kept through a silence was not performed: %v", r.called) + } } From 0559b808874bdd01ecc502e334b0c01d115f79c0 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:59:20 +0200 Subject: [PATCH 10/15] Move to mesh-sdk 16984aa, rebased on its main, which refuses a warrant with no time or for an ask with no expiry --- go.mod | 2 +- go.sum | 6 ++++-- vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go | 12 ++++++++++-- vendor/modules.txt | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 84b6bb66..d93602d8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39 + git.novox.be/novox/mesh-sdk/go v0.1.10-0.20261009115850-16984aafc689 github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 52ecaa40..8e1e030c 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8 h1:soqhLNpEX git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009081503-d4077b473ea8/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39 h1:WHW6CgbuTxP7M+qRBOgzsiG9vT49xdkZ/rarc9/vKMA= git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.10-0.20261009115850-16984aafc689 h1:Ti2P9nwders7YQ/hq3X/dPo+CXMj5pdUcfA5P/c12CU= +git.novox.be/novox/mesh-sdk/go v0.1.10-0.20261009115850-16984aafc689/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -50,10 +52,10 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go index efe2a661..81737ac4 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go @@ -374,8 +374,16 @@ func (w Warrant) For(asker string, a Ask) (Option, error) { if w.Level != o.Level { return Option{}, fmt.Errorf("the option %s is %s, and the warrant says %s", o.ID, o.Level, w.Level) } - // A choice made after the ask expired is no answer to it, whatever the router said. - if !w.At.IsZero() && w.At.After(a.Expires) { + // A choice made after the ask expired is no answer to it, whatever the router said. An ask that says no + // expiry, or a warrant that says no time, is no answer either: neither can be shown to be in time (the + // confirmation review of 2026-10-09). + if a.Expires.IsZero() { + return Option{}, fmt.Errorf("the ask %s says no expiry, so no answer to it can be in time", a.ID) + } + if w.At.IsZero() { + return Option{}, fmt.Errorf("the warrant for the ask %s says no time it was given, so it cannot be shown to be in time", a.ID) + } + if w.At.After(a.Expires) { return Option{}, fmt.Errorf("the warrant was given at %s, after the ask %s expired at %s", w.At.UTC().Format(time.RFC3339), a.ID, a.Expires.UTC().Format(time.RFC3339)) } diff --git a/vendor/modules.txt b/vendor/modules.txt index dcaf690d..93fd8b8f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.8-0.20261009095928-76902998cd39 +# git.novox.be/novox/mesh-sdk/go v0.1.10-0.20261009115850-16984aafc689 ## explicit; go 1.22 git.novox.be/novox/mesh-sdk/go/asks # github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op From 568955340607b833c99c01e16e62d7ef1426ccb2 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 17:10:22 +0200 Subject: [PATCH 11/15] =?UTF-8?q?Take=20a=20module's=20own=20secret=20thro?= =?UTF-8?q?ugh=20a=20hidden=20prompt=20on=20the=20operator's=20desk,=20so?= =?UTF-8?q?=20a=20bot=20token=20never=20passes=20through=20an=20agent's=20?= =?UTF-8?q?session=20(hq=20ADR=200259=20=C2=A710)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/desk_secret.go | 167 +++++++++++++++++++ cmd/mesh-controller/desk_secret_test.go | 141 ++++++++++++++++ cmd/mesh-controller/handacts.go | 3 + cmd/mesh-controller/seatverbs.go | 5 + cmd/mesh-controller/seatverbs_schema_test.go | 7 + cmd/mesh-controller/secret.go | 11 +- cmd/mesh-controller/signals.go | 2 +- internal/broker/nats.go | 8 + internal/broker/testdata/composed.conf | 2 +- internal/catalogue/graphical_session.go | 12 ++ internal/catalogue/graphical_session_test.go | 2 +- internal/catalogue/verbs.go | 12 ++ internal/inventory/secrets.go | 13 ++ module.json | 1 + 14 files changed, 382 insertions(+), 4 deletions(-) create mode 100644 cmd/mesh-controller/desk_secret.go create mode 100644 cmd/mesh-controller/desk_secret_test.go diff --git a/cmd/mesh-controller/desk_secret.go b/cmd/mesh-controller/desk_secret.go new file mode 100644 index 00000000..d0441c9d --- /dev/null +++ b/cmd/mesh-controller/desk_secret.go @@ -0,0 +1,167 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/secrets" +) + +// A module's own secret given at the operator's desk (novox/hq ADR 0259 §10). +// +// **The value never passes through whoever asked for it.** An agent, or the operator at the mesh MCP +// server, calls `give` with the machine, the module, the secret's name and the desk — never a value. The +// controller makes a sealing keypair for this one call, asks the desk's `node-launcher.secret` to prompt the +// operator without showing what is typed, and is answered with what was typed **sealed to that key**: no +// plaintext on the bus, in a runtime's log or in any call's record. It opens it here, seals it to the +// module's machine exactly as `secret accept` does, and forgets it. What it answers says only that the +// value was taken, or why not. +// +// **What remains** (ADR 0234's accepted residual risk): on an X11 desk any program of the operator's +// account can read the keys as they are typed. And a program that calls the desk's prompt itself, with a +// key of its own, is answered with what the operator typed into a prompt they did not ask for — as it could +// draw a window of its own. The prompt says who asks and for what, so the operator types only into a +// prompt they started. + +// deskPromptWithin is how long the prompt waits for the operator: below the runtime's thirty seconds for +// one call, as the launcher's menu is. +const deskPromptWithin = 25 + +// deskGive is the desk path, its four reaches given so a test needs no store and no bus. +type deskGive struct { + // declares refuses a module or a secret the mesh would refuse, before anybody is asked to type. + declares func(module, name string) error + // ask asks one machine's node-launcher.secret and answers its result, or the holder's refusal. + ask func(machine string, args map[string]any) (json.RawMessage, error) + // accept seals the value as `secret accept` does, and says whether it lives until the module's start. + accept func(value string) (untilStart bool, err error) + // record writes the act in the hand-act log. + record func(link.HandAct) error +} + +// errNothingGiven is a prompt dismissed, or not answered in time: nothing changes. +var errNothingGiven = errors.New("nothing was given") + +// give asks the desk for the value and seals it; it answers the words said to the caller. +func (d deskGive) give(node, module, name, desk string) (string, error) { + for what, v := range map[string]string{"the machine": node, "the module": module, "the secret": name, "the desk": desk} { + if strings.TrimSpace(v) == "" { + return "", fmt.Errorf("%s is not named", what) + } + } + if err := d.declares(module, name); err != nil { + return "", fmt.Errorf("nobody was asked to type anything: %w", err) + } + public, private, err := secrets.Keypair() + if err != nil { + return "", fmt.Errorf("no key could be made to take the value: %w", err) + } + raw, err := d.ask(desk, map[string]any{ + "prompt": name + " for " + module, + "message": fmt.Sprintf("The mesh asks for %s, the own secret of %s on %s. What you type is not shown, "+ + "and is sealed before it leaves this machine. Type it only if you asked for this.", name, module, node), + "seal_to": public, + "timeout_seconds": deskPromptWithin, + }) + if err != nil { + return "", fmt.Errorf("the desk on %s could not be asked: %w", desk, err) + } + var answer struct { + Sealed string `json:"sealed"` + Cancelled bool `json:"cancelled"` + TimedOut bool `json:"timed_out"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + return "", fmt.Errorf("the desk on %s answered something that is not the prompt's answer", desk) + } + switch { + case answer.TimedOut: + return "", fmt.Errorf("%w: the prompt on %s was not answered within %d seconds", errNothingGiven, desk, deskPromptWithin) + case answer.Cancelled: + return "", fmt.Errorf("%w: the prompt on %s was dismissed", errNothingGiven, desk) + case answer.Sealed == "": + return "", fmt.Errorf("the desk on %s answered no sealed value", desk) + } + opened, err := secrets.Open(private, answer.Sealed) + if err != nil { + // Never the value, never what failed to open: only that it was not sealed to this call. + return "", fmt.Errorf("the desk on %s answered a value not sealed to this call; nothing was taken", desk) + } + value := asSupplied(string(opened)) + for i := range opened { + opened[i] = 0 + } + if strings.TrimSpace(value) == "" { + return "", fmt.Errorf("%w: the prompt on %s was answered empty", errNothingGiven, desk) + } + untilStart, err := d.accept(value) + value = "" + if err != nil { + return "", err + } + act := link.HandAct{Verb: "secret accept", Args: []string{node, module, name, "--at-desk", desk}, + Why: fmt.Sprintf("the operator gave %s for %s on %s at the desk on %s", name, module, node, desk), + Cause: "given-at-the-desk"} + recorded := "" + if err := d.record(act); err != nil { + recorded = fmt.Sprintf("\n this act could NOT be recorded in the hand-act log, and is done anyway: %v", err) + } + words := fmt.Sprintf("%s on %s now holds %q, given at the desk on %s and sealed to %s; the mesh cannot read it "+ + "back.\n run `push %s` to send it", module, node, name, desk, node, node) + if untilStart { + words += fmt.Sprintf("\n it lives until %s next starts well under the mesh, and is then replaced with a value "+ + "the mesh makes (ADR 0228)", module) + } + return words + recorded, nil +} + +// giveAtDesk is `secret accept --at-desk `: the desk path, on this +// controller's stores and bus. +func giveAtDesk(ctx context.Context, node, module, name, desk string) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + d := deskGive{ + declares: func(module, name string) error { return open.inventory.DeclaresOwnSecret(ctx, module, name) }, + ask: func(machine string, args map[string]any) (json.RawMessage, error) { + var result json.RawMessage + err := onTheBus(func(conn *nats.Conn) error { + answer, err := link.AskSeatTool(ctx, conn, "node-launcher", "secret", machine, args, + time.Duration(deskPromptWithin+5)*time.Second) + if err != nil { + return err + } + if answer.Error != "" { + return errors.New(answer.Error) + } + result = answer.Result + return nil + }) + return result, err + }, + accept: func(value string) (bool, error) { + return open.inventory.AcceptGivenSecret(ctx, node, module, name, value) + }, + record: func(act link.HandAct) error { + return onTheBus(func(conn *nats.Conn) error { + _, err := link.RecordHandAct(ctx, conn, act) + return err + }) + }, + } + words, err := d.give(node, module, name, desk) + if err != nil { + return err + } + fmt.Println(words) + return nil +} diff --git a/cmd/mesh-controller/desk_secret_test.go b/cmd/mesh-controller/desk_secret_test.go new file mode 100644 index 00000000..6cd29545 --- /dev/null +++ b/cmd/mesh-controller/desk_secret_test.go @@ -0,0 +1,141 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/secrets" +) + +const typed = "123456789:AAEhBP0av28P4XFQnIuR-o-7Xnz1kkUzW3g" + +// aDesk is the desk path with a prompt the test answers as the operator would, and what it was asked kept. +func aDesk(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) (deskGive, *[]string, *[]link.HandAct, *[]map[string]any) { + t.Helper() + var accepted []string + var acts []link.HandAct + var asked []map[string]any + return deskGive{ + declares: func(module, name string) error { + if module != "telegram" || name != "telegram-token" { + return errors.New(module + " does not declare " + name + " as an own secret") + } + return nil + }, + ask: func(machine string, args map[string]any) (json.RawMessage, error) { + asked = append(asked, args) + return answer(args) + }, + accept: func(value string) (bool, error) { accepted = append(accepted, value); return false, nil }, + record: func(a link.HandAct) error { acts = append(acts, a); return nil }, + }, &accepted, &acts, &asked +} + +func sealedTo(t *testing.T, value string) func(args map[string]any) (json.RawMessage, error) { + return func(args map[string]any) (json.RawMessage, error) { + sealed, err := secrets.Seal(args["seal_to"].(string), []byte(value+"\n")) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(map[string]any{"sealed": sealed}) + return raw, nil + } +} + +// novox/hq ADR 0259 §10: the value typed at the desk is sealed as `secret accept` seals it, and is in no +// answer, no prompt argument and no act recorded. +func TestASecretGivenAtTheDeskIsSealedAndSaidNowhere(t *testing.T) { + d, accepted, acts, asked := aDesk(t, sealedTo(t, typed)) + words, err := d.give("anchor", "telegram", "telegram-token", "laptop") + if err != nil { + t.Fatal(err) + } + if len(*accepted) != 1 || (*accepted)[0] != typed { + t.Fatalf("the value sealed is not what was typed, its line ending taken off") + } + if len(*acts) != 1 || (*acts)[0].Verb != "secret accept" || (*acts)[0].Cause != "given-at-the-desk" || + !strings.Contains((*acts)[0].Why, "at the desk on laptop") { + t.Errorf("the act: %+v", *acts) + } + raw, _ := json.Marshal(struct { + Words string + Acts []link.HandAct + Asked []map[string]any + }{words, *acts, *asked}) + if strings.Contains(string(raw), typed) || strings.Contains(string(raw), "AAEhBP0") { + t.Fatal("the value appears in what was said, asked or recorded") + } + if !strings.Contains(words, "push anchor") || !strings.Contains(words, "given at the desk on laptop") { + t.Errorf("%q", words) + } + if p := (*asked)[0]; p["seal_to"] == "" || p["timeout_seconds"] != deskPromptWithin { + t.Errorf("the prompt was asked %v", p) + } +} + +func TestNothingIsAskedForASecretTheMeshWouldRefuse(t *testing.T) { + for _, c := range [][2]string{{"telegram", "chat-id"}, {"nobody", "telegram-token"}} { + d, accepted, _, asked := aDesk(t, sealedTo(t, typed)) + if _, err := d.give("anchor", c[0], c[1], "laptop"); err == nil || !strings.Contains(err.Error(), "nobody was asked") { + t.Errorf("%v: %v", c, err) + } + if len(*asked) != 0 || len(*accepted) != 0 { + t.Errorf("%v: the operator was asked anyway", c) + } + } + d, _, _, _ := aDesk(t, sealedTo(t, typed)) + if _, err := d.give("anchor", "telegram", "telegram-token", ""); err == nil { + t.Error("no desk was refused nowhere") + } +} + +func TestADismissedEmptyLateOrForeignAnswerTakesNothing(t *testing.T) { + for want, answer := range map[string]func(map[string]any) (json.RawMessage, error){ + "not answered within 25 seconds": func(map[string]any) (json.RawMessage, error) { + return json.RawMessage(`{"cancelled":true,"timed_out":true}`), nil + }, + "was dismissed": func(map[string]any) (json.RawMessage, error) { return json.RawMessage(`{"cancelled":true}`), nil }, + "answered empty": sealedTo(t, " "), + "not sealed to this call": func(map[string]any) (json.RawMessage, error) { + other, _, _ := secrets.Keypair() + sealed, _ := secrets.Seal(other, []byte(typed)) + raw, _ := json.Marshal(map[string]any{"sealed": sealed}) + return raw, nil + }, + "could not be asked": func(map[string]any) (json.RawMessage, error) { return nil, errors.New("no session answers") }, + } { + d, accepted, acts, _ := aDesk(t, answer) + _, err := d.give("anchor", "telegram", "telegram-token", "laptop") + if err == nil || !strings.Contains(err.Error(), want) || strings.Contains(err.Error(), typed) { + t.Errorf("want %q, got %v", want, err) + } + if len(*accepted) != 0 || len(*acts) != 0 { + t.Errorf("%s: something was taken or recorded", want) + } + } +} + +func TestTheGiveVerbRunsTheDeskPathAndTheControllerMayAskTheDesk(t *testing.T) { + argv, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token", "at": "laptop"}) + if err != nil || strings.Join(argv, " ") != "secret accept anchor telegram telegram-token --at-desk laptop" { + t.Fatalf("%v %v", argv, err) + } + if _, err := argvFor("give", map[string]any{"node": "anchor", "module": "telegram", "secret": "telegram-token"}); err == nil { + t.Error("give without a desk was taken") + } + perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController}) + if err != nil { + t.Fatal(err) + } + found := false + for _, p := range perms.Publish { + found = found || p == "mesh.seat.node-launcher.tool.secret.*" + } + if !found { + t.Error("the controller may not ask the desk's prompt") + } +} diff --git a/cmd/mesh-controller/handacts.go b/cmd/mesh-controller/handacts.go index 0f2fae3e..ae1916bf 100644 --- a/cmd/mesh-controller/handacts.go +++ b/cmd/mesh-controller/handacts.go @@ -110,6 +110,9 @@ var handActVerbs = []handActVerb{ // to that judgement. Several values rotate for one leak, and a leak that recurs is a defect of the // module that prints them, an issue against it, not a healer that rotates. A rotation for any other // cause — a credential that stopped working — counts: a schedule or a healer could take it over. + // A value given at the desk (novox/hq ADR 0259 §10): an outside party's key, such as a bot token, which + // only a person can give. Their word, never a repair. + {Verb: "secret accept", Decision: "a value an outside party issued is given by a person, at their desk"}, {Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word", DecidedFor: []string{causeLeakedInLogs}}, } diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index a9441024..308fcbbd 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -762,6 +762,11 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, "--probe", p) } return append(argv, "--json"), nil + case "give": + if err := need("node", "module", "secret", "at"); err != nil { + return nil, err + } + return []string{"secret", "accept", str("node"), str("module"), str("secret"), "--at-desk", str("at")}, nil case "rotate": if p := str("provision"); p != "" { argv := []string{"rotate", p} diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index 176b3455..2a6a6410 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -275,6 +275,13 @@ var accountedFlags = map[string]map[string]string{ "json": "set by the verb: the answer is data", "all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it", }, + // The desk path of `secret accept` (novox/hq ADR 0259 §10): a value is never an argument of a call. + "secret accept": { + "at-desk": "=at", + "from": "withheld: a file of the control node's is read at a shell, never named by a call", + "provider": "withheld: a pair credential's value is given at a shell; give takes a module's own secret", + "local": "withheld: it goes with --provider", + }, "hand-acts": {"json": "set by the verb: the answer is data"}, "conditions": {"json": "set by the verb: the answer is data"}, "retire": {"json": "set by the verb: the answer is data"}, diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index 46321e1f..2d489658 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -55,6 +55,9 @@ func secretCommand(ctx context.Context, args []string) error { provider := set.String("provider", "", "the node providing : the value becomes the PAIR credential between on "+ "and that provider, sealed to both — the vault's operator-delivered secret (ADR 0092)") + desk := set.String("at-desk", "", + "ask the operator for the value in a prompt that does not show it, on this machine's desk; the "+ + "answer comes back sealed to this call alone (novox/hq ADR 0259 §10)") local := set.String("local", "", "with --provider: the name the credential goes by inside , where its manifest keeps "+ "several for (ADR 0094)") @@ -65,6 +68,12 @@ func secretCommand(ctx context.Context, args []string) error { return errors.New(secretUsage) } node, module, name := rest[0], rest[1], rest[2] + if *desk != "" { + if *from != "" || *provider != "" { + return errors.New("--at-desk gives a module's own secret, and takes neither --from nor --provider") + } + return giveAtDesk(ctx, node, module, name, *desk) + } value, err := valueFor(node, module, name, *from) if err != nil { @@ -118,7 +127,7 @@ func secretCommand(ctx context.Context, args []string) error { } const secretUsage = "secret rotate [--why [--cause ]]\n" + - "secret accept [--from ] [--provider [--local ]]\n" + + "secret accept [--from | --at-desk ] [--provider [--local ]]\n" + "secret recover --key [--out ] [--from-export ] [--provider ]\n" + "secret export [--out ]" diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 3b2e82a2..287aab96 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -84,7 +84,7 @@ const ( // callBounds are the verbs that may run longer than callDefault, and how long (S7). var callBounds = map[string]time.Duration{ - "push": 30 * time.Minute, "rotate": 30 * time.Minute, "assign": 15 * time.Minute, + "push": 30 * time.Minute, "rotate": 30 * time.Minute, "give": 5 * time.Minute, "assign": 15 * time.Minute, "unassign": 15 * time.Minute, "command": 30 * time.Minute, "doctor": 3 * time.Minute, } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 3ac3f1b9..415059eb 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -183,6 +183,10 @@ var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: // the controller's grant that acts, and only through the step a person starts. var VerbsTheBusStepAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}} +// VerbsTheControllerAsksForASecret are the seat verbs `give` calls (novox/hq ADR 0259 §10): the operator's +// desk opens a prompt that does not show what is typed, and answers it sealed to the controller's call. +var VerbsTheControllerAsksForASecret = []SeatVerb{{Seat: "node-launcher", Verb: "secret"}} + // VerbsTheControllerAsksTheDeliveryOwner are the mesh-delivery seat's verbs the controller calls (novox/hq // ADR 0239): its self-check reads `stalled`, and healer H2 takes the one transition the table allows // through `close`. A mesh seat's verb is flat: no machine in the subject. @@ -392,6 +396,10 @@ func PermissionsFor(p Principal) (Permissions, error) { for _, v := range VerbsTheBusStepAsks { pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*") } + // And the operator's desk, for a secret given there (ADR 0259 §10). + for _, v := range VerbsTheControllerAsksForASecret { + pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*") + } // And the delivery's owner, a mesh seat, asked on its flat subjects (ADR 0239). for _, v := range VerbsTheControllerAsksTheDeliveryOwner { pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb) diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 2b1aa418..953f80f1 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-service-manager.tool.restart.*"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/catalogue/graphical_session.go b/internal/catalogue/graphical_session.go index a03a5082..09e738ae 100644 --- a/internal/catalogue/graphical_session.go +++ b/internal/catalogue/graphical_session.go @@ -78,6 +78,18 @@ func graphicalSessionSeats() []Seat { "description": "the lines to choose between, in order"}, "prompt": map[string]any{"type": "string", "description": "what the menu asks (optional)"}, }}}, + // A value the operator types and nobody sees (novox/hq ADR 0259 §10): a hidden prompt whose answer + // is sealed to the asker's key, so it is never plaintext on the bus or in any call's record. + // **Optional while its holders catch up** (ADR 0246): rofi serves it once this is live. + {Name: "secret", Optional: true, Description: "Ask the operator for a value in a prompt that " + + "does not show what is typed, and answer it sealed to the key the asker gives — never in " + + "the clear — or cancelled when the prompt was dismissed or not answered in time.", + Input: schema(map[string]string{ + "prompt": "what the prompt asks", + "message": "a line saying who asks and for what (optional)", + "seal_to": "the asker's public sealing key: the answer is sealed to it", + "timeout_seconds": "give up after this long (optional)", + }, []string{"prompt", "seal_to"})}, }}, {Name: NotifierSeat, Scope: ScopeNode, Decision: decided, Serves: []Verb{ {Name: "send", Description: "Show the operator a notification.", diff --git a/internal/catalogue/graphical_session_test.go b/internal/catalogue/graphical_session_test.go index d349e1d0..f412e246 100644 --- a/internal/catalogue/graphical_session_test.go +++ b/internal/catalogue/graphical_session_test.go @@ -16,7 +16,7 @@ func TestTheGraphicalSessionsSeatsAreTheMeshsOwnWithTheirVerbs(t *testing.T) { DisplayServerSeat: {"displays", "layout"}, DisplaySessionSeat: {"reload", "workspaces", "windows"}, TerminalEmulatorSeat: {"open"}, - LauncherSeat: {"menu"}, + LauncherSeat: {"menu", "secret"}, NotifierSeat: {"send", "history"}, LockScreenSeat: {"lock"}, ClipboardSeat: {"history", "copy"}, diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index f63b91df..a052ab98 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -230,6 +230,18 @@ var ControllerVerbs = []Verb{ "why": "an own secret: why it is rotated — recorded in the hand-act log (optional)", "cause": "with why: the cause in a word, the word a second rotation for the same reason uses (optional)", }, nil)}, + {Name: "give", Description: "Take a module's own secret from the operator at their desk (novox/hq ADR 0259 " + + "§10): a prompt that does not show what is typed opens on the machine named by at, its answer comes " + + "back sealed to this call alone, and is sealed to the module's machine as `secret accept` seals it. " + + "The value is never an argument and never in the answer: the answer says it was taken, or why not. " + + "Recorded in the hand-act log as a value given at the desk. The prompt waits 25 seconds; dismissed " + + "or unanswered, nothing changes. Then push the machine.", + Input: schema(map[string]string{ + "node": "the machine the module runs on, which the secret is sealed to", + "module": "the module's name", + "secret": "the own secret's name in the module's definition", + "at": "the machine the operator sits at, where the prompt opens", + }, []string{"node", "module", "secret", "at"})}, {Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " + "machine as the module's own secret named broker, read at the next push of that machine. For a module " + "whose definition declares that secret; refused with the reason otherwise. Issued again, it replaces the account.", diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index cebe1df4..6e001eeb 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -554,6 +554,19 @@ func (i *Inventory) declared(ctx context.Context, module string) (catalogue.Mani return m, nil } +// DeclaresOwnSecret refuses, in words, a module the mesh does not know or an own secret its definition does +// not declare: asked before anybody is asked for a value, so nobody types one the mesh would refuse. +func (i *Inventory) DeclaresOwnSecret(ctx context.Context, module, name string) error { + m, err := i.declared(ctx, module) + if err != nil { + return err + } + if _, ok := m.OwnSecrets[name]; !ok { + return fmt.Errorf("%s does not declare %q as an own secret; %s", module, name, declaresOwn(m)) + } + return nil +} + func declaresOwn(m catalogue.Manifest) string { if len(m.OwnSecrets) == 0 { return "it declares no own secrets" diff --git a/module.json b/module.json index 89f35a66..3d49f65a 100644 --- a/module.json +++ b/module.json @@ -48,6 +48,7 @@ "unpin", "push", "rotate", + "give", "issue", "token", "settings", From be59f29f46286f12ae776d419f41f8817d305942 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 12:12:25 +0200 Subject: [PATCH 12/15] give: take only a value a person holds, ask the desk by name, let the controller alone ask it, and announce every value given The review of 2026-10-09 (M4): - give refuses broker (the bus account issue mints) and any own secret the mesh may make itself; - the desk's prompt is asked by module, secret and machine, never with words of the caller's, and the bus denies the prompt's subjects to every principal but the controller, however wide its grant, so the prompt's 'the controller asks' is the bus's word (Permissions.PublishDeny, broker.ControllerOnly); - secret accept with a value is refused through a verb: a value comes from the terminal or the desk; - every value given for an own secret, at the terminal or the desk, raises the urgent condition secret-given on every channel, until the operator silences it. --- cmd/mesh-controller/desk_secret.go | 46 ++++++++++++- cmd/mesh-controller/desk_secret_test.go | 87 +++++++++++++++++++++++++ cmd/mesh-controller/secret.go | 9 +++ internal/broker/nats.go | 59 +++++++++++++++-- internal/catalogue/graphical_session.go | 10 ++- internal/inventory/givable_test.go | 28 ++++++++ internal/inventory/secrets.go | 24 ++++++- 7 files changed, 250 insertions(+), 13 deletions(-) create mode 100644 internal/inventory/givable_test.go diff --git a/cmd/mesh-controller/desk_secret.go b/cmd/mesh-controller/desk_secret.go index d0441c9d..963f3ac5 100644 --- a/cmd/mesh-controller/desk_secret.go +++ b/cmd/mesh-controller/desk_secret.go @@ -10,6 +10,7 @@ import ( "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/secrets" ) @@ -44,6 +45,9 @@ type deskGive struct { accept func(value string) (untilStart bool, err error) // record writes the act in the hand-act log. record func(link.HandAct) error + // announce raises the condition that says a module's own secret was given (secretGivenObservation), on + // every channel; nil announces nothing (a test that does not look). + announce func(node, module, name, how string) error } // errNothingGiven is a prompt dismissed, or not answered in time: nothing changes. @@ -63,10 +67,12 @@ func (d deskGive) give(node, module, name, desk string) (string, error) { if err != nil { return "", fmt.Errorf("no key could be made to take the value: %w", err) } + // By name, never by words: the holder writes the prompt from these, and says the controller asks, which + // the bus alone makes true (broker.ControllerOnly). raw, err := d.ask(desk, map[string]any{ - "prompt": name + " for " + module, - "message": fmt.Sprintf("The mesh asks for %s, the own secret of %s on %s. What you type is not shown, "+ - "and is sealed before it leaves this machine. Type it only if you asked for this.", name, module, node), + "module": module, + "secret": name, + "node": node, "seal_to": public, "timeout_seconds": deskPromptWithin, }) @@ -113,6 +119,11 @@ func (d deskGive) give(node, module, name, desk string) (string, error) { if err := d.record(act); err != nil { recorded = fmt.Sprintf("\n this act could NOT be recorded in the hand-act log, and is done anyway: %v", err) } + if d.announce != nil { + if err := d.announce(node, module, name, "at the desk on "+desk); err != nil { + recorded += fmt.Sprintf("\n this change could NOT be announced on the operator's channels: %v", err) + } + } words := fmt.Sprintf("%s on %s now holds %q, given at the desk on %s and sealed to %s; the mesh cannot read it "+ "back.\n run `push %s` to send it", module, node, name, desk, node, node) if untilStart { @@ -157,6 +168,7 @@ func giveAtDesk(ctx context.Context, node, module, name, desk string) error { return err }) }, + announce: func(node, module, name, how string) error { return announceSecretGiven(ctx, node, module, name, how) }, } words, err := d.give(node, module, name, desk) if err != nil { @@ -165,3 +177,31 @@ func giveAtDesk(ctx context.Context, node, module, name, desk string) error { fmt.Println(words) return nil } + +// kindSecretGiven is the condition every value given for a module's own secret raises (the review of 2026-10-09, +// M4): on every channel, so a bot token changed by somebody else — a channel that now answers for them — is +// heard of. It stays until the operator silences or clears it. +const kindSecretGiven = "secret-given" + +// secretGivenObservation is that condition: which secret, of which module on which machine, how and when. +func secretGivenObservation(node, module, name, how string, at time.Time) conditions.Observation { + key := node + "." + module + "." + name + return conditions.Observation{Scope: conditions.ScopeMachine, ID: key, Token: kindSecretGiven, Kind: kindSecretGiven, + Machine: node, Severity: conditions.Urgent, Source: kindSecretGiven, + Summary: fmt.Sprintf("%s of %s on %s was given %s at %s", name, module, node, how, + at.Local().Format("2006-01-02 15:04")), + Headline: "Secret of " + module + " changed", + Explanation: fmt.Sprintf("The secret %s of %s on %s was given %s at %s. If you did not do this, "+ + "somebody else holds what %s acts with.", name, module, node, how, at.Local().Format("15:04"), module), + Needs: "silence this if you gave it; if you did not, give the secret again yourself and unlink what it serves.", + Resolved: "You saw that " + name + " of " + module + " was changed", + Actions: []conditions.Action{conditions.SilenceAction(conditions.Key(conditions.ScopeMachine, key, kindSecretGiven))}} +} + +// announceSecretGiven raises it on this controller's keeper. +func announceSecretGiven(ctx context.Context, node, module, name, how string) error { + return withKeeper(ctx, func(k *conditions.Keeper) error { + _, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, time.Now())) + return err + }) +} diff --git a/cmd/mesh-controller/desk_secret_test.go b/cmd/mesh-controller/desk_secret_test.go index 6cd29545..eef0bc04 100644 --- a/cmd/mesh-controller/desk_secret_test.go +++ b/cmd/mesh-controller/desk_secret_test.go @@ -1,12 +1,15 @@ package main import ( + "context" "encoding/json" "errors" "strings" "testing" + "time" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/secrets" ) @@ -139,3 +142,87 @@ func TestTheGiveVerbRunsTheDeskPathAndTheControllerMayAskTheDesk(t *testing.T) { t.Error("the controller may not ask the desk's prompt") } } + +// The review of 2026-10-09 (M4): the desk's prompt says who asks in words the caller does not choose — the +// controller, which the bus alone lets ask it — and what for, from names the controller checked; the prompt +// carries no free text of the caller's. +func TestThePromptIsAskedByNameNeverByWordsTheCallerChose(t *testing.T) { + d, _, _, asked := aDesk(t, sealedTo(t, typed)) + if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil { + t.Fatal(err) + } + p := (*asked)[0] + if p["module"] != "telegram" || p["secret"] != "telegram-token" || p["node"] != "anchor" { + t.Errorf("the prompt was not asked by name: %v", p) + } + for _, free := range []string{"prompt", "message"} { + if _, there := p[free]; there { + t.Errorf("the prompt carries the caller's %s: %v", free, p) + } + } +} + +// Every value given for a module's own secret is announced as a condition, on every channel (the review of +// 2026-10-09, M4): a bot token changed by somebody else is a channel that now answers for them. +func TestAValueGivenAtTheDeskIsAnnounced(t *testing.T) { + d, _, _, _ := aDesk(t, sealedTo(t, typed)) + var said []string + d.announce = func(node, module, name, how string) error { + said = append(said, node+" "+module+" "+name+" "+how) + return nil + } + if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil { + t.Fatal(err) + } + if len(said) != 1 || !strings.Contains(said[0], "anchor telegram telegram-token") || !strings.Contains(said[0], "laptop") { + t.Fatalf("announced %v", said) + } + o := secretGivenObservation("anchor", "telegram", "telegram-token", "at the desk on laptop", time.Date(2026, 10, 9, 12, 3, 0, 0, time.UTC)) + if o.Severity != conditions.Urgent || !strings.Contains(o.Explanation, "telegram-token") || + len(o.Actions) == 0 || o.Key() == "" { + t.Errorf("the announcement %+v", o) + } + if strings.Contains(o.Summary+o.Explanation+o.Said, typed) { + t.Error("the announcement carries the value") + } +} + +// The bus lets the controller alone ask the desk's prompt (the review of 2026-10-09, M4): the runtime, which +// carries every agent's calls, and a person granted every tool are denied it, however wide their grant. +func TestOnlyTheControllerMayAskTheDesksPrompt(t *testing.T) { + for _, p := range []broker.Principal{ + {Kind: broker.KindNodeTools, Node: "laptop"}, + {Kind: broker.KindPerson, Module: "operator", Invokes: []string{"*"}}, + {Kind: broker.KindModule, Node: "laptop", Module: "lab", Invokes: []string{"seat:node-launcher.secret"}}, + } { + perms, err := broker.PermissionsFor(p) + if err != nil { + t.Fatal(err) + } + for _, subject := range []string{"mesh.seat.node-launcher.tool.secret.laptop", "mesh.seat.node-launcher.tool.secret", + "mesh.mod.rofi.tool.node-launcher.secret", "mesh.mod.rofi.tool.node-launcher.secret.laptop"} { + if broker.MayPublish(perms, subject) { + t.Errorf("%s may publish %s", p.Username(), subject) + } + } + } + perms, _ := broker.PermissionsFor(broker.Principal{Kind: broker.KindController}) + if !broker.MayPublish(perms, "mesh.seat.node-launcher.tool.secret.laptop") { + t.Error("the controller may not ask the desk's prompt") + } +} + +// A value for a secret comes from the terminal or the desk, never through a verb (the review of 2026-10-09, +// M4): `secret accept` with a value, run for a verb, is refused before anything is read. +func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) { + t.Setenv(verbVar, "mesh-controller.command") + for _, args := range [][]string{ + {"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"}, + {"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"}, + } { + err := secretCommand(context.Background(), args) + if err == nil || !strings.Contains(err.Error(), "never through a verb") { + t.Errorf("%v: %v", args, err) + } + } +} diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index 2d489658..8acdbf3d 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -68,6 +68,12 @@ func secretCommand(ctx context.Context, args []string) error { return errors.New(secretUsage) } node, module, name := rest[0], rest[1], rest[2] + // A value comes from the terminal or the desk, never through a verb (the review of 2026-10-09, M4): a + // verb's caller may be an agent, and a value it chose would become what a module acts with. + if verb, through := throughAVerb(); through && *desk == "" { + return fmt.Errorf("a secret's value is given at the controller's terminal or at the desk (`give`), never "+ + "through a verb (this line came through %q): nothing was read or sealed", verb) + } if *desk != "" { if *from != "" || *provider != "" { return errors.New("--at-desk gives a module's own secret, and takes neither --from nor --provider") @@ -106,6 +112,9 @@ func secretCommand(ctx context.Context, args []string) error { if err != nil { return err } + if err := announceSecretGiven(ctx, node, module, name, "at the controller's terminal"); err != nil { + fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", err) + } // Not printed back, and there is nowhere it could be printed from: it is sealed to that // machine and the mesh cannot read it again. fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name) diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 415059eb..355c8b1d 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -187,6 +187,35 @@ var VerbsTheBusStepAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}} // desk opens a prompt that does not show what is typed, and answers it sealed to the controller's call. var VerbsTheControllerAsksForASecret = []SeatVerb{{Seat: "node-launcher", Verb: "secret"}} +// ControllerOnly are the subjects the controller alone may publish, however wide another's grant (the review +// of 2026-10-09, M4): the desk's hidden prompt, on its seat's subjects and on any holder's own module +// subjects. A grant of every tool — the runtime's, which carries every agent's calls, or a person's `*` — would +// otherwise reach it, and the prompt says the controller asks: only the bus makes that true. +func ControllerOnly() []string { + var out []string + for _, v := range VerbsTheControllerAsksForASecret { + for _, base := range []string{"mesh.seat." + v.Seat + ".tool." + v.Verb, "mesh.mod.*.tool." + v.Seat + "." + v.Verb} { + out = append(out, base, base+".*") + } + } + return out +} + +// MayPublish says whether permissions let a principal publish one subject: an allow covers it and no deny does. +func MayPublish(perms Permissions, subject string) bool { + for _, d := range perms.PublishDeny { + if SubjectsOverlap(d, subject) { + return false + } + } + for _, a := range perms.Publish { + if SubjectsOverlap(a, subject) { + return true + } + } + return false +} + // VerbsTheControllerAsksTheDeliveryOwner are the mesh-delivery seat's verbs the controller calls (novox/hq // ADR 0239): its self-check reads `stalled`, and healer H2 takes the one transition the table allows // through `close`. A mesh seat's verb is flat: no machine in the subject. @@ -257,8 +286,11 @@ func (p Principal) inbox() string { return "_INBOX." + p.Username() + ".>" } // Permissions is what a principal may publish and subscribe, and whether it may answer. type Permissions struct { - Publish []string - Subscribe []string + Publish []string + // PublishDeny are subjects refused although an allow covers them: the controller's alone (ControllerOnly), + // denied to everybody whose grant is wide enough to reach them. The server's deny outranks its allow. + PublishDeny []string + Subscribe []string // AllowResponses lets a principal reply to a request it received, on the reply subject that // request carried, once. // @@ -804,9 +836,22 @@ func PermissionsFor(p Principal) (Permissions, error) { if err := CheckWriters(p, pub); err != nil { return Permissions{}, err } + // What the controller alone may publish is denied to everybody else whose grant reaches it. + var deny []string + if p.Kind != KindController { + for _, only := range ControllerOnly() { + for _, a := range pub { + if SubjectsOverlap(a, only) { + deny = append(deny, only) + break + } + } + } + } return Permissions{ - Publish: pub, - Subscribe: sub, + Publish: pub, + PublishDeny: deny, + Subscribe: sub, // A module answers what it was asked — a tool call reaches it on its own namespace, so the // authority is bounded by having been asked — and so does the controller. A node and a // person are never asked anything, and are granted nothing here. @@ -1028,7 +1073,11 @@ func ComposeAccounts(principals []Principal) (string, error) { return "", fmt.Errorf("%s has no password hash: a user without one is a user anybody is", p.Username()) } fmt.Fprintf(&b, " { user: %q, password: %q, permissions: {\n", p.Username(), p.PasswordHash) - fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish)) + if len(perms.PublishDeny) > 0 { + fmt.Fprintf(&b, " publish: { allow: [%s], deny: [%s] }\n", quoted(perms.Publish), quoted(perms.PublishDeny)) + } else { + fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish)) + } fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe)) if perms.AllowResponses { fmt.Fprintf(&b, " allow_responses: { max: 1, ttl: \"%dm\" }\n", int(ResponseTTL/time.Minute)) diff --git a/internal/catalogue/graphical_session.go b/internal/catalogue/graphical_session.go index 09e738ae..b9d767e7 100644 --- a/internal/catalogue/graphical_session.go +++ b/internal/catalogue/graphical_session.go @@ -84,12 +84,16 @@ func graphicalSessionSeats() []Seat { {Name: "secret", Optional: true, Description: "Ask the operator for a value in a prompt that " + "does not show what is typed, and answer it sealed to the key the asker gives — never in " + "the clear — or cancelled when the prompt was dismissed or not answered in time.", + // By name, never by words (the review of 2026-10-09, M4): the holder writes the prompt from the + // module, the secret and the machine, and says the controller asks — the bus lets nobody else + // ask it (broker.ControllerOnly) — so no caller puts words of its own before the operator. Input: schema(map[string]string{ - "prompt": "what the prompt asks", - "message": "a line saying who asks and for what (optional)", + "module": "the module whose own secret is asked for", + "secret": "the own secret's name", + "node": "the machine the module runs on", "seal_to": "the asker's public sealing key: the answer is sealed to it", "timeout_seconds": "give up after this long (optional)", - }, []string{"prompt", "seal_to"})}, + }, []string{"module", "secret", "node", "seal_to"})}, }}, {Name: NotifierSeat, Scope: ScopeNode, Decision: decided, Serves: []Verb{ {Name: "send", Description: "Show the operator a notification.", diff --git a/internal/inventory/givable_test.go b/internal/inventory/givable_test.go new file mode 100644 index 00000000..308ed1cf --- /dev/null +++ b/internal/inventory/givable_test.go @@ -0,0 +1,28 @@ +package inventory + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// `give` takes only a value nobody but a person has (the review of 2026-10-09, M4): never the module's bus +// account, which `issue` mints, nor a secret the mesh may make itself. +func TestOnlyASecretThePersonHoldsIsGivenAtTheDesk(t *testing.T) { + m := catalogue.Manifest{Module: "telegram", OwnSecrets: catalogue.OwnSecrets{ + "telegram-token": {Path: "/s/telegram-token"}, + "broker": {Path: "/s/broker"}, + "session": {Path: "/s/session", Taken: catalogue.TakenAtStart}, + }} + if err := GivableAtDesk(m, "telegram-token"); err != nil { + t.Errorf("the bot token was refused: %v", err) + } + for _, name := range []string{"broker", "session", "chat-id"} { + if err := GivableAtDesk(m, name); err == nil { + t.Errorf("%s was givable", name) + } else if name != "chat-id" && !strings.Contains(err.Error(), "the mesh makes") { + t.Errorf("%s: %v", name, err) + } + } +} diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 6e001eeb..843ae7db 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -561,8 +561,28 @@ func (i *Inventory) DeclaresOwnSecret(ctx context.Context, module, name string) if err != nil { return err } - if _, ok := m.OwnSecrets[name]; !ok { - return fmt.Errorf("%s does not declare %q as an own secret; %s", module, name, declaresOwn(m)) + return GivableAtDesk(m, name) +} + +// BrokerSecret is the own secret that is a module's bus account, which `issue` mints. +const BrokerSecret = "broker" + +// GivableAtDesk refuses, in words, an own secret a module does not declare, and one the mesh makes itself +// (the review of 2026-10-09, M4): the module's bus account, which `issue` mints, and any the mesh may make +// in place of a value given (catalogue.OwnSecret.MeshMayMake). The desk takes only what a person holds and +// the mesh cannot make — a bot's token — so nobody is asked to type the mesh's own credential into a prompt. +func GivableAtDesk(m catalogue.Manifest, name string) error { + own, ok := m.OwnSecrets[name] + if !ok { + return fmt.Errorf("%s does not declare %q as an own secret; %s", m.Module, name, declaresOwn(m)) + } + switch { + case name == BrokerSecret: + return fmt.Errorf("%q is %s's account on the bus, which the mesh makes (`issue`), never a value a person gives", + name, m.Module) + case own.MeshMayMake(): + return fmt.Errorf("%q of %s is a secret the mesh makes itself (it may replace a value given at the module's "+ + "start, ADR 0228); a person gives it only at the controller's terminal, with `secret accept`", name, m.Module) } return nil } From ed331eb9728978c275e7881df32d3a9616507964 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:32:02 +0200 Subject: [PATCH 13/15] =?UTF-8?q?Let=20the=20give=20verb's=20exact=20line?= =?UTF-8?q?=20past=20the=20terminal=20rule=20for=20secrets,=20and=20nothin?= =?UTF-8?q?g=20else=20(hq=20ADR=200259=20=C2=A710,=20ADR=200266)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restacked on #157, which carries #164's rule that no verb runs secret accept. give's line carries no value: the operator types it into the desk's hidden prompt, sealed to the call and then to the module's machine. Only that exact line passes: a value, a file, a provider or any extra word stays the terminal's. --- cmd/mesh-controller/desk_secret_test.go | 19 +++++++++++++++++++ cmd/mesh-controller/seatverbs.go | 20 ++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/cmd/mesh-controller/desk_secret_test.go b/cmd/mesh-controller/desk_secret_test.go index eef0bc04..c63364fc 100644 --- a/cmd/mesh-controller/desk_secret_test.go +++ b/cmd/mesh-controller/desk_secret_test.go @@ -226,3 +226,22 @@ func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) { } } } + +// The `give` verb's own line passes the terminal-only rule of ADR 0266, and no other `secret accept` does: a +// value, a file, a provider or an extra word is still the terminal's alone. +func TestOnlyTheGiveLinePassesTheTerminalRuleForSecrets(t *testing.T) { + if err := terminalOnly([]string{"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"}); err != nil { + t.Errorf("give's line refused: %v", err) + } + for _, argv := range [][]string{ + {"secret", "accept", "anchor", "telegram", "telegram-token"}, + {"secret", "accept", "anchor", "telegram", "telegram-token", "--from", "/tmp/x"}, + {"secret", "accept", "anchor", "telegram", "telegram-token", "--at-desk", "laptop", "--local"}, + {"secret", "accept", "anchor", "telegram", "--provider", "--at-desk", "laptop"}, + {"secret", "export", "anchor", "telegram", "telegram-token", "--at-desk", "laptop"}, + } { + if err := terminalOnly(argv); err == nil { + t.Errorf("%v passed the terminal rule", argv) + } + } +} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 308fcbbd..48b17938 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -1500,6 +1500,20 @@ var terminalOnlyCommands = map[string]string{ "licence": "the licences' secrets", } +// givenAtTheDesk is exactly the line the `give` verb composes, and nothing beside it: `secret accept +// --at-desk `, with no other word — no value, no file, no provider. +func givenAtTheDesk(argv []string) bool { + if len(argv) != 7 || argv[0] != "secret" || argv[1] != "accept" || argv[5] != "--at-desk" { + return false + } + for _, w := range argv[2:5] { + if w == "" || strings.HasPrefix(w, "-") { + return false + } + } + return argv[6] != "" && !strings.HasPrefix(argv[6], "-") +} + // terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal // alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and // `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself @@ -1513,8 +1527,10 @@ func terminalOnly(argv []string) error { return terminalRefusal("%s is run at the controller's terminal only, never through a verb: it holds %s, and "+ "whoever may call a verb includes agents (novox/hq ADR 0266). Nothing was done", argv[0], what) } - // Of a secret's commands only rotation, which seals the new value to the machine that uses it. - if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") { + // Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the + // `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this + // call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10). + if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) { return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+ "recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+ "0266). Nothing was done", strings.Join(argv[1:], " ")) From 3e5086a2f67379e3ef8d6db0e7928e7487e46070 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:58:19 +0200 Subject: [PATCH 14/15] =?UTF-8?q?give:=20never=20take=20a=20trusted=20part?= =?UTF-8?q?y's=20secret=20at=20a=20desk,=20and=20announce=20it=20before=20?= =?UTF-8?q?it=20is=20kept=20(hq=20ADR=200259=20=C2=A710,=20the=20confirmat?= =?UTF-8?q?ion=20review's=20N1-give)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A desk's prompt is answered over the desk machine's bus, on a subject that machine's runtime answers; on a desk machine agents run as the operator, who holds that credential, so an agent could answer first with a bot token of its own sealed to the call's key. The secret of a module running as an account of its own (the Telegram bot's token) is now refused at the desk, before anybody is asked, naming the line to type at the controller's terminal; there it is announced on every channel, the old one among them, before it is kept, and not kept when that announcement fails. What is typed at the terminal is not echoed. Tests: the refusal with nobody asked, and only the desk machine's launcher may answer its prompt (MaySubscribe). --- cmd/mesh-controller/desk_secret.go | 22 ++++++++++++ cmd/mesh-controller/desk_secret_test.go | 48 +++++++++++++++++++++++++ cmd/mesh-controller/hidden_input.go | 26 ++++++++++++++ cmd/mesh-controller/secret.go | 21 +++++++++-- internal/broker/nats.go | 10 ++++++ internal/inventory/secrets.go | 11 ++++++ 6 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 cmd/mesh-controller/hidden_input.go diff --git a/cmd/mesh-controller/desk_secret.go b/cmd/mesh-controller/desk_secret.go index 963f3ac5..5d91a9e1 100644 --- a/cmd/mesh-controller/desk_secret.go +++ b/cmd/mesh-controller/desk_secret.go @@ -39,6 +39,9 @@ const deskPromptWithin = 25 type deskGive struct { // declares refuses a module or a secret the mesh would refuse, before anybody is asked to type. declares func(module, name string) error + // trusted says a module runs as an account of its own: its secret is never taken at a desk (below). Nil is + // never (a test that does not look). + trusted func(module string) (bool, error) // ask asks one machine's node-launcher.secret and answers its result, or the holder's refusal. ask func(machine string, args map[string]any) (json.RawMessage, error) // accept seals the value as `secret accept` does, and says whether it lives until the module's start. @@ -63,6 +66,24 @@ func (d deskGive) give(node, module, name, desk string) (string, error) { if err := d.declares(module, name); err != nil { return "", fmt.Errorf("nobody was asked to type anything: %w", err) } + // **A trusted party's secret is not taken at a desk** (the confirmation review of 2026-10-09, N1-give). The + // prompt's answer comes back over the desk machine's bus, on a subject that machine's runtime answers — and + // on a desk machine agents run as the operator, who holds that runtime's credential. An agent could answer + // first, with a bot token of its own sealed to this call's key, and the channel the operator's answers are + // proven on would be the agent's. So the value of a module running as its own account is typed at the + // controller's terminal, where no bus carries it. + if d.trusted != nil { + trusted, err := d.trusted(module) + if err != nil { + return "", fmt.Errorf("nobody was asked to type anything: whether %s is a trusted party could not be read: %w", module, err) + } + if trusted { + return "", fmt.Errorf("nobody was asked to type anything: %s runs as an account of its own and proves the "+ + "operator's answers, so its %s is given at the controller's terminal alone — there, run `mesh-controller "+ + "secret accept %s %s %s` and type it at its prompt. A desk's prompt is answered over the desk machine's "+ + "bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name) + } + } public, private, err := secrets.Keypair() if err != nil { return "", fmt.Errorf("no key could be made to take the value: %w", err) @@ -143,6 +164,7 @@ func giveAtDesk(ctx context.Context, node, module, name, desk string) error { defer open.Close() d := deskGive{ declares: func(module, name string) error { return open.inventory.DeclaresOwnSecret(ctx, module, name) }, + trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) }, ask: func(machine string, args map[string]any) (json.RawMessage, error) { var result json.RawMessage err := onTheBus(func(conn *nats.Conn) error { diff --git a/cmd/mesh-controller/desk_secret_test.go b/cmd/mesh-controller/desk_secret_test.go index c63364fc..b3ad1873 100644 --- a/cmd/mesh-controller/desk_secret_test.go +++ b/cmd/mesh-controller/desk_secret_test.go @@ -245,3 +245,51 @@ func TestOnlyTheGiveLinePassesTheTerminalRuleForSecrets(t *testing.T) { } } } + +// The confirmation review of 2026-10-09, N1-give: a desk's prompt is answered over the desk machine's bus, and +// on a desk machine agents run as the operator, who holds its runtime's credential — so a trusted party's +// secret (a module running as an account of its own: the Telegram bot's token) is never taken at a desk. +// Refused before anybody is asked to type, whoever called, naming the terminal's line. +func TestATrustedPartysSecretIsNeverTakenAtADesk(t *testing.T) { + d, accepted, acts, asked := aDesk(t, sealedTo(t, typed)) + d.trusted = func(module string) (bool, error) { return module == "telegram", nil } + _, err := d.give("anchor", "telegram", "telegram-token", "laptop") + if err == nil || !strings.Contains(err.Error(), "controller's terminal alone") || + !strings.Contains(err.Error(), "secret accept anchor telegram telegram-token") { + t.Fatalf("a trusted party's secret was taken at the desk, or refused without the line: %v", err) + } + if len(*asked)+len(*accepted)+len(*acts) != 0 { + t.Errorf("asked %v, accepted %d, recorded %v", *asked, len(*accepted), *acts) + } + d.trusted = func(string) (bool, error) { return false, errors.New("the store did not answer") } + if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err == nil || len(*asked) != 0 { + t.Errorf("a module not known to be untrusted was asked at the desk: %v", err) + } +} + +// And who may answer the desk's prompt at all: only the runtime of the machine it is asked on, carrying the +// launcher that holds the seat there — never the controller, another machine's runtime, or a module's own +// account (the confirmation review of 2026-10-09, N1-give). +func TestOnlyTheDeskMachinesLauncherMayAnswerItsPrompt(t *testing.T) { + launcher := broker.Declared{Module: "rofi", Holds: []broker.Seat{{Name: "node-launcher", Scope: "node", + Serves: []string{"run", "secret"}}}} + subject := "mesh.seat.node-launcher.tool.secret.laptop" + for _, c := range []struct { + p broker.Principal + answers bool + }{ + {broker.Principal{Kind: broker.KindNodeTools, Node: "laptop", Module: broker.RuntimeModule, Carries: []broker.Declared{launcher}}, true}, + {broker.Principal{Kind: broker.KindNodeTools, Node: "anchor", Module: broker.RuntimeModule, Carries: []broker.Declared{launcher}}, false}, + {broker.Principal{Kind: broker.KindController}, false}, + {broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "lab"}, false}, + {broker.Principal{Kind: broker.KindNode, Node: "laptop"}, false}, + } { + perms, err := broker.PermissionsFor(c.p) + if err != nil { + t.Fatal(err) + } + if got := broker.MaySubscribe(perms, subject); got != c.answers { + t.Errorf("%s may answer %s: %v, want %v", c.p.Username(), subject, got, c.answers) + } + } +} diff --git a/cmd/mesh-controller/hidden_input.go b/cmd/mesh-controller/hidden_input.go new file mode 100644 index 00000000..460b8a34 --- /dev/null +++ b/cmd/mesh-controller/hidden_input.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// hideTyping turns a terminal's echo off while a secret is typed at it, and gives back what restores it. On +// anything that is not a terminal (a pipe, a file) it does nothing. +func hideTyping(f *os.File) func() { + fd := int(f.Fd()) + before, err := unix.IoctlGetTermios(fd, unix.TCGETS) + if err != nil { + return func() {} + } + hidden := *before + hidden.Lflag &^= unix.ECHO + if err := unix.IoctlSetTermios(fd, unix.TCSETS, &hidden); err != nil { + return func() {} + } + return func() { + _ = unix.IoctlSetTermios(fd, unix.TCSETS, before) + _, _ = os.Stderr.WriteString("\n") + } +} diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index 8acdbf3d..52430a4d 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -108,12 +108,27 @@ func secretCommand(ctx context.Context, args []string) error { fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node) return nil } + // A trusted party's secret is announced before it is kept (the confirmation review of 2026-10-09, N1-give): + // on every channel, the one it replaces among them, which still runs on its old value until the next push. + // Not announced, it is not kept: a channel whose token changed unheard of answers for somebody else. + trusted, err := open.inventory.RunsAsItsOwnAccount(ctx, module) + if err != nil { + return err + } + if trusted { + if err := announceSecretGiven(ctx, node, module, name, "at the controller's terminal"); err != nil { + return fmt.Errorf("%s runs as an account of its own, and the change of its %s could not be announced on "+ + "your channels first, so nothing was kept: %w", module, name, err) + } + } untilStart, err := open.inventory.AcceptGivenSecret(ctx, node, module, name, value) if err != nil { return err } - if err := announceSecretGiven(ctx, node, module, name, "at the controller's terminal"); err != nil { - fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", err) + if !trusted { + if err := announceSecretGiven(ctx, node, module, name, "at the controller's terminal"); err != nil { + fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", err) + } } // Not printed back, and there is nowhere it could be printed from: it is sealed to that // machine and the mesh cannot read it again. @@ -387,6 +402,8 @@ func valueFor(node, module, name, from string) (string, error) { fmt.Fprintf(os.Stderr, "reading %s's %q for %s from standard input; it is not echoed anywhere\n", module, name, node) + // At a terminal, what is typed is not shown either: echo off while it is read. + defer hideTyping(os.Stdin)() line, err := bufio.NewReader(os.Stdin).ReadString('\n') if err != nil && line == "" { return "", fmt.Errorf("nothing was given on standard input: %w", err) diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 355c8b1d..cfeb34f4 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -216,6 +216,16 @@ func MayPublish(perms Permissions, subject string) bool { return false } +// MaySubscribe says whether a principal with these permissions may subscribe to (and so answer) a subject. +func MaySubscribe(perms Permissions, subject string) bool { + for _, a := range perms.Subscribe { + if SubjectsOverlap(a, subject) { + return true + } + } + return false +} + // VerbsTheControllerAsksTheDeliveryOwner are the mesh-delivery seat's verbs the controller calls (novox/hq // ADR 0239): its self-check reads `stalled`, and healer H2 takes the one transition the table allows // through `close`. A mesh seat's verb is flat: no machine in the subject. diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 843ae7db..32534e70 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -564,6 +564,17 @@ func (i *Inventory) DeclaresOwnSecret(ctx context.Context, module, name string) return GivableAtDesk(m, name) } +// RunsAsItsOwnAccount says a module runs as an account of its own (novox/hq ADR 0259 §8): a trusted party — the +// router, a channel that proves its sender or shows a link's code — whose own secret is what the operator's +// answers are believed by. Its value is given at the controller's terminal alone. +func (i *Inventory) RunsAsItsOwnAccount(ctx context.Context, module string) (bool, error) { + m, err := i.declared(ctx, module) + if err != nil { + return false, err + } + return m.RunsAs != "", nil +} + // BrokerSecret is the own secret that is a module's bus account, which `issue` mints. const BrokerSecret = "broker" From fa19a2d71871621209f2e4aaec079df7f76e13eb Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 15:36:24 +0200 Subject: [PATCH 15/15] give: test that a trusted party's secret given at the terminal is announced before it is kept, and refuse an unknown machine before anybody types The terminal path's order is one function, keepGiven, so the test fails when the announcement before a trusted party's secret is removed, and when a failed announcement still keeps it. give also refuses a machine the mesh does not know, as the secret's or as the desk, before the prompt (the final review). --- cmd/mesh-controller/desk_secret.go | 17 ++++++- cmd/mesh-controller/desk_secret_test.go | 66 +++++++++++++++++++++++++ cmd/mesh-controller/secret.go | 36 ++++++++++---- 3 files changed, 109 insertions(+), 10 deletions(-) diff --git a/cmd/mesh-controller/desk_secret.go b/cmd/mesh-controller/desk_secret.go index 5d91a9e1..1872cd1f 100644 --- a/cmd/mesh-controller/desk_secret.go +++ b/cmd/mesh-controller/desk_secret.go @@ -39,6 +39,9 @@ const deskPromptWithin = 25 type deskGive struct { // declares refuses a module or a secret the mesh would refuse, before anybody is asked to type. declares func(module, name string) error + // known refuses a machine the mesh does not know, before anybody is asked to type; nil knows every one + // (a test that does not look). + known func(machine string) error // trusted says a module runs as an account of its own: its secret is never taken at a desk (below). Nil is // never (a test that does not look). trusted func(module string) (bool, error) @@ -63,6 +66,14 @@ func (d deskGive) give(node, module, name, desk string) (string, error) { return "", fmt.Errorf("%s is not named", what) } } + if d.known != nil { + for what, machine := range map[string]string{"the machine the secret is for": node, "the desk": desk} { + if err := d.known(machine); err != nil { + return "", fmt.Errorf("nobody was asked to type anything: %s, %s, is not a machine this mesh knows: %w", + what, machine, err) + } + } + } if err := d.declares(module, name); err != nil { return "", fmt.Errorf("nobody was asked to type anything: %w", err) } @@ -164,7 +175,11 @@ func giveAtDesk(ctx context.Context, node, module, name, desk string) error { defer open.Close() d := deskGive{ declares: func(module, name string) error { return open.inventory.DeclaresOwnSecret(ctx, module, name) }, - trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) }, + known: func(machine string) error { + _, err := open.inventory.NodeByName(ctx, machine) + return err + }, + trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) }, ask: func(machine string, args map[string]any) (json.RawMessage, error) { var result json.RawMessage err := onTheBus(func(conn *nats.Conn) error { diff --git a/cmd/mesh-controller/desk_secret_test.go b/cmd/mesh-controller/desk_secret_test.go index b3ad1873..7c87b3de 100644 --- a/cmd/mesh-controller/desk_secret_test.go +++ b/cmd/mesh-controller/desk_secret_test.go @@ -293,3 +293,69 @@ func TestOnlyTheDeskMachinesLauncherMayAnswerItsPrompt(t *testing.T) { } } } + +// N1-give at the controller's terminal (the confirmation review of 2026-10-09): a trusted party's secret is +// announced before it is kept, and not kept when the announcement fails; another module's is kept first and +// a failed announcement is said, not undone. +func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) { + var order []string + announce := func(fail bool) func() error { + return func() error { + order = append(order, "announce") + if fail { + return errors.New("no channel") + } + return nil + } + } + keep := func() (bool, error) { order = append(order, "keep"); return false, nil } + + order = nil + if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil || + strings.Join(order, ",") != "announce,keep" { + t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order) + } + order = nil + if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" { + t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order) + } + order = nil + if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil || + strings.Join(order, ",") != "keep,announce" { + t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order) + } + order = nil + failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") } + if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" { + t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order) + } +} + +// A machine the mesh does not know, as the secret's or as the desk, is refused before anybody is asked to type. +func TestAGiveNamingAMachineTheMeshDoesNotKnowAsksNobody(t *testing.T) { + for _, unknown := range []string{"elsewhere", "nodesk"} { + d, accepted, acts, asked := aDesk(t, func(map[string]any) (json.RawMessage, error) { + t.Fatal("the desk was asked") + return nil, nil + }) + d.known = func(machine string) error { + if machine == unknown { + return errors.New("no node " + machine) + } + return nil + } + node, desk := "anchor", "laptop" + if unknown == "elsewhere" { + node = unknown + } else { + desk = unknown + } + _, err := d.give(node, "telegram", "telegram-token", desk) + if err == nil || !strings.Contains(err.Error(), "nobody was asked") || !strings.Contains(err.Error(), unknown) { + t.Errorf("%s: %v", unknown, err) + } + if len(*accepted)+len(*acts)+len(*asked) != 0 { + t.Errorf("%s: something happened: %v %v %v", unknown, *accepted, *acts, *asked) + } + } +} diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index 52430a4d..bcb3dd46 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -115,20 +115,18 @@ func secretCommand(ctx context.Context, args []string) error { if err != nil { return err } - if trusted { - if err := announceSecretGiven(ctx, node, module, name, "at the controller's terminal"); err != nil { + untilStart, unannounced, err := keepGiven(trusted, + func() error { return announceSecretGiven(ctx, node, module, name, "at the controller's terminal") }, + func() (bool, error) { return open.inventory.AcceptGivenSecret(ctx, node, module, name, value) }) + if err != nil { + if trusted && unannounced != nil { return fmt.Errorf("%s runs as an account of its own, and the change of its %s could not be announced on "+ "your channels first, so nothing was kept: %w", module, name, err) } - } - untilStart, err := open.inventory.AcceptGivenSecret(ctx, node, module, name, value) - if err != nil { return err } - if !trusted { - if err := announceSecretGiven(ctx, node, module, name, "at the controller's terminal"); err != nil { - fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", err) - } + if unannounced != nil { + fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", unannounced) } // Not printed back, and there is nowhere it could be printed from: it is sealed to that // machine and the mesh cannot read it again. @@ -487,3 +485,23 @@ func whoAsked() string { } return "the mesh" } + +// keepGiven keeps a value given at the controller's terminal, and announces it on the operator's channels +// (the confirmation review of 2026-10-09, N1-give). **A trusted party's — a module running as an account of its +// own: the router, a verified channel — is announced before it is kept, and not kept when the announcement +// fails**: a channel whose token changed unheard of answers for somebody else. Any other module's is kept first +// and announced after, and a failed announcement is said (unannounced) without undoing it. +func keepGiven(trusted bool, announce func() error, keep func() (bool, error)) (untilStart bool, unannounced, err error) { + if trusted { + if err := announce(); err != nil { + return false, err, err + } + untilStart, err = keep() + return untilStart, nil, err + } + untilStart, err = keep() + if err != nil { + return false, nil, err + } + return untilStart, announce(), nil +}