Every one of the 48 core failures of research 031 was found by a person looking; the mesh's answers carried the fact for whoever asked and told nobody. - The condition store (to-be 45 §2): mesh-controller_conditions, one key per open condition, written by compare-and-set so a person's silence and the watchdogs never lose each other's word; every transition kept ninety days in mesh-controller_condition-history and said as the seat's events condition-raised / condition-changed / condition-cleared (the condition at the top level, with event, at, change, why, show), offered again while the bus is away. Raised and cleared by observation only; a clearing reopened within ten minutes is the same condition with its count up, its silence kept. Verbs: conditions, conditions show, conditions silence (a hand act, at most a week), conditions history. - ADR 0224's provider standing is the first kind, provider-failing, held by the provider's events; the provider_standing table is no longer read or written (left in place: dropping it is the operator's word). - status leads with the open conditions, urgent first, and says all well only with none open; conditions it cannot read are said and not well. - The signals table compiled in, one watchdog loop over it every 30s: S1 heartbeat (3 intervals, asleep machines excepted, control node urgent after 30 min), S2 report after a send, S3 plan tier, S4 event loop deaf, S5 merge not acted, S6 ask lost, S7 call hung, S8 provider silent, S9 advisories, S10 self-check silent, S11 node tools silent, S13 stale refusals; S12, S14, S15 deferred with their reasons. A row that cannot see raises probe-failed and clears nothing. A test generated from the table suppresses each signal inside and past its bound. - The bus's advisories (maximum deliveries, a mesh consumer deleted) and the controller's own slow consumer and refused subjects, said in the mesh's words. - doctor: the probe registry D1-D10 (D5 deferred) and DW, every five minutes, each in thirty seconds; a probe that cannot run is never a pass. D1 validates with mesh-host's own validator. Every run ends with the doctor-heartbeat event mesh-watcher listens for. - The controller is granted its new buckets, events, the two advisories and $SRV.INFO; the node tools their tools-alive heartbeat. The streams and consumers the controller asserts and the ones D6/D7 expect are one derivation.
900 lines
31 KiB
Go
900 lines
31 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/nats-io/nats.go/micro"
|
|
"os"
|
|
"os/exec"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The mesh's own verbs, served as the mesh-controller seat's tools (novox/hq ADR 0154, design 33).
|
|
//
|
|
// **Each tool runs the command it names, in this same binary, and answers what it printed.** That is
|
|
// ADR 0035 taken literally: the logic lives once, in the command, and a surface is an adapter with no
|
|
// decisions in it. Running a fresh process rather than calling the function keeps two things true
|
|
// that calling it would not — every command opens and closes its own stores the way it does from a
|
|
// shell, and nothing a command prints to the process's standard output can leak into another call's
|
|
// answer. It also means a refusal is the same refusal in the same words, because it is the same
|
|
// output.
|
|
|
|
// verbAnswer is what a verb answers: what the command printed, whether it succeeded, and — where the
|
|
// command speaks JSON — the same as data.
|
|
type verbAnswer struct {
|
|
Output string `json:"output"`
|
|
OK bool `json:"ok"`
|
|
Answer any `json:"answer,omitempty"`
|
|
}
|
|
|
|
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
|
|
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
|
|
//
|
|
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
|
|
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
|
|
// the verb declares but did not use for the command line it composed — given beside another that
|
|
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
|
|
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
|
|
// not take that" would have stopped it before anything was sent.
|
|
func argvFor(verb string, args map[string]any) ([]string, error) {
|
|
a, err := readArguments(verb, args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
argv, err := a.commandLine()
|
|
if len(a.misread) > 0 {
|
|
// The table and the command line disagree: the verb reads an argument no caller can see
|
|
// in its schema, so no caller could ever pass it.
|
|
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
|
|
"table and its command lines disagree", verb, quoteAll(a.misread))
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if unused := a.unused(); len(unused) > 0 {
|
|
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
|
|
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
|
|
}
|
|
return argv, nil
|
|
}
|
|
|
|
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
|
|
// command line was composed from.
|
|
type verbArguments struct {
|
|
verb string
|
|
given map[string]string
|
|
used map[string]bool
|
|
declared map[string]bool
|
|
misread []string // arguments the command line read that the schema does not declare: a bug here
|
|
}
|
|
|
|
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
|
|
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
|
|
// wrote.
|
|
func controllerVerb(name string) (catalogue.Verb, bool) {
|
|
for _, v := range catalogue.ControllerVerbs {
|
|
if v.Name == name {
|
|
return v, true
|
|
}
|
|
}
|
|
return catalogue.Verb{}, false
|
|
}
|
|
|
|
// declaredArguments are a schema's properties, and which of them are switches.
|
|
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
|
|
switches = map[string]bool{}
|
|
props, _ := v.Input["properties"].(map[string]any)
|
|
for name, p := range props {
|
|
names = append(names, name)
|
|
desc, _ := p.(map[string]any)
|
|
switch enum := desc["enum"].(type) {
|
|
case []string:
|
|
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
|
case []any:
|
|
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
|
}
|
|
}
|
|
sort.Strings(names)
|
|
return names, switches
|
|
}
|
|
|
|
// readArguments refuses what the verb does not take, before anything is composed.
|
|
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
|
|
v, known := controllerVerb(verb)
|
|
if !known {
|
|
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
|
|
}
|
|
names, switches := declaredArguments(v)
|
|
declared := map[string]bool{}
|
|
for _, n := range names {
|
|
declared[n] = true
|
|
}
|
|
takes := "none"
|
|
if len(names) > 0 {
|
|
takes = quoteAll(names)
|
|
}
|
|
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
|
|
keys := make([]string, 0, len(args))
|
|
for k := range args {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
if !declared[k] {
|
|
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
|
|
}
|
|
var value string
|
|
switch x := args[k].(type) {
|
|
case nil:
|
|
continue
|
|
case string:
|
|
value = strings.TrimSpace(x)
|
|
case bool:
|
|
if !switches[k] {
|
|
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
|
|
}
|
|
value = fmt.Sprint(x)
|
|
default:
|
|
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
|
|
}
|
|
if switches[k] {
|
|
switch value {
|
|
case "true":
|
|
case "false", "":
|
|
continue // said and off: the same as not given, and nothing passed over
|
|
default:
|
|
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
|
|
}
|
|
}
|
|
if value != "" {
|
|
a.given[k] = value
|
|
}
|
|
}
|
|
return a, nil
|
|
}
|
|
|
|
// str is one argument's value, marked as used.
|
|
func (a *verbArguments) str(key string) string {
|
|
if !a.declared[key] {
|
|
a.misread = append(a.misread, key)
|
|
}
|
|
a.used[key] = true
|
|
return a.given[key]
|
|
}
|
|
|
|
// on is a switch, marked as used.
|
|
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
|
|
|
|
// need refuses a call missing a required argument, in the verb's own words.
|
|
func (a *verbArguments) need(keys ...string) error {
|
|
for _, k := range keys {
|
|
if a.str(k) == "" {
|
|
return fmt.Errorf("%s needs %q", a.verb, k)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// unused are the arguments given that the command line was not composed from.
|
|
func (a *verbArguments) unused() []string {
|
|
var out []string
|
|
for k := range a.given {
|
|
if !a.used[k] {
|
|
out = append(out, k)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
func (a *verbArguments) usedGiven() []string {
|
|
var out []string
|
|
for k := range a.given {
|
|
if a.used[k] {
|
|
out = append(out, k)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
if len(out) == 0 {
|
|
return []string{"nothing"}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func quoteAll(xs []string) string {
|
|
q := make([]string, len(xs))
|
|
for i, x := range xs {
|
|
if x == "nothing" {
|
|
q[i] = x
|
|
continue
|
|
}
|
|
q[i] = fmt.Sprintf("%q", x)
|
|
}
|
|
return strings.Join(q, ", ")
|
|
}
|
|
|
|
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
|
|
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
|
|
func (a *verbArguments) commandLine() ([]string, error) {
|
|
verb, str, on, need := a.verb, a.str, a.on, a.need
|
|
switch verb {
|
|
case "command":
|
|
// The generic verb: the command line as given, split as a shell would split it, with
|
|
// nothing added — the named verbs add flags a caller cannot reach; this one is the whole
|
|
// binary and says so in its description (novox/hq ADR 0154, 0175).
|
|
if err := need("command"); err != nil {
|
|
return nil, err
|
|
}
|
|
argv, err := splitCommandLine(str("command"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(argv) == 0 {
|
|
return nil, errors.New("command names no command")
|
|
}
|
|
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
|
// it says why, as it would through its own verb.
|
|
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
|
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
|
"line (recorded in the hand-act log). Nothing was done", repair)
|
|
}
|
|
return argv, nil
|
|
case "tools":
|
|
return nil, errors.New("tools is answered from the records, not by a command")
|
|
case "status":
|
|
return []string{"status", "--json"}, nil
|
|
case "nodes":
|
|
return []string{"node", "list", "--json"}, nil
|
|
case "node":
|
|
if err := need("node"); err != nil {
|
|
return nil, err
|
|
}
|
|
return []string{"node", "show", str("node")}, nil
|
|
case "modules":
|
|
return []string{"module", "list", "--json"}, nil
|
|
case "seats":
|
|
return []string{"seats", "--json"}, nil
|
|
case "builds":
|
|
if id := str("log"); id != "" {
|
|
return []string{"builds", "--log", id}, nil
|
|
}
|
|
argv := []string{"builds"}
|
|
if n := str("limit"); n != "" {
|
|
argv = append(argv, "-n", n)
|
|
}
|
|
if m := str("module"); m != "" {
|
|
argv = append(argv, m)
|
|
}
|
|
return argv, nil
|
|
case "plans":
|
|
if r := str("repository"); r != "" {
|
|
argv := []string{"plans", "--what-if", r}
|
|
if p := str("paths"); p != "" {
|
|
argv = append(argv, "--paths", p)
|
|
}
|
|
if m := str("modules"); m != "" {
|
|
argv = append(argv, "--modules", m)
|
|
}
|
|
return argv, nil
|
|
}
|
|
for _, act := range []string{"stop", "close", "retry"} {
|
|
if id := str(act); id != "" {
|
|
argv := []string{"plans", act, id}
|
|
if act == "retry" {
|
|
return argv, nil
|
|
}
|
|
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
|
|
if w := str("why"); w != "" {
|
|
argv = append(argv, "--why", w)
|
|
}
|
|
if c := str("cause"); c != "" {
|
|
argv = append(argv, "--cause", c)
|
|
}
|
|
return argv, nil
|
|
}
|
|
}
|
|
if id := str("id"); id != "" {
|
|
return []string{"plans", id}, nil
|
|
}
|
|
argv := []string{"plans"}
|
|
if n := str("limit"); n != "" {
|
|
argv = append(argv, "-n", n)
|
|
}
|
|
return argv, nil
|
|
// The build queue (novox/hq ADR 0219).
|
|
case "queue":
|
|
return []string{"queue"}, nil
|
|
case "cancel", "kill":
|
|
if err := need("id"); err != nil {
|
|
return nil, err
|
|
}
|
|
return []string{verb, str("id")}, nil
|
|
case "clear":
|
|
if on("dead") {
|
|
return []string{"clear", "--dead"}, nil
|
|
}
|
|
return []string{"clear"}, nil
|
|
case "rebuild":
|
|
if err := need("what"); err != nil {
|
|
return nil, err
|
|
}
|
|
return []string{"rebuild", str("what")}, nil
|
|
case "replay":
|
|
if err := need("id"); err != nil {
|
|
return nil, err
|
|
}
|
|
argv := []string{"replay", str("id")}
|
|
if on("register") {
|
|
argv = append(argv, "--register")
|
|
}
|
|
if on("older") {
|
|
argv = append(argv, "--older")
|
|
}
|
|
return argv, nil
|
|
case "pause", "resume":
|
|
if n := str("node"); n != "" {
|
|
return []string{verb, n}, nil
|
|
}
|
|
return []string{verb}, nil
|
|
case "plan":
|
|
if err := need("node"); err != nil {
|
|
return nil, err
|
|
}
|
|
if on("files") {
|
|
return []string{"plan", str("node"), "--files"}, nil
|
|
}
|
|
return []string{"plan", str("node"), "--json"}, nil
|
|
case "assign", "unassign":
|
|
if err := need("node", "module"); err != nil {
|
|
return nil, err
|
|
}
|
|
// Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of
|
|
// the seats that apply resources depend on each other and go on together.
|
|
return append([]string{verb, str("node")}, splitModules(str("module"))...), nil
|
|
case "pin":
|
|
if err := need("node", "provision", "from", "module"); err != nil {
|
|
return nil, err
|
|
}
|
|
return []string{"pin", str("node"), str("provision"), str("from"), str("module")}, nil
|
|
case "unpin":
|
|
if err := need("node", "provision"); err != nil {
|
|
return nil, err
|
|
}
|
|
return []string{"unpin", str("node"), str("provision")}, nil
|
|
case "push":
|
|
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
|
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
|
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
|
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
|
if err := need("why"); err != nil {
|
|
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
|
}
|
|
why := []string{"--why", str("why")}
|
|
if c := str("cause"); c != "" {
|
|
why = append(why, "--cause", c)
|
|
}
|
|
if n := str("node"); n != "" {
|
|
// behind is not read here: given with a machine, it is refused as passed over — naming
|
|
// a machine and asking for every machine behind are two requests, and guessing one
|
|
// would push a machine nobody named, or not push one somebody did.
|
|
return append([]string{"push", n, "--wait", "0"}, why...), nil
|
|
}
|
|
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
|
// first, so a caller who meant one machine reads that it was not one.
|
|
on("behind")
|
|
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
|
|
case "hand-act":
|
|
if err := need("what", "why", "cause"); err != nil {
|
|
return nil, err
|
|
}
|
|
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
|
|
if c := str("condition"); c != "" {
|
|
argv = append(argv, "--condition", c)
|
|
}
|
|
return argv, nil
|
|
case "hand-acts":
|
|
argv := []string{"hand-acts", "--json"}
|
|
if d := str("days"); d != "" {
|
|
argv = append(argv, "--days", d)
|
|
}
|
|
return argv, nil
|
|
case "durations":
|
|
argv := []string{"durations", "--json"}
|
|
if k := str("kind"); k != "" {
|
|
argv = append(argv, "--kind", k)
|
|
}
|
|
if d := str("days"); d != "" {
|
|
argv = append(argv, "--days", d)
|
|
}
|
|
return argv, nil
|
|
case "conditions":
|
|
// One verb, four shapes, as `plans` (novox/hq to-be 45 §2): a silence, one condition, the
|
|
// history, or the open ones filtered.
|
|
if key := str("silence"); key != "" {
|
|
if err := need("for", "why"); err != nil {
|
|
return nil, err
|
|
}
|
|
argv := []string{"conditions", "silence", key, "--for", str("for"), "--why", str("why")}
|
|
if c := str("cause"); c != "" {
|
|
argv = append(argv, "--cause", c)
|
|
}
|
|
return argv, nil
|
|
}
|
|
if on("history") {
|
|
argv := []string{"conditions", "history", "--json"}
|
|
if d := str("days"); d != "" {
|
|
argv = append(argv, "--days", d)
|
|
}
|
|
if k := str("key"); k != "" {
|
|
argv = append(argv, "--key", k)
|
|
}
|
|
return argv, nil
|
|
}
|
|
if k := str("key"); k != "" {
|
|
return []string{"conditions", "show", k, "--json"}, nil
|
|
}
|
|
argv := []string{"conditions", "--json"}
|
|
for _, filter := range []string{"scope", "severity", "machine"} {
|
|
if v := str(filter); v != "" {
|
|
argv = append(argv, "--"+filter, v)
|
|
}
|
|
}
|
|
return argv, nil
|
|
case "doctor":
|
|
which := 0
|
|
argv := []string{"doctor"}
|
|
for _, sub := range []string{"run", "probes", "signals"} {
|
|
if on(sub) {
|
|
which++
|
|
argv = append(argv, sub)
|
|
}
|
|
}
|
|
if which > 1 {
|
|
return nil, errors.New("doctor answers one of run, probes or signals at a time")
|
|
}
|
|
return append(argv, "--json"), nil
|
|
case "rotate":
|
|
if p := str("provision"); p != "" {
|
|
argv := []string{"rotate", p}
|
|
if c := str("consumer"); c != "" {
|
|
argv = append(argv, "--consumer", c)
|
|
}
|
|
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
|
|
// and secret stay the other shape's, and are refused as passed over.
|
|
if m := str("module"); m != "" {
|
|
argv = append(argv, "--module", m)
|
|
}
|
|
return argv, nil
|
|
}
|
|
_, node := a.given["node"]
|
|
_, module := a.given["module"]
|
|
_, secret := a.given["secret"]
|
|
if node || module || secret {
|
|
if err := need("node", "module", "secret"); err != nil {
|
|
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
|
|
}
|
|
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
|
|
}
|
|
// Neither shape: the command says its usage, which names both, and that is the answer the
|
|
// caller needs.
|
|
return []string{"rotate"}, nil
|
|
case "settings":
|
|
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
|
// because a tool has no file to hand the command; the command reads either.
|
|
if err := need("module"); err != nil {
|
|
return nil, err
|
|
}
|
|
var argv []string
|
|
if on("clear") {
|
|
argv = []string{"settings", "clear", str("module")}
|
|
} else {
|
|
argv = []string{"settings", "set", str("module")}
|
|
// Neither values nor clear: the command says its usage, which names both.
|
|
if v := str("values"); v != "" {
|
|
argv = append(argv, v)
|
|
}
|
|
}
|
|
if n := str("node"); n != "" {
|
|
argv = append(argv, "--node", n)
|
|
}
|
|
return argv, nil
|
|
case "issue":
|
|
// The same act as `module issue` at a shell (novox/hq design 25 §4): the account is minted
|
|
// into the mesh's records and delivered at the machine's next push, which is the caller's to
|
|
// ask for — so the mesh is never pushed as a side effect of a credential.
|
|
if err := need("node", "module"); err != nil {
|
|
return nil, err
|
|
}
|
|
return []string{"module", "issue", str("module"), "--node", str("node")}, nil
|
|
case "build":
|
|
if err := need("repository"); err != nil {
|
|
return nil, err
|
|
}
|
|
// Not waited for: a tool call cannot hold a connection for the minutes a build takes; the
|
|
// daemon takes the outcome in when it comes and the id follows the build (issue 176). A
|
|
// repository given without a scheme is a path on the forge holding the git seat.
|
|
argv := []string{"build", str("repository"), "--wait", "0"}
|
|
if !strings.Contains(str("repository"), "://") && !strings.HasPrefix(str("repository"), "git@") {
|
|
argv = append(argv, "--self")
|
|
}
|
|
if p := str("path"); p != "" {
|
|
argv = append(argv, "--path", p)
|
|
}
|
|
if r := str("ref"); r != "" {
|
|
argv = append(argv, "--ref", r)
|
|
}
|
|
return argv, nil
|
|
}
|
|
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
|
|
verb, catalogue.ControllerSeatName)
|
|
}
|
|
|
|
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
|
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
|
"hand-acts": true, "durations": true, "conditions": true, "doctor": true}
|
|
|
|
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
|
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
|
func repairingCommand(argv []string) string {
|
|
switch {
|
|
case argv[0] == "push":
|
|
return "push"
|
|
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close"):
|
|
return "plans " + argv[1]
|
|
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
|
return "broker consumer-reset"
|
|
case argv[0] == "hand-act":
|
|
return "hand-act record"
|
|
case argv[0] == "conditions" && len(argv) > 1 && argv[1] == "silence":
|
|
return "conditions silence"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func isWhyFlag(word string) bool {
|
|
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
|
|
}
|
|
|
|
// runVerb runs this binary with the given command line and gathers what it said.
|
|
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
|
self, err := os.Executable()
|
|
if err != nil {
|
|
return verbAnswer{}, err
|
|
}
|
|
cmd := exec.CommandContext(ctx, self, argv...)
|
|
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
|
// from a shell in this container would have, because it is that.
|
|
cmd.Env = os.Environ()
|
|
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
|
caller := link.CallerIn(ctx)
|
|
if caller == "" {
|
|
caller = "a seat call whose caller the bus did not name"
|
|
}
|
|
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
|
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
|
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
|
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
|
// nothing (2026-09-30, the first status asked through the console had no `answer`).
|
|
var stdout, stderr bytes.Buffer
|
|
cmd.Stdout = &stdout
|
|
cmd.Stderr = &stderr
|
|
runErr := cmd.Run()
|
|
answer := verbAnswer{Output: stdout.String() + stderr.String(), OK: runErr == nil}
|
|
if jsonVerbs[argv[0]] && runErr == nil {
|
|
var parsed any
|
|
if json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &parsed) == nil {
|
|
answer.Answer = parsed
|
|
}
|
|
}
|
|
var exit *exec.ExitError
|
|
if runErr != nil && !errors.As(runErr, &exit) {
|
|
// Not the command refusing — the command not running at all, which is this process's fault.
|
|
return answer, fmt.Errorf("could not run %s: %w", strings.Join(argv, " "), runErr)
|
|
}
|
|
return answer, nil
|
|
}
|
|
|
|
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
|
|
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
|
|
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
|
|
// would take the whole control plane down for one word (novox/hq ADR 0185).
|
|
func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
|
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
|
|
if !known {
|
|
return nil, nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
|
|
}
|
|
var behind []string
|
|
handlers := map[string]link.ToolHandler{}
|
|
for _, v := range seat.Serves {
|
|
verb := v.Name
|
|
if inProcess[verb] {
|
|
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
|
args := map[string]any{}
|
|
if len(bytes.TrimSpace(raw)) > 0 {
|
|
if err := json.Unmarshal(raw, &args); err != nil {
|
|
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
|
|
}
|
|
}
|
|
// Refused like any verb's: what a verb does not take is not ignored.
|
|
a, err := readArguments(verb, args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if verb == "calls" {
|
|
return callsAnswer(link.Calls, a.given["call"])
|
|
}
|
|
if verb == "doctor" {
|
|
// From the serving controller, which runs the self-check and hears the signals
|
|
// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
|
|
argv, err := a.commandLine()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sub := ""
|
|
if len(argv) > 2 {
|
|
sub = argv[1]
|
|
}
|
|
return doctorAnswer(ctx, sub)
|
|
}
|
|
return seatTools(), nil
|
|
}
|
|
continue
|
|
}
|
|
if _, err := argvFor(verb, sampleArguments(v)); err != nil {
|
|
// **A row ahead of this binary is not a reason to go silent.**
|
|
//
|
|
// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
|
|
// this build does not know means the row was widened by a newer one — the ordinary
|
|
// state of a roll-out, and of a push that put an older control plane back. Refusing to
|
|
// serve at all made that transient fatal: on 2026-10-02 one unknown verb took the whole
|
|
// mesh off the bus for ten minutes, and the way back was a human running the binary by
|
|
// hand, because the thing that would have repaired it is the thing that was down
|
|
// (novox/hq 04-ISSUES/201, ADR 0185).
|
|
//
|
|
// So the verbs this binary knows are served, and this one answers the reason instead of
|
|
// nothing: a caller gets a sentence naming the fault, and everything else keeps working
|
|
// — including the push that replaces this binary with the one whose verb it is.
|
|
behind = append(behind, verb)
|
|
reason := err
|
|
handlers[verb] = func(context.Context, json.RawMessage) (any, error) {
|
|
return nil, fmt.Errorf("%s is in this mesh's %s row and the control plane running "+
|
|
"here cannot run it: %w. It is a verb of a newer build; this one is behind",
|
|
verb, catalogue.ControllerSeatName, reason)
|
|
}
|
|
continue
|
|
}
|
|
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
|
args := map[string]any{}
|
|
if len(raw) > 0 {
|
|
if err := json.Unmarshal(raw, &args); err != nil {
|
|
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
|
|
}
|
|
}
|
|
argv, err := argvFor(verb, args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if verb == "status" && statusFrom != nil {
|
|
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
|
return statusFrom.answer(ctx)
|
|
}
|
|
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
|
// Whatever it did, `status` is composed again once it has.
|
|
defer statusFrom.nudge()
|
|
}
|
|
if answersFirst(argv) {
|
|
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
|
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
|
link.Acknowledge(ctx)
|
|
}
|
|
return runVerb(ctx, argv)
|
|
}
|
|
}
|
|
return handlers, behind, nil
|
|
}
|
|
|
|
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
|
func actsOnAPlan(args map[string]any) bool {
|
|
for _, act := range []string{"stop", "close", "retry"} {
|
|
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
|
// the records, `calls` from what this process served.
|
|
var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
|
|
|
|
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
|
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
|
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
|
|
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
|
|
func answersFirst(argv []string) bool {
|
|
return len(argv) > 0 && argv[0] == "push"
|
|
}
|
|
|
|
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
|
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
|
|
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
|
|
// the reason said beside it.
|
|
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
|
if id != "" {
|
|
c, ok, err := log.Get(id)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
|
|
"bus could not be read: %w", id, err)
|
|
}
|
|
if !ok {
|
|
if log.IsDurable() {
|
|
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
|
|
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
|
|
}
|
|
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
|
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
|
}
|
|
return c, nil
|
|
}
|
|
recent, err := log.Recent()
|
|
for i := range recent {
|
|
recent[i].Answer = nil
|
|
}
|
|
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
|
|
if log.IsDurable() {
|
|
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
|
|
broker.KeptCallsDurably, broker.CallsKeptFor)
|
|
} else {
|
|
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
|
|
}
|
|
if err != nil {
|
|
answer["unread"] = err.Error()
|
|
}
|
|
return answer, nil
|
|
}
|
|
|
|
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
|
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
|
|
func seatTools() map[string]any {
|
|
var seats []map[string]any
|
|
for _, s := range catalogue.SeatsWithAProtocol() {
|
|
if len(s.Serves) == 0 {
|
|
continue
|
|
}
|
|
var tools []map[string]any
|
|
for _, v := range s.Serves {
|
|
tools = append(tools, map[string]any{
|
|
"name": v.Name, "description": v.Description, "input": v.Input, "output": v.Output,
|
|
})
|
|
}
|
|
seats = append(seats, map[string]any{"seat": s.Name, "scope": s.Scope, "tools": tools})
|
|
}
|
|
return map[string]any{"seats": seats}
|
|
}
|
|
|
|
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
|
|
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
|
|
// more, since an argument a verb does not declare is refused.
|
|
func sampleArguments(v catalogue.Verb) map[string]any {
|
|
sample := map[string]any{}
|
|
switch required := v.Input["required"].(type) {
|
|
case []string:
|
|
for _, k := range required {
|
|
sample[k] = "x"
|
|
}
|
|
case []any:
|
|
for _, k := range required {
|
|
if name, ok := k.(string); ok {
|
|
sample[name] = "x"
|
|
}
|
|
}
|
|
}
|
|
return sample
|
|
}
|
|
|
|
// splitCommandLine splits a command line into words the way a POSIX shell does for the simple
|
|
// cases a controller command needs: spaces separate, single or double quotes group, a backslash
|
|
// escapes the next character inside double quotes or outside any. No expansion of anything.
|
|
func splitCommandLine(line string) ([]string, error) {
|
|
var words []string
|
|
var cur strings.Builder
|
|
inWord := false
|
|
quote := rune(0)
|
|
runes := []rune(line)
|
|
for i := 0; i < len(runes); i++ {
|
|
r := runes[i]
|
|
switch {
|
|
case quote == '\'':
|
|
if r == '\'' {
|
|
quote = 0
|
|
} else {
|
|
cur.WriteRune(r)
|
|
}
|
|
case quote == '"':
|
|
if r == '"' {
|
|
quote = 0
|
|
} else if r == '\\' && i+1 < len(runes) {
|
|
i++
|
|
cur.WriteRune(runes[i])
|
|
} else {
|
|
cur.WriteRune(r)
|
|
}
|
|
case r == '\'' || r == '"':
|
|
quote = r
|
|
inWord = true
|
|
case r == '\\' && i+1 < len(runes):
|
|
i++
|
|
cur.WriteRune(runes[i])
|
|
inWord = true
|
|
case r == ' ' || r == '\t' || r == '\n':
|
|
if inWord {
|
|
words = append(words, cur.String())
|
|
cur.Reset()
|
|
inWord = false
|
|
}
|
|
default:
|
|
cur.WriteRune(r)
|
|
inWord = true
|
|
}
|
|
}
|
|
if quote != 0 {
|
|
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
|
|
}
|
|
if inWord {
|
|
words = append(words, cur.String())
|
|
}
|
|
return words, nil
|
|
}
|
|
|
|
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
|
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
|
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
|
func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
|
|
about := map[string]catalogue.Verb{}
|
|
for _, s := range catalogue.SeatsWithAProtocol() {
|
|
if s.Name == catalogue.ControllerSeatName {
|
|
for _, v := range s.Serves {
|
|
about[v.Name] = v
|
|
}
|
|
}
|
|
}
|
|
verbs := make([]string, 0, len(handlers))
|
|
for verb := range handlers {
|
|
verbs = append(verbs, verb)
|
|
}
|
|
sort.Strings(verbs)
|
|
var endpoints []micro.EndpointInfo
|
|
for _, verb := range verbs {
|
|
schema, _ := json.Marshal(about[verb].Input)
|
|
// The same shape every tool runtime announces in (node-tools' announce package): the name is
|
|
// `<seat>__<verb>`, as the protocol's characters allow; the metadata is what identifies it.
|
|
endpoints = append(endpoints, micro.EndpointInfo{
|
|
Name: catalogue.ControllerSeatName + "__" + verb,
|
|
Subject: link.SeatToolSubject(catalogue.ControllerSeatName, verb),
|
|
QueueGroup: "seat." + catalogue.ControllerSeatName,
|
|
Metadata: map[string]string{
|
|
"kind": "seat", "module": catalogue.ControllerSeatName, "tool": verb,
|
|
"seat": catalogue.ControllerSeatName, "scope": "mesh", "interchangeable": "false",
|
|
"description": about[verb].Description, "schema": string(schema),
|
|
},
|
|
})
|
|
}
|
|
return micro.Info{
|
|
ServiceIdentity: micro.ServiceIdentity{
|
|
Name: catalogue.ControllerSeatName, ID: "controller", Version: "0.1.0",
|
|
Metadata: map[string]string{"seat": catalogue.ControllerSeatName, "scope": "mesh"},
|
|
},
|
|
Description: "the mesh's own verbs, answered by the holder of the mesh-controller seat",
|
|
Endpoints: endpoints,
|
|
}
|
|
}
|