Files
mesh-controller/cmd/mesh-controller/seatverbs.go
T
jochen 801552c0eb Answer every seat call within ten seconds and keep what came of it (hq issue 265)
A push outlasted the console's 30s wait and, when it sent the bus its
changed user list, the broker's reload forgot the reply it may send:
the push happened and its caller was told it did not answer. Calls now
answer in full or as running with an id, a push answers before it
sends, refused answers are recorded on their call, and 'calls' reads
them back.
2026-10-06 01:14:58 +02:00

719 lines
25 KiB
Go

package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go/micro"
"os"
"os/exec"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
// The mesh's own verbs, served as the mesh-controller seat's tools (novox/hq ADR 0154, design 33).
//
// **Each tool runs the command it names, in this same binary, and answers what it printed.** That is
// ADR 0035 taken literally: the logic lives once, in the command, and a surface is an adapter with no
// decisions in it. Running a fresh process rather than calling the function keeps two things true
// that calling it would not — every command opens and closes its own stores the way it does from a
// shell, and nothing a command prints to the process's standard output can leak into another call's
// answer. It also means a refusal is the same refusal in the same words, because it is the same
// output.
// verbAnswer is what a verb answers: what the command printed, whether it succeeded, and — where the
// command speaks JSON — the same as data.
type verbAnswer struct {
Output string `json:"output"`
OK bool `json:"ok"`
Answer any `json:"answer,omitempty"`
}
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
//
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
// the verb declares but did not use for the command line it composed — given beside another that
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
// not take that" would have stopped it before anything was sent.
func argvFor(verb string, args map[string]any) ([]string, error) {
a, err := readArguments(verb, args)
if err != nil {
return nil, err
}
argv, err := a.commandLine()
if len(a.misread) > 0 {
// The table and the command line disagree: the verb reads an argument no caller can see
// in its schema, so no caller could ever pass it.
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
"table and its command lines disagree", verb, quoteAll(a.misread))
}
if err != nil {
return nil, err
}
if unused := a.unused(); len(unused) > 0 {
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
}
return argv, nil
}
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
// command line was composed from.
type verbArguments struct {
verb string
given map[string]string
used map[string]bool
declared map[string]bool
misread []string // arguments the command line read that the schema does not declare: a bug here
}
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
// wrote.
func controllerVerb(name string) (catalogue.Verb, bool) {
for _, v := range catalogue.ControllerVerbs {
if v.Name == name {
return v, true
}
}
return catalogue.Verb{}, false
}
// declaredArguments are a schema's properties, and which of them are switches.
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
switches = map[string]bool{}
props, _ := v.Input["properties"].(map[string]any)
for name, p := range props {
names = append(names, name)
desc, _ := p.(map[string]any)
switch enum := desc["enum"].(type) {
case []string:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
case []any:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
}
}
sort.Strings(names)
return names, switches
}
// readArguments refuses what the verb does not take, before anything is composed.
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
v, known := controllerVerb(verb)
if !known {
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
names, switches := declaredArguments(v)
declared := map[string]bool{}
for _, n := range names {
declared[n] = true
}
takes := "none"
if len(names) > 0 {
takes = quoteAll(names)
}
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
keys := make([]string, 0, len(args))
for k := range args {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if !declared[k] {
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
}
var value string
switch x := args[k].(type) {
case nil:
continue
case string:
value = strings.TrimSpace(x)
case bool:
if !switches[k] {
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
}
value = fmt.Sprint(x)
default:
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
}
if switches[k] {
switch value {
case "true":
case "false", "":
continue // said and off: the same as not given, and nothing passed over
default:
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
}
}
if value != "" {
a.given[k] = value
}
}
return a, nil
}
// str is one argument's value, marked as used.
func (a *verbArguments) str(key string) string {
if !a.declared[key] {
a.misread = append(a.misread, key)
}
a.used[key] = true
return a.given[key]
}
// on is a switch, marked as used.
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
// need refuses a call missing a required argument, in the verb's own words.
func (a *verbArguments) need(keys ...string) error {
for _, k := range keys {
if a.str(k) == "" {
return fmt.Errorf("%s needs %q", a.verb, k)
}
}
return nil
}
// unused are the arguments given that the command line was not composed from.
func (a *verbArguments) unused() []string {
var out []string
for k := range a.given {
if !a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
return out
}
func (a *verbArguments) usedGiven() []string {
var out []string
for k := range a.given {
if a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
if len(out) == 0 {
return []string{"nothing"}
}
return out
}
func quoteAll(xs []string) string {
q := make([]string, len(xs))
for i, x := range xs {
if x == "nothing" {
q[i] = x
continue
}
q[i] = fmt.Sprintf("%q", x)
}
return strings.Join(q, ", ")
}
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
func (a *verbArguments) commandLine() ([]string, error) {
verb, str, on, need := a.verb, a.str, a.on, a.need
switch verb {
case "command":
// The generic verb: the command line as given, split as a shell would split it, with
// nothing added — the named verbs add flags a caller cannot reach; this one is the whole
// binary and says so in its description (novox/hq ADR 0154, 0175).
if err := need("command"); err != nil {
return nil, err
}
argv, err := splitCommandLine(str("command"))
if err != nil {
return nil, err
}
if len(argv) == 0 {
return nil, errors.New("command names no command")
}
return argv, nil
case "tools":
return nil, errors.New("tools is answered from the records, not by a command")
case "status":
return []string{"status", "--json"}, nil
case "nodes":
return []string{"node", "list", "--json"}, nil
case "node":
if err := need("node"); err != nil {
return nil, err
}
return []string{"node", "show", str("node")}, nil
case "modules":
return []string{"module", "list", "--json"}, nil
case "seats":
return []string{"seats", "--json"}, nil
case "builds":
if id := str("log"); id != "" {
return []string{"builds", "--log", id}, nil
}
argv := []string{"builds"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
if m := str("module"); m != "" {
argv = append(argv, m)
}
return argv, nil
case "plans":
if r := str("repository"); r != "" {
argv := []string{"plans", "--what-if", r}
if p := str("paths"); p != "" {
argv = append(argv, "--paths", p)
}
if m := str("modules"); m != "" {
argv = append(argv, "--modules", m)
}
return argv, nil
}
for _, act := range []string{"stop", "close", "retry"} {
if id := str(act); id != "" {
return []string{"plans", act, id}, nil
}
}
if id := str("id"); id != "" {
return []string{"plans", id}, nil
}
argv := []string{"plans"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
return argv, nil
// The build queue (novox/hq ADR 0219).
case "queue":
return []string{"queue"}, nil
case "cancel", "kill":
if err := need("id"); err != nil {
return nil, err
}
return []string{verb, str("id")}, nil
case "clear":
if on("dead") {
return []string{"clear", "--dead"}, nil
}
return []string{"clear"}, nil
case "rebuild":
if err := need("what"); err != nil {
return nil, err
}
return []string{"rebuild", str("what")}, nil
case "replay":
if err := need("id"); err != nil {
return nil, err
}
argv := []string{"replay", str("id")}
if on("register") {
argv = append(argv, "--register")
}
if on("older") {
argv = append(argv, "--older")
}
return argv, nil
case "pause", "resume":
if n := str("node"); n != "" {
return []string{verb, n}, nil
}
return []string{verb}, nil
case "plan":
if err := need("node"); err != nil {
return nil, err
}
if on("files") {
return []string{"plan", str("node"), "--files"}, nil
}
return []string{"plan", str("node"), "--json"}, nil
case "assign", "unassign":
if err := need("node", "module"); err != nil {
return nil, err
}
// Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of
// the seats that apply resources depend on each other and go on together.
return append([]string{verb, str("node")}, splitModules(str("module"))...), nil
case "pin":
if err := need("node", "provision", "from", "module"); err != nil {
return nil, err
}
return []string{"pin", str("node"), str("provision"), str("from"), str("module")}, nil
case "unpin":
if err := need("node", "provision"); err != nil {
return nil, err
}
return []string{"unpin", str("node"), str("provision")}, nil
case "push":
// Sent and not waited for: the asker reads `status` for what the machine did, which is
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
// time out on every machine that takes a minute, and say nothing about the ones that did not.
if n := str("node"); n != "" {
// behind is not read here: given with a machine, it is refused as passed over — naming
// a machine and asking for every machine behind are two requests, and guessing one
// would push a machine nobody named, or not push one somebody did.
return []string{"push", n, "--wait", "0"}, nil
}
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
// first, so a caller who meant one machine reads that it was not one.
on("behind")
return []string{"push", "--behind", "--wait", "0"}, nil
case "rotate":
if p := str("provision"); p != "" {
argv := []string{"rotate", p}
if c := str("consumer"); c != "" {
argv = append(argv, "--consumer", c)
}
return argv, nil
}
_, node := a.given["node"]
_, module := a.given["module"]
_, secret := a.given["secret"]
if node || module || secret {
if err := need("node", "module", "secret"); err != nil {
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
}
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
}
// Neither shape: the command says its usage, which names both, and that is the answer the
// caller needs.
return []string{"rotate"}, nil
case "settings":
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
// because a tool has no file to hand the command; the command reads either.
if err := need("module"); err != nil {
return nil, err
}
var argv []string
if on("clear") {
argv = []string{"settings", "clear", str("module")}
} else {
argv = []string{"settings", "set", str("module")}
// Neither values nor clear: the command says its usage, which names both.
if v := str("values"); v != "" {
argv = append(argv, v)
}
}
if n := str("node"); n != "" {
argv = append(argv, "--node", n)
}
return argv, nil
case "issue":
// The same act as `module issue` at a shell (novox/hq design 25 §4): the account is minted
// into the mesh's records and delivered at the machine's next push, which is the caller's to
// ask for — so the mesh is never pushed as a side effect of a credential.
if err := need("node", "module"); err != nil {
return nil, err
}
return []string{"module", "issue", str("module"), "--node", str("node")}, nil
case "build":
if err := need("repository"); err != nil {
return nil, err
}
// Not waited for: a tool call cannot hold a connection for the minutes a build takes; the
// daemon takes the outcome in when it comes and the id follows the build (issue 176). A
// repository given without a scheme is a path on the forge holding the git seat.
argv := []string{"build", str("repository"), "--wait", "0"}
if !strings.Contains(str("repository"), "://") && !strings.HasPrefix(str("repository"), "git@") {
argv = append(argv, "--self")
}
if p := str("path"); p != "" {
argv = append(argv, "--path", p)
}
if r := str("ref"); r != "" {
argv = append(argv, "--ref", r)
}
return argv, nil
}
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
verb, catalogue.ControllerSeatName)
}
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
// runVerb runs this binary with the given command line and gathers what it said.
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
self, err := os.Executable()
if err != nil {
return verbAnswer{}, err
}
cmd := exec.CommandContext(ctx, self, argv...)
// The same environment: the stores' credentials, the bus, the broker — everything a command run
// from a shell in this container would have, because it is that.
cmd.Env = os.Environ()
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
// prints its warnings beside the document, and a JSON parsed from the two together parsed
// nothing (2026-09-30, the first status asked through the console had no `answer`).
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
runErr := cmd.Run()
answer := verbAnswer{Output: stdout.String() + stderr.String(), OK: runErr == nil}
if jsonVerbs[argv[0]] && runErr == nil {
var parsed any
if json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &parsed) == nil {
answer.Answer = parsed
}
}
var exit *exec.ExitError
if runErr != nil && !errors.As(runErr, &exit) {
// Not the command refusing — the command not running at all, which is this process's fault.
return answer, fmt.Errorf("could not run %s: %w", strings.Join(argv, " "), runErr)
}
return answer, nil
}
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
// would take the whole control plane down for one word (novox/hq ADR 0185).
func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
if !known {
return nil, nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
}
var behind []string
handlers := map[string]link.ToolHandler{}
for _, v := range seat.Serves {
verb := v.Name
if inProcess[verb] {
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
args := map[string]any{}
if len(bytes.TrimSpace(raw)) > 0 {
if err := json.Unmarshal(raw, &args); err != nil {
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
}
}
// Refused like any verb's: what a verb does not take is not ignored.
a, err := readArguments(verb, args)
if err != nil {
return nil, err
}
if verb == "calls" {
return callsAnswer(link.Calls, a.given["call"])
}
return seatTools(), nil
}
continue
}
if _, err := argvFor(verb, sampleArguments(v)); err != nil {
// **A row ahead of this binary is not a reason to go silent.**
//
// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
// this build does not know means the row was widened by a newer one — the ordinary
// state of a roll-out, and of a push that put an older control plane back. Refusing to
// serve at all made that transient fatal: on 2026-10-02 one unknown verb took the whole
// mesh off the bus for ten minutes, and the way back was a human running the binary by
// hand, because the thing that would have repaired it is the thing that was down
// (novox/hq 04-ISSUES/201, ADR 0185).
//
// So the verbs this binary knows are served, and this one answers the reason instead of
// nothing: a caller gets a sentence naming the fault, and everything else keeps working
// — including the push that replaces this binary with the one whose verb it is.
behind = append(behind, verb)
reason := err
handlers[verb] = func(context.Context, json.RawMessage) (any, error) {
return nil, fmt.Errorf("%s is in this mesh's %s row and the control plane running "+
"here cannot run it: %w. It is a verb of a newer build; this one is behind",
verb, catalogue.ControllerSeatName, reason)
}
continue
}
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
args := map[string]any{}
if len(raw) > 0 {
if err := json.Unmarshal(raw, &args); err != nil {
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
}
}
argv, err := argvFor(verb, args)
if err != nil {
return nil, err
}
if answersFirst(argv) {
// Before anything is sent: a push sends the bus's own machine first, and a broker
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
link.Acknowledge(ctx)
}
return runVerb(ctx, argv)
}
}
return handlers, behind, nil
}
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
// the records, `calls` from what this process served.
var inProcess = map[string]bool{"tools": true, "calls": true}
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
// through `command`. A push sends the machine holding the bus first when its user list changed, the
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
func answersFirst(argv []string) bool {
return len(argv) > 0 && argv[0] == "push"
}
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
// one call whole.
func callsAnswer(log *link.CallLog, id string) (any, error) {
if id != "" {
c, ok := log.Get(id)
if !ok {
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
}
return c, nil
}
recent := log.Recent()
for i := range recent {
recent[i].Answer = nil
}
return map[string]any{"calls": recent, "kept": link.KeptCalls,
"note": "newest first; `calls` with a call's id gives its whole answer"}, nil
}
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
func seatTools() map[string]any {
var seats []map[string]any
for _, s := range catalogue.SeatsWithAProtocol() {
if len(s.Serves) == 0 {
continue
}
var tools []map[string]any
for _, v := range s.Serves {
tools = append(tools, map[string]any{
"name": v.Name, "description": v.Description, "input": v.Input, "output": v.Output,
})
}
seats = append(seats, map[string]any{"seat": s.Name, "scope": s.Scope, "tools": tools})
}
return map[string]any{"seats": seats}
}
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
// more, since an argument a verb does not declare is refused.
func sampleArguments(v catalogue.Verb) map[string]any {
sample := map[string]any{}
switch required := v.Input["required"].(type) {
case []string:
for _, k := range required {
sample[k] = "x"
}
case []any:
for _, k := range required {
if name, ok := k.(string); ok {
sample[name] = "x"
}
}
}
return sample
}
// splitCommandLine splits a command line into words the way a POSIX shell does for the simple
// cases a controller command needs: spaces separate, single or double quotes group, a backslash
// escapes the next character inside double quotes or outside any. No expansion of anything.
func splitCommandLine(line string) ([]string, error) {
var words []string
var cur strings.Builder
inWord := false
quote := rune(0)
runes := []rune(line)
for i := 0; i < len(runes); i++ {
r := runes[i]
switch {
case quote == '\'':
if r == '\'' {
quote = 0
} else {
cur.WriteRune(r)
}
case quote == '"':
if r == '"' {
quote = 0
} else if r == '\\' && i+1 < len(runes) {
i++
cur.WriteRune(runes[i])
} else {
cur.WriteRune(r)
}
case r == '\'' || r == '"':
quote = r
inWord = true
case r == '\\' && i+1 < len(runes):
i++
cur.WriteRune(runes[i])
inWord = true
case r == ' ' || r == '\t' || r == '\n':
if inWord {
words = append(words, cur.String())
cur.Reset()
inWord = false
}
default:
cur.WriteRune(r)
inWord = true
}
}
if quote != 0 {
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
}
if inWord {
words = append(words, cur.String())
}
return words, nil
}
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
about := map[string]catalogue.Verb{}
for _, s := range catalogue.SeatsWithAProtocol() {
if s.Name == catalogue.ControllerSeatName {
for _, v := range s.Serves {
about[v.Name] = v
}
}
}
verbs := make([]string, 0, len(handlers))
for verb := range handlers {
verbs = append(verbs, verb)
}
sort.Strings(verbs)
var endpoints []micro.EndpointInfo
for _, verb := range verbs {
schema, _ := json.Marshal(about[verb].Input)
// The same shape every tool runtime announces in (node-tools' announce package): the name is
// `<seat>__<verb>`, as the protocol's characters allow; the metadata is what identifies it.
endpoints = append(endpoints, micro.EndpointInfo{
Name: catalogue.ControllerSeatName + "__" + verb,
Subject: link.SeatToolSubject(catalogue.ControllerSeatName, verb),
QueueGroup: "seat." + catalogue.ControllerSeatName,
Metadata: map[string]string{
"kind": "seat", "module": catalogue.ControllerSeatName, "tool": verb,
"seat": catalogue.ControllerSeatName, "scope": "mesh", "interchangeable": "false",
"description": about[verb].Description, "schema": string(schema),
},
})
}
return micro.Info{
ServiceIdentity: micro.ServiceIdentity{
Name: catalogue.ControllerSeatName, ID: "controller", Version: "0.1.0",
Metadata: map[string]string{"seat": catalogue.ControllerSeatName, "scope": "mesh"},
},
Description: "the mesh's own verbs, answered by the holder of the mesh-controller seat",
Endpoints: endpoints,
}
}