Compare commits

..
Author SHA1 Message Date
jschoubben e7fcebcfa1 Judge a send by when a fault began, not when it was last raised (hq issue 348)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request
On 2026-10-09 the control node's resolver refused from 10:57:57 UTC. A
node-engine restarted by the 10:59:34 send said its names undecided, that
statement cleared the network condition, the next look raised it again
after the send, and the gate put back two builds for a fault older than
them.

- A condition keeps First across a reopening; the gate reads Began.
- An undecided network statement (unknown, starting) clears nothing.
- D10 counts what a release's tier names as rolling, so a walked
  node-engine is not core-behind on its own first machine.
- D2 raises a resolver that refuses every try at once: a refusal is an
  answer, not a loaded resolver (issue 277).
2026-10-09 13:33:22 +02:00
26 changed files with 408 additions and 1323 deletions
-20
View File
@@ -51,9 +51,6 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
if len(modules) == 0 {
return "", fmt.Errorf("assign %s names no module", node)
}
if err := refusedMovingTheController(modules); err != nil {
return "", err
}
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
// be taken without ever having been previewed (novox/hq ADR 0100).
ctx, release, err := holdNodes(ctx, open, []string{node})
@@ -212,9 +209,6 @@ func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, mo
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
// modules in one act are judged together, so a holder and its dependents come off in one command.
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
if err := refusedMovingTheController(modules); err != nil {
return "", err
}
if len(modules) == 0 {
return "", fmt.Errorf("unassign %s names no module", node)
}
@@ -366,17 +360,3 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
}
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
}
// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every
// change that says who may change what.
func refusedMovingTheController(modules []string) error {
verb, through := throughAVerb()
if !through || !slices.Contains(modules, controllerModule) {
return nil
}
return fmt.Errorf("%s is assigned and unassigned at the controller's terminal only (mesh-cli on the control-node), "+
"never through a verb (this came through %q): the node it runs on is the control-node, whose operator is the "+
"terminal (novox/hq ADR 0272). Nothing was assigned", controllerModule, verb)
}
+1 -11
View File
@@ -47,20 +47,10 @@ const servedVar = "MESH_SERVED_BY_THE_CONTROLLER"
// not anything it started, not a verb's command, not a seat call's. The serving controller marks its own
// environment (servedVar), so a build asked in it, or by any process it starts, never reads as the terminal's;
// runVerb also names the verb and the caller.
//
// **And a line mesh-cli asked as the controller's terminal** (novox/hq ADR 0272 §4): the serving controller runs it
// without the served mark and without a verb, names its caller, and marks it with cliTerminalVar — a mark only the
// mesh-cli path sets and every other command line the controller runs is stripped of (commandEnvironment).
func startedAtTheTerminal() bool {
if os.Getenv(servedVar) != "" || os.Getenv(verbVar) != "" {
return false
}
return os.Getenv(link.CallerVar) == "" || os.Getenv(cliTerminalVar) != ""
return os.Getenv(servedVar) == "" && os.Getenv(verbVar) == "" && os.Getenv(link.CallerVar) == ""
}
// cliTerminalVar marks a command line the serving controller runs as the controller's terminal for mesh-cli.
const cliTerminalVar = "MESH_CLI_TERMINAL"
// markServed marks this process, and so everything it starts, as the serving controller's.
func markServed() {
if err := os.Setenv(servedVar, "1"); err != nil {
+6 -8
View File
@@ -33,14 +33,12 @@ var deliveryOwnerWithin = 10 * time.Second
// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it.
type stalledLine struct {
ID string `json:"id"`
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
Number int `json:"number,omitempty"`
State string `json:"state"`
For string `json:"for"`
Bound string `json:"bound"`
H2 string `json:"h2"`
Says string `json:"says"`
ID string `json:"id"`
State string `json:"state"`
For string `json:"for"`
Bound string `json:"bound"`
H2 string `json:"h2"`
Says string `json:"says"`
}
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
+4 -3
View File
@@ -209,7 +209,8 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
}
// **No new condition about it**: about the machine itself, or naming the module on that machine,
// raised since the judging began. The gate's own are not evidence about the build.
// raised since the judging began. The gate's own are not evidence about the build. A fault reopened
// since, that began before the send, is not new (Began, novox/hq issue 348).
if f.judged {
if f.openErr != nil {
return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " +
@@ -219,7 +220,7 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
// A wait for a person's new login, or for a directory used as found to be handed over, is the module's
// reading, not a fault raised since the send: the gate reads it from the statement below (ADR 0254,
// novox/hq issue 339).
if c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound {
if c.Source == gateProbe || c.Began().Before(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound {
continue
}
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
@@ -331,7 +332,7 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine ||
slices.Contains(c.Subject.Also, machine))
// A directory used as found waits for a person, whatever the send did (novox/hq issue 339).
if !aboutIt || c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindUsedAsFound {
if !aboutIt || c.Source == gateProbe || c.Began().Before(since) || c.Kind == kindUsedAsFound {
kept = append(kept, c)
continue
}
+212
View File
@@ -0,0 +1,212 @@
package main
import (
"errors"
"net"
"os"
"strings"
"syscall"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address
// at 10:57:57 UTC; the machine's network condition was raised at 10:58:45. The node-engine and the
// controller were sent at 10:59:34. The new node-engine's first statement judged the names once — unknown,
// "one look failed; a second decides" — and that statement cleared the condition, though its last evidence
// still said "connection refused". The next look raised it again at 11:00:23, after the send, and both
// builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that
// began before they were sent.
// namesRefused is the control node's names part as its node-engine said it in the outage: its own
// resolver, at its own address, refusing.
func namesRefused(state string, streak int) link.NetworkPart {
p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak}
if state == link.StateUnhealthy {
p.Reason = "1 of its 2 resolvers do not answer as the mesh's do"
p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused"
p.Toward = []string{"10.77.0.1"}
}
return p
}
func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth {
state := link.StateHealthy
for _, p := range parts {
switch {
case p.State == link.StateUnhealthy:
state = link.StateUnhealthy
case p.State == link.StateUnknown && state == link.StateHealthy:
state = link.StateUnknown
}
}
return &link.NetworkHealth{State: state, Since: h0, Parts: parts}
}
// TestReplay348 replays the statements of the outage: the condition raised before the send is not
// cleared by the restarted engine's first, undecided statement, and the gate does not count it against
// the builds sent after it began.
func TestReplay348(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv, k := open.inventory, conditionsFrom
say := func(at time.Time, n *link.NetworkHealth) {
t.Helper()
if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
t.Fatal(err)
}
}
network := func() (conditions.Condition, bool) {
t.Helper()
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
for _, c := range list {
if c.Key == "machine.anchor.network" {
return c, true
}
}
return conditions.Condition{}, false
}
// 10:58:45 — the second failing look: raised.
say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2)))
raised, ok := network()
if !ok {
t.Fatal("the resolver refusing on the control node raised nothing")
}
time.Sleep(5 * time.Millisecond)
sent := time.Now().UTC()
time.Sleep(5 * time.Millisecond)
// 10:59:42 — the restarted engine's first statement: one look failed, a second decides.
say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1)))
if _, ok := network(); !ok {
t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " +
"said the fault was gone while it still refused")
}
// 11:00:23 — its second look: unhealthy again, the same raising.
say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2)))
again, ok := network()
if !ok || !again.Began().Equal(raised.Raised) {
t.Fatalf("the same fault was said as one that began at %s (raised first at %s)", again.Began(), raised.Raised)
}
// The gate on the control node, for a build sent after the fault began.
open2, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
f := gateFacts{judged: true, open: open2}
if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 {
t.Fatalf("a fault from before the send held the build: %+v", w)
}
// Decided healthy: cleared.
say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0}))
if c, ok := network(); ok {
t.Fatalf("a statement that decides the names healthy left %s open", c.Key)
}
}
// A reopening is the same fault: it began when it was first raised, and the gate reads when it began. The
// controller that cleared it on an undecided statement — or any clearing and raising within ReopenWithin —
// no longer makes a fault from before a send into one raised since it.
func TestAReopenedFaultFromBeforeTheSendIsNotRaisedSinceIt(t *testing.T) {
since := h0
c := conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork,
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"},
Summary: "anchor's network is not healthy", Source: sourceNetwork, Count: 2,
First: since.Add(-time.Minute), Raised: since.Add(time.Minute)}
f := gateFacts{judged: true, open: []conditions.Condition{c}}
if w := aboutTheMachine("anchor", []string{"mesh-controller"}, since, f); w.whole != "" || len(w.on) != 0 {
t.Fatalf("a fault reopened after the send, first raised before it, held the machine: %+v", w)
}
// The same raised first after the send is the send's to answer for.
c.First = since.Add(30 * time.Second)
f.open = []conditions.Condition{c}
if w := aboutTheMachine("anchor", []string{"mesh-controller"}, since, f); w.whole == "" {
t.Fatalf("a fault that began after the send held nothing: %+v", w)
}
// And a module's own: judgeHealth reads it the same way.
at := since.Add(2 * time.Minute)
g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor",
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing",
First: since.Add(-time.Hour), Raised: since.Add(time.Minute)}}}
if h, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") {
t.Fatalf("a module's own fault, reopened after its send and first raised before it: %v (%s)", h, why)
}
g.open[0].First = time.Time{}
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") {
t.Fatalf("a module's own fault raised after its send was not counted: %s", why)
}
}
// An undecided statement — unknown or starting — keeps the machine's network conditions; a decided one
// does not. Pure.
func TestAnUndecidedStatementKeepsTheMachinesNetworkConditions(t *testing.T) {
f := netFacts(map[string]*inventory.NetworkHealth{
"anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1}),
"laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}),
"spare": aNetwork(link.StateUnhealthy, noRoute),
})
got := undecidedMachines(f)
if !got["anchor"] || got["laptop"] || got["spare"] || len(got) != 1 {
t.Fatalf("undecided: %v", got)
}
}
// A release walks its modules without a record per module: D10 counts what its tier names as rolling,
// so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine.
func TestAReleaseRollsOutWhatItsTierNames(t *testing.T) {
plans := []inventory.Plan{
{ID: "release-1", State: inventory.PlanRolling, Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{}},
{ID: "plan-2", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"letta": {}}},
}
got := rollingModules(plans)
if !got["mesh-host"] || !got["letta"] || len(got) != 2 {
t.Fatalf("rolling: %v", got)
}
}
// A resolver that refuses every try — nothing listens on its port — is said at once, not held for the
// next run five minutes later: a refusal is the machine's answer, not a loaded resolver (issue 277).
func TestAResolverThatRefusesEveryTryIsSaidAtOnce(t *testing.T) {
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
_, port, _ := net.SplitHostPort(conn.LocalAddr().String())
_ = conn.Close() // nothing listens there now: every question is refused
quickResolvers(t, port)
got := askEveryResolver(t.Context(), map[string]string{"anchor.internal": "127.0.0.1"}, anchorOnTheNetwork, "internal")
if len(got) != 1 {
t.Fatalf("%+v", got)
}
if got[0].Confirm || !strings.Contains(got[0].Said, "refused") {
t.Fatalf("a resolver refusing every try was held for the next run: %+v", got[0])
}
}
func TestOnlyARefusalOnEveryTryIsAllRefused(t *testing.T) {
refused := &net.OpError{Op: "read", Net: "udp", Err: os.NewSyscallError("read", syscall.ECONNREFUSED)}
if !(resolverAsked{errs: []error{refused, refused}}).allRefused() {
t.Fatal("refused on every try is not all refused")
}
if (resolverAsked{errs: []error{refused, errors.New("i/o timeout")}}).allRefused() {
t.Fatal("a timeout among the tries is all refused")
}
if (resolverAsked{answered: true, errs: []error{refused}}).allRefused() || (resolverAsked{}).allRefused() {
t.Fatal("an answered question, or one never asked, is all refused")
}
}
+32 -1
View File
@@ -98,8 +98,9 @@ func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.
problems = append(problems, err.Error())
}
}
undecided := undecidedMachines(f)
for _, c := range open {
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] {
if !slices.Contains(networkKinds, c.Kind) || said[c.Key] || undecided[c.Subject.Machine] {
continue
}
why := "no machine says it any more"
@@ -116,6 +117,36 @@ func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.
return nil
}
// undecidedMachines is every machine whose newest statement has a part not yet judged: starting, or
// unknown — one look failed and a second decides (novox/hq issue 348). Such a statement does not say a
// fault is gone, so an open network condition about that machine is kept until a statement decides.
//
// On 2026-10-09 the control node's resolver refused every question from 10:58 to 11:18 UTC. A build of
// the node-engine sent at 10:59:34 restarted it; its first statement after the restart judged the names
// once (unknown, "one look failed; a second decides"), and that statement cleared the control node's
// network condition while its last evidence still said "connection refused". The second look raised it
// again forty seconds later — after the send — and the gate failed the build for a fault from before it.
// Pure.
func undecidedMachines(f networkFacts) map[string]bool {
out := map[string]bool{}
for m, h := range f.healths {
if h.Network == nil {
continue
}
if h.Network.State == link.StateUnknown || h.Network.State == link.StateStarting {
out[m] = true
continue
}
for _, p := range h.Network.Parts {
if p.State == link.StateUnknown || p.State == link.StateStarting {
out[m] = true
break
}
}
}
return out
}
// pointed is one machine's failing part that points at another machine.
type pointed struct {
from string
-4
View File
@@ -216,10 +216,6 @@ func run() error {
func usage() {
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
On a node the operator types these as 'mesh-cli <command>' (in zsh, 'nox <command>'): asked
through the node's engine, and run as the controller's terminal only on the control-node
(novox/hq ADR 0272).
migrate bring each context's schema up to date
prepare the same, asked the way the mesh asks any module (ADR 0135)
node add <name> [--adopted] create a node record; --adopted: the machine is in use
-261
View File
@@ -1,261 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"os/exec"
"slices"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The operator's command, `mesh-cli` (alias `nox`), answered here (novox/hq ADR 0272).
//
// mesh-cli asks the node-engine on its own machine; the engine reads the asking account from the kernel and asks
// this controller on its own node's subject. Here it is judged — the controller's terminal, an ordinary call, or
// nothing — and run as every verb's line is: a fresh process of this binary, with this process's environment.
//
// **The terminal** is a line from a node's operator account, on the control-node (§4). Phase 2 adds a node whose
// agents run under an account of their own, judged unable to become root (ADR 0266, not built yet); until then no
// other node is the terminal, because agents there run as its operator. **An ordinary call** is a line from that
// account elsewhere: it meets every refusal of the generic `command` verb and runs with `MESH_VERB=mesh-cli`, so a
// terminal-only change is refused with its reason. **Any other account is refused**, root included: those two
// accounts already reach the mesh's verbs through the mesh MCP server, and no other account gains anything here.
// cliVerb is what a line from mesh-cli that is not the terminal runs as: the verb its process names, so every
// terminal-only refusal applies and says it came through mesh-cli.
const cliVerb = "mesh-cli"
// cliServers are commands that serve until stopped. Run for mesh-cli they would hold a second server under this
// one until the call's bound killed it.
var cliServers = map[string]bool{"serve": true, "api": true, "board": true}
// cliVerdict is how a line is run, or why it is not.
type cliVerdict struct {
terminal bool
// why says why the line is or is not the terminal, in words the operator reads under the answer.
why string
// refused says why nothing runs.
refused string
}
// judgeCLI decides how a line from mesh-cli runs (ADR 0272 §4). nodes is every node the mesh knows; control is
// every node the controller's module is assigned to, which is exactly one in a mesh that is well.
func judgeCLI(node string, asked link.CLIAsked, nodes []inventory.Node, control []string) cliVerdict {
var record *inventory.Node
for i := range nodes {
if nodes[i].Name == node {
record = &nodes[i]
break
}
}
switch {
case record == nil:
return cliVerdict{refused: fmt.Sprintf("%s is not a node this mesh knows, so nothing ran", node)}
case asked.UID == 0 || asked.Account == "root":
return cliVerdict{refused: "mesh-cli answers the operator's own account, never root: run it as yourself, " +
"not under sudo. Nothing ran"}
case record.Account == "":
return cliVerdict{refused: fmt.Sprintf("the mesh does not know %s's operator account (`node account <node> <login>`), "+
"so no account there is answered. Nothing ran", node)}
case asked.Account != record.Account:
return cliVerdict{refused: fmt.Sprintf("mesh-cli answers %s's operator account (%s) only, and this line came "+
"from %s. Nothing ran", node, record.Account, asked.Account)}
}
if len(control) != 1 {
return cliVerdict{why: fmt.Sprintf("not the controller's terminal: the mesh names %d control-nodes, and the "+
"terminal is the one control-node's operator account", len(control))}
}
if control[0] == node {
// **A person at a terminal, not a service of the operator's** (review of ADR 0272): the tool runner and the
// account's user units run as the operator too, and only a login session is the terminal.
if asked.Session == "" {
return cliVerdict{why: fmt.Sprintf("not the controller's terminal: %s on %s asked from no login session — a "+
"service or a user unit running as the operator, not a person at a terminal", asked.Account, node)}
}
return cliVerdict{terminal: true, why: fmt.Sprintf("the controller's terminal: %s on the control-node %s, "+
"login session %s", asked.Account, node, asked.Session)}
}
return cliVerdict{why: fmt.Sprintf("not the controller's terminal: agents on %s may run as %s, so a "+
"terminal-only change is made from the control-node %s (novox/hq ADR 0272)", node, asked.Account, control[0])}
}
// controlNodes is every node the controller's module is assigned to.
func controlNodes(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) ([]string, error) {
var out []string
for _, n := range nodes {
modules, err := inv.Assigned(ctx, n.Name)
if err != nil {
return nil, err
}
if slices.Contains(modules, controllerModule) {
out = append(out, n.Name)
}
}
return out, nil
}
// controllerModule is the module that runs the controller, and so marks the control-node.
const controllerModule = "mesh-controller"
// answerMeshCLI is the serving controller's answer to mesh-cli.
func answerMeshCLI(inv *inventory.Inventory) link.CLIHandler {
return func(ctx context.Context, node string, asked link.CLIAsked) link.CLIAnswer {
if handingOver.Load() {
return link.CLIRefusal("this controller is stopping and runs no new command; ask again in a moment. Nothing ran")
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return link.CLIRefusal("the mesh's nodes cannot be read, so nothing ran: " + err.Error())
}
control, err := controlNodes(ctx, inv, nodes)
if err != nil {
return link.CLIRefusal("which node is the control-node cannot be read, so nothing ran: " + err.Error())
}
return runForMeshCLI(ctx, node, asked, judgeCLI(node, asked, nodes, control))
}
}
// cliJournal says one line in the controller's journal (its standard output); a variable so a test can read it.
var cliJournal = func(line string) { fmt.Println(line) }
// runForMeshCLI runs a judged line and answers what it said. Every line is said in the journal first: its call,
// who asked on which node, its command word only, and how it runs (review of ADR 0272).
func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliVerdict) link.CLIAnswer {
how := "as an ordinary call"
switch {
case v.refused != "":
how = "refused: " + v.refused
case v.terminal:
how = "as the controller's terminal"
}
call := link.CallIDIn(ctx)
if call == "" {
call = "(no call)"
}
cliJournal(fmt.Sprintf("mesh-cli %s: %s on %s asked %q, %s", call, asked.Account, node, asked.Line[0], how))
if v.refused != "" {
return link.CLIRefusal(v.refused)
}
if cliServers[asked.Line[0]] {
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: fmt.Sprintf("%s serves until stopped, and is not a "+
"command line mesh-cli runs. Nothing ran", asked.Line[0])}
}
verb, line := "", asked.Line
if !v.terminal {
composed, err := ordinaryLine(asked.Line)
if err != nil {
return link.CLIAnswer{Exit: 1, Why: v.why, Refused: err.Error()}
}
verb, line = cliVerb, composed
}
cmd := selfCommand(ctx, line)
cmd.Env = commandEnvironment(fmt.Sprintf("%s through mesh-cli on %s", asked.Account, node), verb, v.terminal)
// No standard input: a command that reads one gets nothing, and fails saying so (ADR 0272 §5).
cmd.Stdin = nil
var stdout, stderr bytes.Buffer
cmd.Stdout, cmd.Stderr = &stdout, &stderr
err := cmd.Run()
answer := link.CLIAnswer{Stdout: stdout.Bytes(), Stderr: stderr.Bytes(), Terminal: v.terminal, Why: v.why}
var exit *exec.ExitError
switch {
case err == nil:
case errors.As(err, &exit):
answer.Exit = exit.ExitCode()
if answer.Exit < 0 {
// Killed: by the call's bound, or by this controller stopping.
answer.Exit = 1
answer.Stderr = append(answer.Stderr, []byte(fmt.Sprintf("\nmesh-cli: the command was stopped (%v)\n",
exit))...)
}
default:
answer.Exit = 1
answer.Refused = fmt.Sprintf("could not run %s from this controller's own build: %v", strings.Join(asked.Line, " "), err)
}
return answer
}
// settingsForms is what an ordinary `settings` line may say: the settings verb's own forms.
const settingsForms = "settings set <module> <values> [--replace] [--node <node>], settings clear <module> " +
"[--node <node>], settings show <module> [--history] [--node <node>], or settings preferences [<module>] " +
"[--node <node>]"
// ordinaryLine is the command line an ordinary call runs (ADR 0272 §4): a `settings` line composed exactly as the
// settings verb composes its own, so its refusals — the terminal-only keys among them — are that verb's (review of
// ADR 0272); any other line as the generic `command` verb takes it, with its refusals.
func ordinaryLine(argv []string) ([]string, error) {
if len(argv) == 0 || argv[0] != "settings" {
// What the generic verb runs, and nothing it would refuse: its reading forms only, and never a command that
// is the terminal's alone (novox/hq ADR 0266) — the two checks argvFor makes of the `command` verb's line,
// made of the words as given rather than re-split from one string.
if err := refusedAsTheGenericCommand(argv); err != nil {
return nil, err
}
if err := terminalOnly(argv); err != nil {
return nil, err
}
return argv, nil
}
args := map[string]any{}
var words []string
rest := argv[1:]
for i := 0; i < len(rest); i++ {
w := rest[i]
switch {
case w == "--replace" || w == "-replace":
args["replace"] = "true"
case w == "--history" || w == "-history":
args["history"] = "true"
case w == "--node" || w == "-node":
if i+1 >= len(rest) {
return nil, fmt.Errorf("--node names no node; settings takes %s. Nothing ran", settingsForms)
}
i++
args["node"] = rest[i]
case strings.HasPrefix(w, "--node=") || strings.HasPrefix(w, "-node="):
_, args["node"], _ = strings.Cut(w, "=")
case strings.HasPrefix(w, "-"):
return nil, fmt.Errorf("settings takes no %s through mesh-cli outside the terminal; it takes %s. "+
"Nothing ran", w, settingsForms)
default:
words = append(words, w)
}
}
wrong := fmt.Errorf("through mesh-cli outside the terminal, settings takes the settings verb's forms: %s. "+
"Nothing ran", settingsForms)
if len(words) == 0 {
return nil, wrong
}
switch words[0] {
case "set":
if len(words) != 3 {
return nil, wrong
}
args["module"], args["values"] = words[1], words[2]
case "clear":
if len(words) != 2 {
return nil, wrong
}
args["module"], args["clear"] = words[1], "true"
case "show":
if len(words) != 2 {
return nil, wrong
}
args["module"] = words[1]
case "preferences":
if len(words) > 2 {
return nil, wrong
}
args["list"] = "preferences"
if len(words) == 2 {
args["module"] = words[1]
}
default:
return nil, wrong
}
return argvFor("settings", args)
}
-305
View File
@@ -1,305 +0,0 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// echoEnvironment makes the test binary, run as a command line, say the verb and caller it was given (TestMain).
const echoEnvironment = "MESH_TEST_ECHO_ENVIRONMENT"
var cliNodes = []inventory.Node{
{Name: "control", Account: "operator"},
{Name: "laptop", Account: "operator"},
{Name: "unnamed"},
}
func asked(account string, uid uint32, line ...string) link.CLIAsked {
return link.CLIAsked{Line: line, Account: account, UID: uid, Session: "session-1.scope"}
}
// Who is the controller's terminal, and who is an ordinary call or refused (novox/hq ADR 0272 §4).
func TestMeshCLIIsTheTerminalOnlyForTheControlNodesOperator(t *testing.T) {
control := []string{"control"}
cases := []struct {
name, node string
asked link.CLIAsked
control []string
terminal bool
refused string
why string
}{
{"the control-node's operator", "control", asked("operator", 1000, "status"), control, true, "", "the controller's terminal"},
{"another node's operator", "laptop", asked("operator", 1000, "status"), control, false, "", "agents on laptop may run as operator"},
{"another account", "control", asked("agent", 1001, "status"), control, false, "operator account (operator) only", ""},
{"root", "control", asked("root", 0, "status"), control, false, "never root", ""},
{"a node with no operator account", "unnamed", asked("operator", 1000, "status"), control, false, "does not know unnamed's operator account", ""},
{"a node the mesh does not know", "elsewhere", asked("operator", 1000, "status"), control, false, "not a node this mesh knows", ""},
{"two control-nodes", "control", asked("operator", 1000, "status"), []string{"control", "laptop"}, false, "", "2 control-nodes"},
}
for _, c := range cases {
v := judgeCLI(c.node, c.asked, cliNodes, c.control)
if v.terminal != c.terminal {
t.Errorf("%s: terminal %v, want %v (%+v)", c.name, v.terminal, c.terminal, v)
}
if c.refused == "" && v.refused != "" || c.refused != "" && !strings.Contains(v.refused, c.refused) {
t.Errorf("%s: refused %q, want %q", c.name, v.refused, c.refused)
}
if c.why != "" && !strings.Contains(v.why, c.why) {
t.Errorf("%s: why %q, want %q", c.name, v.why, c.why)
}
}
}
// The terminal runs its line without MESH_VERB, even where this process carries one; an ordinary call names
// mesh-cli; both record who asked through mesh-cli.
func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T) {
t.Setenv(echoEnvironment, "1")
t.Setenv("MESH_VERB", "leaked-from-the-serving-process")
// The serving controller marks itself (ADR 0266); its terminal line for mesh-cli is still the terminal's.
t.Setenv(servedVar, "1")
ctx := context.Background()
a := runForMeshCLI(ctx, "control", asked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"})
if a.Exit != 0 || a.Refused != "" || !a.Terminal {
t.Fatalf("the terminal's line did not run: %+v", a)
}
if got := string(a.Stdout); !strings.Contains(got, `verb=""`) || !strings.Contains(got, "operator through mesh-cli on control") ||
!strings.Contains(got, "terminal=true") {
t.Fatalf("the terminal's line ran with %s", got)
}
a = runForMeshCLI(ctx, "laptop", asked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
if a.Exit != 0 || a.Terminal || a.Why != "not the terminal" {
t.Fatalf("an ordinary line did not run as one: %+v", a)
}
if got := string(a.Stdout); !strings.Contains(got, `verb="mesh-cli"`) || !strings.Contains(got, "terminal=false") {
t.Fatalf("an ordinary line ran with %s", got)
}
}
// An ordinary call meets every refusal of the generic command verb, and nothing runs; a server is never run.
func TestAnOrdinaryCallMeetsTheCommandVerbsRefusals(t *testing.T) {
t.Setenv(echoEnvironment, "1")
ctx := context.Background()
ordinary := cliVerdict{why: "not the terminal"}
a := runForMeshCLI(ctx, "laptop", asked("operator", 1000, "cleanup", "delete", "x"), ordinary)
if a.Refused == "" || len(a.Stdout) != 0 || a.Exit != 1 || a.Why != "not the terminal" {
t.Fatalf("a repair without --why ran as an ordinary call: %+v", a)
}
a = runForMeshCLI(ctx, "laptop", asked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary)
if a.Refused != "" || !strings.Contains(string(a.Stdout), `verb="mesh-cli"`) {
t.Fatalf("an ordinary settings set did not run through the settings verb's path with MESH_VERB set: %+v", a)
}
for _, server := range []string{"serve", "api", "board"} {
a := runForMeshCLI(ctx, "control", asked("operator", 1000, server), cliVerdict{terminal: true})
if a.Refused == "" || len(a.Stdout) != 0 {
t.Fatalf("%s was run for mesh-cli: %+v", server, a)
}
}
a = runForMeshCLI(ctx, "control", asked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
if a.Refused != "agent is not answered" || len(a.Stdout) != 0 {
t.Fatalf("a refused line ran: %+v", a)
}
}
// **Which node is the terminal does not follow a verb** (review of ADR 0272): the controller's module is assigned
// and unassigned at the terminal alone, so no caller of `assign` can move the terminal to a node of its choosing.
// Asked through the acts themselves, as the verbs ask them; refused before any store is touched (none is given).
func TestTheControllersModuleIsMovedAtTheTerminalAlone(t *testing.T) {
t.Setenv("MESH_VERB", "assign")
ctx := context.Background()
if _, err := assignWith(ctx, nil, "laptop", assignOptions{}, "zsh", "mesh-controller"); err == nil ||
!strings.Contains(err.Error(), "terminal") {
t.Fatalf("assigning the controller through a verb was not refused: %v", err)
}
if _, err := assign(ctx, nil, "laptop", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") {
t.Fatalf("assigning the controller through a verb was not refused: %v", err)
}
t.Setenv("MESH_VERB", "unassign")
if _, err := unassign(ctx, nil, "control", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") {
t.Fatalf("unassigning the controller through a verb was not refused: %v", err)
}
// Another module through a verb, and the controller's at the terminal, are not refused for it.
if err := refusedMovingTheController([]string{"zsh"}); err != nil {
t.Fatalf("another module was refused: %v", err)
}
t.Setenv("MESH_VERB", "")
if err := refusedMovingTheController([]string{"mesh-controller"}); err != nil {
t.Fatalf("the terminal was refused: %v", err)
}
}
// An ordinary `settings set|clear` goes down the settings verb's own path: composed as that verb composes it, and
// run with MESH_VERB set, so its refusals — the terminal-only keys among them — are the settings command's own,
// not a blanket refusal of the generic command (review of ADR 0272).
func TestAnOrdinarySettingsLineTakesTheSettingsVerbsPath(t *testing.T) {
cases := []struct {
line []string
want string
err string
}{
{[]string{"settings", "set", "zsh", `{"execute":"withhold"}`, "--node", "laptop"}, `settings set zsh {"execute":"withhold"} --node laptop`, ""},
{[]string{"settings", "set", "zsh", "{}", "--replace"}, "settings set zsh {} --replace", ""},
{[]string{"settings", "clear", "zsh", "--node", "laptop"}, "settings clear zsh --node laptop", ""},
{[]string{"settings", "show", "zsh", "--history"}, "settings show zsh --history", ""},
{[]string{"settings", "preferences"}, "settings preferences", ""},
{[]string{"settings", "set", "zsh"}, "", "settings"},
{[]string{"settings", "set", "zsh", "{}", "--sideways"}, "", "--sideways"},
}
for _, c := range cases {
argv, err := ordinaryLine(c.line)
if c.err != "" {
if err == nil || !strings.Contains(err.Error(), c.err) {
t.Errorf("%q: refused with %v, want %q", c.line, err, c.err)
}
continue
}
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%q: composed %q (%v), want %q", c.line, argv, err, c.want)
}
}
// Anything else still meets the generic command verb's refusals.
if _, err := ordinaryLine([]string{"retire", "delete", "x"}); err == nil {
t.Error("a repair composed as an ordinary line")
}
}
// Every line is said in the controller's journal, with its call, who asked where and how it ran — its command word
// only, never the rest of the line (review of ADR 0272).
func TestEveryMeshCLILineIsSaidInTheJournal(t *testing.T) {
t.Setenv(echoEnvironment, "1")
var said []string
was := cliJournal
cliJournal = func(line string) { said = append(said, line) }
t.Cleanup(func() { cliJournal = was })
ctx := link.WithCallID(context.Background(), "call-1")
runForMeshCLI(ctx, "control", asked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`),
cliVerdict{terminal: true, why: "the terminal"})
runForMeshCLI(ctx, "control", asked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"})
all := strings.Join(said, "\n")
if len(said) != 2 || !strings.Contains(all, "call-1") || !strings.Contains(all, "operator on control") ||
!strings.Contains(all, "as the controller's terminal") || !strings.Contains(all, "refused") {
t.Fatalf("the journal said %q", said)
}
if strings.Contains(all, "s3cret") {
t.Fatalf("the journal carries the line's values: %q", said)
}
}
// **An ordinary line runs only what the generic command verb would** (review of ADR 0272, ADR 0266): its reading
// forms, and never a command that is the terminal's alone. Each of these, from another node's operator, is refused
// and runs nothing.
func TestAnOrdinaryLineRunsNothingTheCommandVerbWouldRefuse(t *testing.T) {
t.Setenv(echoEnvironment, "1")
for _, line := range [][]string{
{"node", "account", "control", "x"},
{"node", "agent-account", "control", "x", "--clear"},
{"token", "issue", "laptop"},
{"secret", "export", "x"},
{"operator", "key", "set", "x"},
{"assign", "laptop", "zsh"},
} {
if _, err := ordinaryLine(line); err == nil {
t.Errorf("%q composed as an ordinary line", line)
}
a := runForMeshCLI(context.Background(), "laptop", asked("operator", 1000, line...), cliVerdict{why: "not the terminal"})
if a.Refused == "" || len(a.Stdout) != 0 {
t.Errorf("%q ran as an ordinary line: %+v", line, a)
}
}
if argv, err := ordinaryLine([]string{"status"}); err != nil || argv[0] != "status" {
t.Fatalf("a read was refused: %v", err)
}
}
// **`calls` never shows a mesh-cli line's answer** (review of ADR 0272): the verb's own answer is read for a line
// served over a bus, and neither the answer's text nor the base64 JSON writes its bytes in is there.
func TestTheCallsVerbNeverShowsAMeshCLILinesAnswer(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
stop, err := link.OverNATS{Conn: conn}.ServeCLI(func(context.Context, string, link.CLIAsked) link.CLIAnswer {
return link.CLIAnswer{Stdout: []byte("s3cret-join")}
}, nil)
if err != nil {
t.Fatal(err)
}
defer stop()
body, _ := json.Marshal(link.CLIAsked{Line: []string{"token", "issue", "x"}, Account: "operator"})
msg, err := conn.Request(link.CLISubject("control"), body, 5*time.Second)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(msg.Data), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) {
t.Fatalf("the asker was not given its answer: %s", msg.Data)
}
recent, _ := link.Calls.Recent()
var id string
for _, c := range recent {
if c.Seat == link.CLISeat {
id = c.ID
break
}
}
shown, err := callsAnswer(link.Calls, id)
if err != nil {
t.Fatal(err)
}
said, _ := json.Marshal(shown)
if strings.Contains(string(said), "s3cret-join") ||
strings.Contains(string(said), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) {
t.Fatalf("calls showed a mesh-cli line's answer: %s", said)
}
}
// **Only the terminal's line carries the terminal's mark** (review of ADR 0272): a mark the serving controller's
// environment holds — leaked, or set by anything — is stripped from every other command line it runs, a verb's and
// an ordinary mesh-cli line alike, so neither reads as the terminal.
func TestTheTerminalsMarkIsStrippedFromEveryOtherLine(t *testing.T) {
t.Setenv(echoEnvironment, "1")
t.Setenv(cliTerminalVar, "1")
t.Setenv(servedVar, "1")
for _, env := range [][]string{commandEnvironment("someone", "status", false)} {
for _, kv := range env {
if strings.HasPrefix(kv, cliTerminalVar+"=") {
t.Fatalf("a verb's line carries the terminal's mark: %s", kv)
}
}
}
a := runForMeshCLI(context.Background(), "laptop", asked("operator", 1000, "status"), cliVerdict{why: "not the terminal"})
if got := string(a.Stdout); !strings.Contains(got, "terminal=false") || !strings.Contains(got, `verb="mesh-cli"`) {
t.Fatalf("an ordinary line with the mark in the serving environment ran as %s", got)
}
a = runForMeshCLI(context.Background(), "control", asked("operator", 1000, "status"), cliVerdict{terminal: true})
if got := string(a.Stdout); !strings.Contains(got, "terminal=true") {
t.Fatalf("the terminal's line ran as %s", got)
}
}
// **Only a login session is the terminal** (review of ADR 0272): the operator's account on the control-node, asking
// from a service — the tool runner, a user unit — and not a login session, is an ordinary call.
func TestOnlyALoginSessionIsTheTerminal(t *testing.T) {
control := []string{"control"}
v := judgeCLI("control", link.CLIAsked{Line: []string{"status"}, Account: "operator", UID: 1000}, cliNodes, control)
if v.terminal || v.refused != "" || !strings.Contains(v.why, "login session") {
t.Fatalf("a line from no login session reads %+v", v)
}
v = judgeCLI("control", link.CLIAsked{Line: []string{"status"}, Account: "operator", UID: 1000, Session: "session-3.scope"},
cliNodes, control)
if !v.terminal {
t.Fatalf("a line from a login session on the control-node reads %+v", v)
}
}
+2 -2
View File
@@ -1107,7 +1107,7 @@ func refuseTerminalSettingsThroughAVerb(ctx context.Context, inv *inventory.Inve
}
// A trusted mergeable file takes any key, so its module's whole layer is the terminal's (novox/hq issue 340).
if files := catalogue.TrustedMergeable(shelf[module]); len(files) > 0 && !sameLayer(before, after) {
return fmt.Errorf("the settings of %s on %s are set at the controller's terminal only (`mesh-cli` on the control-node), never through a verb (this "+
return fmt.Errorf("the settings of %s on %s are set at the controller's terminal only, never through a verb (this "+
"line came through %q): %s merges whatever key a layer sets into a file root or a consumer trusts, so "+
"any key could point the module at a listener of the caller's, and whoever may call a verb includes "+
"agents (novox/hq issue 340; a file nothing trusts says \"trusted\": false). Nothing was changed",
@@ -1119,7 +1119,7 @@ func refuseTerminalSettingsThroughAVerb(ctx context.Context, inv *inventory.Inve
if string(was) == string(now) {
continue
}
return fmt.Errorf("%s of %s on %s is set at the controller's terminal only (`mesh-cli` on the control-node), never through a verb (this "+
return fmt.Errorf("%s of %s on %s is set at the controller's terminal only, never through a verb (this "+
"line came through %q): it says where root creates and owns a module's directories, which of "+
"the machine's paths are mounted into its container, what the mesh's consumers trust, or what a file "+
"root or a person's session obeys takes, and whoever may call a verb includes agents (novox/hq issue 339; "+
-19
View File
@@ -748,25 +748,6 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
actions []conditions.Action) {
repository, _, _ := strings.Cut(l.ID, "@")
name := repoName(repository)
if l.State == "unannounced" {
// A pull request the forge never announced (novox/hq issue 347): no delivery exists, so there is nothing to
// stop, release or close; a new commit on its branch is announced and checked.
long := "for too long"
if d, err := time.ParseDuration(l.For); err == nil {
long = "for " + humanDuration(d)
}
if o.Resolver == conditions.ResolverOperator {
needs = "push a new commit to its branch; the forge announces it and the mesh checks it."
}
pull := "A pull request of " + name
if l.Number > 0 {
pull = fmt.Sprintf("Pull request %s #%d", name, l.Number)
}
return fmt.Sprintf("%s has had no merge check %s", pull, long),
fmt.Sprintf("%s has been open %s on a branch that requires the merge check, and the mesh was never asked "+
"to check it: the forge never announced it. It cannot merge until it is checked.", pull, long),
fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil
}
held := l.State
if held == "" {
held = "held"
-21
View File
@@ -309,24 +309,3 @@ func TestANeedNoNotificationAnswersNamesTheMeshMCPServer(t *testing.T) {
}
check("updates held", releaseHeldWords([]string{"openrazer"}, []string{"g14"}))
}
// **A pull request the forge never announced says what to do about it** (novox/hq issue 347): mesh-delivery says one
// as a stalled line in state `unannounced` — no delivery exists, so there is nothing to stop, release or close —
// and the operator's one act is a new commit on its branch, which the forge announces.
func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) {
l := stalledLine{ID: "novox/hq@bfe82315f42c", Number: 243, State: "unannounced", For: "11m0s", Bound: "10m0s",
H2: "none: the forge never announced it, so there is no delivery to close — the operator's",
Says: "novox/hq#243 is open on main, which requires the merge check, and its head has had no merge check"}
obs := stalledObservations([]stalledLine{l})
if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator {
t.Fatalf("an unannounced pull request is not the operator's: %+v", obs)
}
o := obs[0]
if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") {
t.Fatalf("it says to %q", o.Needs)
}
if !strings.Contains(o.Headline, "no merge check") || !strings.Contains(o.Headline, "#243") ||
!strings.Contains(o.Explanation, "never") {
t.Fatalf("it reads %q / %q", o.Headline, o.Explanation)
}
}
+1 -12
View File
@@ -512,15 +512,6 @@ func sortedKeysOf(m map[string]string) []string {
return out
}
// sayPlanDiff is `plan --diff`: what the declaration leaves out, then the diff. A module left out is not in
// the body, so the diff alone would say "nothing would change" for a module just assigned whose settings
// cannot compose — success-shaped silence. Said first, with why, as push and the plain plan say it
// (novox/hq ADR 0163, rule 6).
func sayPlanDiff(node string, declared sendable, diff func() error) error {
reportLeftOut(node, declared)
return diff()
}
// reportLeftOut says which of a machine's modules its declaration leaves out and why (novox/hq ADR
// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out.
func reportLeftOut(node string, declared sendable) {
@@ -1248,9 +1239,7 @@ func planCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
return sayPlanDiff(args[0], declared, func() error {
return writePlanDiff(ctx, open.inventory, args[0], body)
})
return writePlanDiff(ctx, open.inventory, args[0], body)
}
if *asJSON {
declared, err := declarationFor(ctx, open, args[0], plan, settings)
@@ -1,25 +0,0 @@
package main
import (
"os"
"strings"
"testing"
)
// 2026-10-09: nfs-server was assigned to the home server, its file asked for a setting nothing set, and
// `plan --diff` said "nothing would change". The diff now says what is left out, and why, before the diff.
func TestPlanDiffSaysAModuleLeftOutBeforeTheDiff(t *testing.T) {
declared := sendable{LeftOut: []string{"nfs-server"},
leftOutWhy: map[string]string{"nfs-server": `nothing sets "shares" for it`}}
said := printed(t, func() error {
return sayPlanDiff("home", declared, func() error {
writeDiff(os.Stdout, "home", sentDiff{}, nil)
return nil
})
})
left := strings.Index(said, "home: nfs-server left out")
nothing := strings.Index(said, "home: nothing would change")
if left < 0 || !strings.Contains(said, `nothing sets "shares" for it`) || nothing < left {
t.Errorf("the left-out module and why, then the diff:\n%s", said)
}
}
+43 -8
View File
@@ -11,6 +11,7 @@ import (
"sort"
"strings"
"sync"
"syscall"
"time"
"github.com/nats-io/nats.go"
@@ -216,6 +217,8 @@ func askEveryResolver(ctx context.Context, resolvers map[string]string, places [
type verdict struct {
wrong, unanswered, said []string
asked int
// refused says every try of every unanswered question was refused (nothing listens there).
refused bool
}
byHolder := map[string]*verdict{}
for _, q := range questions {
@@ -232,6 +235,7 @@ func askEveryResolver(ctx context.Context, resolvers map[string]string, places [
v.wrong = append(v.wrong, words)
v.said = append(v.said, said)
default:
v.refused = (len(v.unanswered) == 0 || v.refused) && q.asked.allRefused()
v.unanswered = append(v.unanswered, words)
v.said = append(v.said, said)
}
@@ -251,8 +255,11 @@ func askEveryResolver(ctx context.Context, resolvers map[string]string, places [
o.Summary = fmt.Sprintf("the mesh's resolver on %s does not answer machine names as it must: %s%s", node,
v.wrong[0], andMore(len(v.wrong)-1))
} else {
// Nothing but silence: held for the next run, which raises it if the resolver is still silent.
o.Confirm = true
// Nothing but silence: held for the next run, which raises it if the resolver is still silent —
// unless every try of every question was refused. A refusal is an answer from the machine:
// nothing listens there. It is not a loaded resolver slow to answer (issue 277), and holding it
// for the next run cost the control node's resolver five minutes on 2026-10-09 (issue 348).
o.Confirm = !v.refused
o.Summary = fmt.Sprintf("the mesh's resolver on %s does not answer: %d of the %d question(s) about the "+
"machines' names went unanswered, each asked %d times — the first, %s", node, len(v.unanswered), v.asked,
resolverTries, v.unanswered[0])
@@ -341,6 +348,19 @@ type resolverAsked struct {
errs []error
}
// allRefused says every try was refused: the resolver's machine said nothing listens on its port.
func (a resolverAsked) allRefused() bool {
if a.answered || len(a.errs) == 0 {
return false
}
for _, err := range a.errs {
if !errors.Is(err, syscall.ECONNREFUSED) {
return false
}
}
return true
}
// askResolverPatiently asks one question until it is answered as right says, at most resolverTries
// times. A wrong answer is asked again too — a resolver restarting may say NXDOMAIN for a moment — and
// stands if no try answers rightly.
@@ -1040,12 +1060,7 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
return nil, err
}
hostVersions := deliveredVersions(shelf[hostModule])
rolling := map[string]bool{}
for _, p := range plans {
for m := range p.Modules {
rolling[m] = true
}
}
rolling := rollingModules(plans)
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
@@ -1103,6 +1118,26 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
return out, nil
}
// rollingModules is every module an open plan is rolling out: those it keeps a record of, and those its
// tiers name. **A release keeps no record per module** — its walk is per machine, its modules only in its
// tier — so reading the records alone, D10 said "no plan is rolling them out" about the node-engine while
// a release walked it, and the gate on that release's first machine waited on what its own send causes
// (novox/hq issue 348). Pure.
func rollingModules(plans []inventory.Plan) map[string]bool {
rolling := map[string]bool{}
for _, p := range plans {
for m := range p.Modules {
rolling[m] = true
}
for _, tier := range p.Tiers {
for _, m := range tier {
rolling[m] = true
}
}
}
return rolling
}
// deliveredVersions are the versions a module's registered build is delivered as: the last element
// of every resource path under a `versions/` directory, which registration filled from the artifact's
// digest (catalogue `${version}`). The node-engine names itself by that directory.
-6
View File
@@ -266,12 +266,6 @@ func serve(ctx context.Context) (err error) {
return err
}
defer stopServing()
// And the operator's command on every node, asked through each node's engine (novox/hq ADR 0272).
stopCLI, err := bus.ServeCLI(answerMeshCLI(open.inventory), log.New(os.Stdout, "", log.LstdFlags))
if err != nil {
return err
}
defer stopCLI()
// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
if err != nil {
-6
View File
@@ -26,12 +26,6 @@ import (
// shape fails the suite, naming its source. The keeper would say such a summary in words at run time;
// this is where the producer is made to say it rightly in the first place.
func TestMain(m *testing.M) {
// The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and
// ends (meshcli_test.go).
if os.Getenv(echoEnvironment) != "" {
fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal())
os.Exit(0)
}
conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) {
unsaid.note(o, field, r)
}
+26 -56
View File
@@ -229,35 +229,6 @@ func quoteAll(xs []string) string {
return strings.Join(q, ", ")
}
// refusedAsTheGenericCommand is what the generic `command` verb refuses of a command line, and so what every
// line asked through a verb's route refuses: the `command` verb's, and an ordinary line from mesh-cli (novox/hq ADR
// 0272 §4). One function, so the two routes cannot drift apart.
func refusedAsTheGenericCommand(argv []string) error {
if len(argv) == 0 {
return errors.New("command names no command")
}
// A layer is written through the settings verb, never the generic one (novox/hq issue 339): the
// settings verb is where what a verb may not set is refused, and one route is one set of words.
// The command refuses places and accesses through any verb as well; this says so before it runs.
if argv[0] == "settings" && slices.ContainsFunc(argv[1:], func(w string) bool { return w == "set" || w == "clear" }) {
return &heldAtTheTerminal{msg: "settings are set and cleared through the settings verb, not the " +
"generic command; and places and accesses only at the controller's terminal (novox/hq issue 339). " +
"Nothing was done"}
}
// The generic verb only reads (novox/hq ADR 0266): what writes has a named verb that composes its own
// line, or is the operator's at the controller's terminal.
if err := commandReads(argv); err != nil {
return err
}
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
// it says why, as it would through its own verb.
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
return fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
"line (recorded in the hand-act log). Nothing was done", repair)
}
return nil
}
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
func (a *verbArguments) commandLine() ([]string, error) {
@@ -274,9 +245,28 @@ func (a *verbArguments) commandLine() ([]string, error) {
if err != nil {
return nil, err
}
if err := refusedAsTheGenericCommand(argv); err != nil {
if len(argv) == 0 {
return nil, errors.New("command names no command")
}
// A layer is written through the settings verb, never the generic one (novox/hq issue 339): the
// settings verb is where what a verb may not set is refused, and one route is one set of words.
// The command refuses places and accesses through any verb as well; this says so before it runs.
if argv[0] == "settings" && slices.ContainsFunc(argv[1:], func(w string) bool { return w == "set" || w == "clear" }) {
return nil, &heldAtTheTerminal{msg: "settings are set and cleared through the settings verb, not the " +
"generic command; and places and accesses only at the controller's terminal (novox/hq issue 339). " +
"Nothing was done"}
}
// The generic verb only reads (novox/hq ADR 0266): what writes has a named verb that composes its own
// line, or is the operator's at the controller's terminal.
if err := commandReads(argv); err != nil {
return nil, err
}
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
// it says why, as it would through its own verb.
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
"line (recorded in the hand-act log). Nothing was done", repair)
}
return argv, nil
case "tools":
return nil, errors.New("tools is answered from the records, not by a command")
@@ -929,30 +919,6 @@ func isWhyFlag(word string) bool {
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
}
// commandEnvironment is the environment of a command line this controller runs for someone: its own — the
// stores' credentials, the bus, the broker, everything a command run from a shell beside it would have, because it
// is that — with who asked, and how it came.
//
// **terminal** is said, never inferred (novox/hq ADR 0272): only a line mesh-cli asked as the controller's terminal
// passes true. Its line has no `MESH_VERB`, not the served mark ADR 0266 puts on everything the serving controller
// starts, and the terminal's mark (cliTerminalVar), so startedAtTheTerminal reads yes. Every other line names its
// verb, and is stripped of the terminal's mark whatever this process's environment holds.
func commandEnvironment(caller, verb string, terminal bool) []string {
env := make([]string, 0, len(os.Environ())+3)
for _, kv := range os.Environ() {
if strings.HasPrefix(kv, verbVar+"=") || strings.HasPrefix(kv, link.CallerVar+"=") ||
strings.HasPrefix(kv, cliTerminalVar+"=") || (terminal && strings.HasPrefix(kv, servedVar+"=")) {
continue
}
env = append(env, kv)
}
env = append(env, link.CallerVar+"="+caller)
if terminal {
return append(env, cliTerminalVar+"=1")
}
return append(env, verbVar+"="+verb)
}
// runVerb runs this binary with the given command line and gathers what it said.
//
// **This binary is the image this process runs, never the file at the path it started from**
@@ -966,17 +932,21 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
return verbAnswer{}, fmt.Errorf("%w: this controller is stopping and runs no new command", link.ErrHandingOver)
}
cmd := selfCommand(ctx, argv)
// The same environment: the stores' credentials, the bus, the broker — everything a command run
// from a shell in this container would have, because it is that.
cmd.Env = os.Environ()
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
caller := link.CallerIn(ctx)
if caller == "" {
caller = "a seat call whose caller the bus did not name"
}
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
// And which verb, so the connection it dials says so in the bus's list (novox/hq issue 327).
verb, _ := ctx.Value(verbKey{}).(string)
if verb == "" {
verb = argv[0]
}
cmd.Env = commandEnvironment(caller+", through the "+catalogue.ControllerSeatName+" seat", verb, false)
cmd.Env = append(cmd.Env, verbVar+"="+verb)
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
// prints its warnings beside the document, and a JSON parsed from the two together parsed
@@ -1198,7 +1168,7 @@ func answersFirst(argv []string) bool {
// the reason said beside it.
func callsAnswer(log *link.CallLog, id string) (any, error) {
if id != "" {
c, ok, err := log.Shown(id)
c, ok, err := log.Get(id)
if err != nil {
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
"bus could not be read: %w", id, err)
+2 -4
View File
@@ -87,10 +87,8 @@ var WritersTable = []WriterRow{
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
// And its health statement between reports (novox/hq ADR 0240): the same writer stating the same
// machine, inside the grant it already had (`mesh.control.<its own>.>`). And a line mesh-cli was given
// on the machine (novox/hq ADR 0272): the node is what the controller judges the terminal by, so that
// subject is this machine's engine's alone.
Subjects: []string{"mesh.control.*.report", "mesh.control.*.health", "mesh.control.*.cli"}, Writes: ownMachine},
// machine, inside the grant it already had (`mesh.control.<its own>.>`).
Subjects: []string{"mesh.control.*.report", "mesh.control.*.health"}, Writes: ownMachine},
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
-5
View File
@@ -86,10 +86,6 @@ func TestASecondWriterIsRefusedAtComposition(t *testing.T) {
[]string{"mesh.control.>"}, "a machine's applied state and its report"},
{"a machine publishing another's report", Principal{Kind: KindNode, Node: "one"},
[]string{"mesh.control.two.report"}, "a machine's applied state and its report"},
{"a machine asking the controller as another (mesh-cli, ADR 0272)", Principal{Kind: KindNode, Node: "one"},
[]string{"mesh.control.two.cli"}, "a machine's applied state and its report"},
{"a module asking the controller as a machine (mesh-cli, ADR 0272)", Principal{Kind: KindModule, Module: "shop"},
[]string{"mesh.control.one.cli"}, "a machine's applied state and its report"},
{"a machine publishing every machine's", Principal{Kind: KindNode, Node: "one"},
[]string{"mesh.control.*.>"}, "a machine's applied state and its report"},
{"a module writing the lease", Principal{Kind: KindModule, Module: "shop"},
@@ -119,7 +115,6 @@ func TestASecondWriterIsRefusedAtComposition(t *testing.T) {
publish []string
}{
{Principal{Kind: KindNode, Node: "one"}, []string{"mesh.control.one.>"}},
{Principal{Kind: KindNode, Node: "one"}, []string{"mesh.control.one.cli"}},
{Principal{Kind: KindController}, []string{"mesh.node.>", "$JS.API.>", "$KV.mesh-controller_lease.>"}},
{Principal{Kind: KindModule, Module: "gitea"}, []string{"mesh.mod.gitea.event.pull.merged"}},
{Principal{Kind: KindModule, Module: "shop"}, []string{"$JS.API.CONSUMER.CREATE.KV_shop_carts.>"}},
+16
View File
@@ -145,6 +145,11 @@ type Condition struct {
// Raised is when it was first observed this time; LastObserved the newest observation.
Raised time.Time `json:"raised"`
LastObserved time.Time `json:"last-observed"`
// First is when this fault was first raised, a reopening within ReopenWithin counted as the same
// fault: Raised of the first raising. Absent on one written before it was kept, and on a first
// raising, where it is Raised (Began). A reopening is the same fault, so what asks when a fault began
// — the gate, asking whether it began after a send — reads Began, never Raised (novox/hq issue 348).
First time.Time `json:"first,omitzero"`
// Observations is how many times it was observed since raised.
Observations int `json:"observations"`
// Count is how many times it has been raised, a reopening within ReopenWithin counted.
@@ -274,3 +279,14 @@ func Order(list []Condition) {
return list[i].Key < list[j].Key
})
}
// Began is when this fault began as the mesh knows it: its first raising, a reopening within
// ReopenWithin being the same fault again (novox/hq issue 348). A condition that cleared on a statement
// that judged nothing yet — a node-engine just restarted — and was raised again a minute later began
// when it was first raised, not at the reopening.
func (c Condition) Began() time.Time {
if !c.First.IsZero() && c.First.Before(c.Raised) {
return c.First
}
return c.Raised
}
+10 -5
View File
@@ -77,8 +77,10 @@ type Keeper struct {
}
type clearing struct {
at time.Time
count int
at time.Time
count int
// began is when the fault that cleared began, so its reopening keeps it (Condition.Began).
began time.Time
silenced *Silence
// tried is what healers tried before it cleared: a reopening is the same fault, and what was
// tried on it is still what was tried.
@@ -120,8 +122,8 @@ func NewKeeper(ctx context.Context, o Options) *Keeper {
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
for _, e := range recent {
if e.Change == ChangeCleared {
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, silenced: e.Condition.Silenced,
tried: e.Condition.Tried}
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, began: e.Condition.Began(),
silenced: e.Condition.Silenced, tried: e.Condition.Tried}
}
}
} else {
@@ -197,6 +199,9 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
k.mu.Lock()
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= ReopenWithin {
c.Count, change = before.count+1, ChangeReopened
if !before.began.IsZero() && before.began.Before(now) {
c.First = before.began
}
// A silence a person gave the condition before it cleared still holds: they said
// they knew, and the same fault again ten minutes later is what they knew about.
if before.silenced != nil && now.Before(before.silenced.Until) {
@@ -313,7 +318,7 @@ func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bo
}
now := k.now().UTC()
k.mu.Lock()
k.cleared[key] = clearing{at: now, count: c.Count, silenced: c.Silenced, tried: c.Tried}
k.cleared[key] = clearing{at: now, count: c.Count, began: c.Began(), silenced: c.Silenced, tried: c.Tried}
k.mu.Unlock()
k.tell(Event{Condition: c, At: now, Change: ChangeCleared, Why: why, Cleared: &now})
return true, nil
+53
View File
@@ -378,3 +378,56 @@ func TestATransitionIsOfferedAgainWhileTheBusIsAway(t *testing.T) {
t.Fatalf("said %+v, unsaid %d", said, k.Unsaid())
}
}
// **A reopening keeps when the fault began** (novox/hq issue 348): Raised is the reopening, Began the
// first raising — also from a keeper that read what cleared from the history — and past the window a
// raising is a new fault that begins then.
func TestAReopeningKeepsWhenTheFaultBegan(t *testing.T) {
k, store, told, c := keeper(t)
ctx := t.Context()
first, err := k.Observe(ctx, silent("ace"))
if err != nil {
t.Fatal(err)
}
if !first.Began().Equal(first.Raised) || !first.First.IsZero() {
t.Fatalf("a first raising began at %s, raised %s, first %s", first.Began(), first.Raised, first.First)
}
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
t.Fatal(err)
}
c.pass(time.Minute)
again, err := k.Observe(ctx, silent("ace"))
if err != nil {
t.Fatal(err)
}
if !again.Raised.After(first.Raised) || !again.Began().Equal(first.Raised) {
t.Fatalf("reopened: raised %s, began %s; first raised %s", again.Raised, again.Began(), first.Raised)
}
// Through a restarted controller, reading the clearing from the history.
if _, err := k.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
t.Fatal(err)
}
settled(t, told, 4)
k.Close(context.Background())
next := NewKeeper(ctx, Options{Store: store, History: store, Now: c.now})
defer next.Close(context.Background())
c.pass(time.Minute)
third, err := next.Observe(ctx, silent("ace"))
if err != nil {
t.Fatal(err)
}
if !third.Began().Equal(first.Raised) {
t.Fatalf("after a restart the reopening began at %s, not %s", third.Began(), first.Raised)
}
if _, err := next.Clear(ctx, "machine.ace.silent", "heard again"); err != nil {
t.Fatal(err)
}
c.pass(ReopenWithin + time.Minute)
fourth, err := next.Observe(ctx, silent("ace"))
if err != nil {
t.Fatal(err)
}
if !fourth.Began().Equal(fourth.Raised) {
t.Fatalf("past the window the fault began at %s, raised %s", fourth.Began(), fourth.Raised)
}
}
-52
View File
@@ -324,15 +324,6 @@ func (l *CallLog) finish(c *Call, answer []byte, failed, answeredAlready bool) {
return
}
onTheBus := *c
if c.Seat == CLISeat {
// **A mesh-cli line's answer is its asker's alone** (ADR 0272): what a command at the controller's terminal
// printed — a token, a secret's reference — and `calls` answers anyone who may call the seat. Kept in this
// process's memory for the asker to follow, and never on the bus.
onTheBus.Answer, _ = json.Marshal(map[string]any{"not kept": "a mesh-cli line's answer, its asker's alone: " +
"kept in the memory of the controller that ran it, for the asker to follow"})
l.keep(onTheBus)
return
}
if len(onTheBus.Answer) > keptOnTheBusAtMost {
// A record on the bus is one message too, and one larger than the bus carries is refused whole —
// the call's state with it (novox/hq issue 314). The answer stays in this process's memory, and
@@ -400,29 +391,6 @@ func (l *CallLog) Recent() ([]Call, error) {
return out, nil
}
// inMemory is one call this process served and still holds, whole.
func (l *CallLog) inMemory(id string) (Call, bool) {
l.mu.Lock()
defer l.mu.Unlock()
for _, c := range l.calls {
if c.ID == id {
return *c, true
}
}
return Call{}, false
}
// Shown is one kept call as `calls` shows it: whole, but for a mesh-cli line's answer, which is its asker's alone
// and followed by it (ADR 0272) — `calls` answers anyone who may call the seat.
func (l *CallLog) Shown(id string) (Call, bool, error) {
c, found, err := l.Get(id)
if found && c.Seat == CLISeat {
c.Answer, _ = json.Marshal(map[string]any{"not shown": "a mesh-cli line's answer is its asker's alone: " +
"mesh-cli follows it"})
}
return c, found, err
}
// Get is one kept call: from memory, or from the bus when this process did not serve it.
func (l *CallLog) Get(id string) (Call, bool, error) {
l.mu.Lock()
@@ -463,11 +431,6 @@ func kept(args json.RawMessage) json.RawMessage {
switch {
case k == "values" || k == "secret":
out[k] = "(given, not kept)"
case k == "line":
// A mesh-cli line (ADR 0272): its command word, never the rest, which may carry settings.
if words, ok := v.([]any); ok && len(words) > 0 {
out[k] = []any{words[0], "(the rest given, not kept)"}
}
case isString && len(s) <= 120:
out[k] = s
case isString:
@@ -550,19 +513,6 @@ var watched struct {
conns map[*nats.Conn]bool
}
type callIDKey struct{}
// WithCallID is ctx carrying the call it serves; CallIDIn reads it back, empty outside one.
func WithCallID(ctx context.Context, id string) context.Context {
return context.WithValue(ctx, callIDKey{}, id)
}
// CallIDIn is the call ctx serves, or empty.
func CallIDIn(ctx context.Context) string {
id, _ := ctx.Value(callIDKey{}).(string)
return id
}
// answerNow is how a handler asks for its caller to be answered before it goes on (Acknowledge).
type answerNowKey struct{}
@@ -615,8 +565,6 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply
c := l.begin(seat, verb, kept(args), reply)
// Who asked travels with the call, so an act it does by hand says so (novox/hq to-be 45 §7).
ctx = context.WithValue(ctx, callerKey{}, c.Caller)
// And which call it is, for what the handler says of it in the journal.
ctx = WithCallID(ctx, c.ID)
acknowledged := make(chan struct{})
var once sync.Once
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
-220
View File
@@ -1,220 +0,0 @@
package link
import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"github.com/nats-io/nats.go"
)
// mesh-cli asks the controller through the node-engine of the machine it runs on (novox/hq ADR 0272 §3). The engine
// reads the asking account from the kernel and asks on its own node's subject, which only that node's engine may
// publish (its grant is `mesh.control.<node>.>`): so the node is a fact the bus server enforces, and the account a
// fact the kernel gave. The engine's side holds the same field names (mesh-host internal/meshcli, a test on each
// side), as for its health statement.
// CLISubjects is every node's mesh-cli subject, which the serving controller answers.
const CLISubjects = "mesh.control.*.cli"
// CLISubject is one node's.
func CLISubject(node string) string { return "mesh.control." + node + ".cli" }
// CLISeat is what a mesh-cli line is recorded under in the calls record, beside the seats' verbs: its verb there is
// the node it was asked on.
const CLISeat = "mesh-cli"
// CLIAtOnce bounds the mesh-cli lines running at once, from every node together. A person types one at a time; one
// over the bound is answered busy at once, and nothing runs.
var CLIAtOnce = 8
// CLIAsked is a line mesh-cli was given on a node, and the account the node-engine says is asking — or, with
// Follow, the call a line runs as, asked again by the same account on the same node until it ends.
type CLIAsked struct {
Line []string `json:"line,omitempty"`
Account string `json:"account"`
UID uint32 `json:"uid"`
Follow string `json:"follow,omitempty"`
// Session is the login session the asking process runs in, as the node-engine read it from the kernel's cgroup
// (`session-<id>.scope` under the account's own slice), or empty: a service, a user unit. Only a login session is
// the terminal (novox/hq ADR 0272).
Session string `json:"session,omitempty"`
}
// CLIAnswer is what the controller answers, as the `result` of a call's answer: what the command printed, how it
// exited, whether it ran as the controller's terminal and why, or why nothing ran. A line still running is answered
// as every call is (`running`, `call`), and followed.
type CLIAnswer struct {
Stdout []byte `json:"stdout,omitempty"`
Stderr []byte `json:"stderr,omitempty"`
Exit int `json:"exit"`
Terminal bool `json:"terminal"`
Why string `json:"why,omitempty"`
Refused string `json:"refused,omitempty"`
Cut bool `json:"cut,omitempty"`
}
// CLIRefusal is an answer saying nothing ran, and why.
func CLIRefusal(why string) CLIAnswer { return CLIAnswer{Exit: 1, Refused: why} }
// CLINode is the node a mesh-cli subject names, or false for any other subject.
func CLINode(subject string) (string, bool) {
rest, ok := strings.CutPrefix(subject, "mesh.control.")
if !ok {
return "", false
}
node, tail, ok := strings.Cut(rest, ".")
if !ok || tail != "cli" || node == "" {
return "", false
}
return node, true
}
// CLIHandler answers one line from one node.
type CLIHandler func(ctx context.Context, node string, asked CLIAsked) CLIAnswer
// ServeCLI answers mesh-cli for every node until stopped: one queue group, so of two controllers during a handover
// one answers.
//
// **Every line is a call** (review of ADR 0272): run, recorded and answered as a seat's verb is (calls.go) — its
// caller answered within AnswerWithin that it is still running, with its call, and the line's answer kept for
// the asker to follow. The bus permits an answer for a minute only (broker.ResponseTTL); a line answered when it
// ends lost every answer after that, and mesh-cli said nothing ran of a line that had.
func (b OverNATS) ServeCLI(handle CLIHandler, logger *log.Logger) (func(), error) {
return b.serveCLI(Calls, CLIAtOnce, handle, logger)
}
func (b OverNATS) serveCLI(calls *CallLog, atOnce int, handle CLIHandler, logger *log.Logger) (func(), error) {
done := make(chan struct{})
slots := make(chan struct{}, atOnce)
bind := func() (*nats.Subscription, error) {
return b.Conn.QueueSubscribe(CLISubjects, "mesh-cli", func(msg *nats.Msg) {
go b.answerCLI(msg, calls, slots, handle, logger)
})
}
sub, err := bind()
if err != nil {
return nil, fmt.Errorf("serving %s: %w", CLISubjects, err)
}
go keepBound(sub, bind, CLISubjects, done, logger)
return func() {
close(done)
_ = sub.Unsubscribe()
}, nil
}
// cliEnvelope is an answer as every call's is: its result.
func cliEnvelope(a CLIAnswer) []byte {
body, _ := json.Marshal(map[string]any{"result": a})
return body
}
func (b OverNATS) answerCLI(msg *nats.Msg, calls *CallLog, slots chan struct{}, handle CLIHandler, logger *log.Logger) {
if msg.Reply == "" {
return
}
respond := func(body []byte) {
if err := msg.Respond(body); err != nil && logger != nil {
logger.Printf("mesh-cli on %s: the answer could not be sent: %v", msg.Subject, err)
}
}
node, ok := CLINode(msg.Subject)
var asked CLIAsked
switch {
case !ok:
respond(cliEnvelope(CLIRefusal("not a mesh-cli subject: " + msg.Subject)))
return
case json.Unmarshal(msg.Data, &asked) != nil:
respond(cliEnvelope(CLIRefusal("the node-engine's request could not be read, so nothing ran")))
return
case asked.Follow != "":
respond(calls.followCLI(asked.Follow, node, asked.Account))
return
case len(asked.Line) == 0:
respond(cliEnvelope(CLIRefusal("the request names no command, so nothing ran")))
return
}
select {
case slots <- struct{}{}:
defer func() { <-slots }()
default:
respond(cliEnvelope(CLIRefusal(fmt.Sprintf("busy: %d lines from mesh-cli are running already; ask again "+
"when one has ended. Nothing ran", cap(slots)))))
return
}
limit := b.Conn.MaxPayload()
calls.serveCallWithin(CLISeat, node, msg.Data, msg.Reply, func(ctx context.Context, _ json.RawMessage) (any, error) {
return fitCLI(handle(ctx, node, asked), limit-4096), nil
}, msg.Respond, limit, logger)
}
// followCLI answers the asker of a line what came of it: running still, its answer, or why that is not known. Only
// the account that asked it, on the node it was asked on: the answer is what the command printed, and `calls`
// keeps it from everyone else.
func (l *CallLog) followCLI(id, node, account string) []byte {
noSuch := func() []byte {
body, _ := json.Marshal(map[string]any{"error": fmt.Sprintf("no mesh-cli call %s was asked by %s on %s", id,
account, node)})
return body
}
c, inMemory := l.inMemory(id)
if !inMemory {
kept, found, err := l.Get(id)
if err != nil || !found {
return noSuch()
}
c = kept
}
var args CLIAsked
_ = json.Unmarshal(c.Args, &args)
if c.Seat != CLISeat || c.Verb != node || args.Account != account {
return noSuch()
}
switch {
case c.State == CallRunning:
return running(&c, AnswerWithin, false, l.Follow)
case c.State == CallAbandoned:
body, _ := json.Marshal(map[string]any{"error": fmt.Sprintf("call %s was running under a controller that "+
"stopped before it finished: it may have done part of what it was asked, and nothing will finish it", id)})
return body
case !inMemory:
body, _ := json.Marshal(map[string]any{"error": fmt.Sprintf("call %s finished (%s), and its answer was kept "+
"only in the memory of the controller that ran it, which has stopped; whether it took effect, the "+
"mesh says (status, the hand-act log)", id, c.State)})
return body
}
return c.Answer
}
// fitCLI is an answer whose streams fit in limit bytes once written: what the command printed is cut, standard
// output first, and the cut is said (ADR 0272 §5) — never silently short.
func fitCLI(a CLIAnswer, limit int64) CLIAnswer {
body, _ := json.Marshal(a)
if limit <= 0 || int64(len(body)) <= limit {
return a
}
a.Cut = true
// Room for the rest of the answer and JSON's base64 of the streams (4 bytes for every 3).
room := (limit - 4096) * 3 / 4
if room < 0 {
room = 0
}
if int64(len(a.Stderr)) > room/2 {
a.Stderr = a.Stderr[:room/2]
}
if left := room - int64(len(a.Stderr)); int64(len(a.Stdout)) > left {
if left < 0 {
left = 0
}
a.Stdout = a.Stdout[:left]
}
return a
}
// FitCLIAnswer is the answer as one bus message of at most limit bytes, written.
func FitCLIAnswer(a CLIAnswer, limit int64) []byte {
body, _ := json.Marshal(fitCLI(a, limit))
return body
}
-269
View File
@@ -1,269 +0,0 @@
package link
import (
"context"
"encoding/base64"
"encoding/json"
"sort"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// The node-engine's request and the answer it hands mesh-cli hold these field names; the engine's side holds the
// same list (mesh-host internal/meshcli, TestTheRequestToTheControllerKeepsItsFieldNames).
func TestTheMeshCLIRequestAndAnswerKeepTheirFieldNames(t *testing.T) {
body, _ := json.Marshal(CLIAsked{Line: []string{"status"}, Account: "a", UID: 1, Session: "session-3.scope"})
if got := keysIn(t, body); got != "account line session uid" {
t.Fatalf("the request's fields are %q", got)
}
body, _ = json.Marshal(CLIAsked{Line: []string{"status"}, Account: "a", UID: 1})
if got := keysIn(t, body); got != "account line uid" {
t.Fatalf("the request's fields are %q", got)
}
body, _ = json.Marshal(CLIAnswer{Stdout: []byte("o"), Stderr: []byte("e"), Exit: 1, Terminal: true, Why: "w",
Refused: "r", Cut: true})
if got := keysIn(t, body); got != "cut exit refused stderr stdout terminal why" {
t.Fatalf("the answer's fields are %q", got)
}
body, _ = json.Marshal(CLIAsked{Follow: "call-1", Account: "a", UID: 1})
if got := keysIn(t, body); got != "account follow uid" {
t.Fatalf("a follow's fields are %q", got)
}
// What a line still running answers, in the envelope every call's answer is in: `result`, then these.
var env struct {
Result json.RawMessage `json:"result"`
}
_ = json.Unmarshal(running(&Call{ID: "call-1", Seat: CLISeat, Verb: "a"}, AnswerWithin, false, ""), &env)
if got := keysIn(t, env.Result); got != "call output running started" {
t.Fatalf("a running answer's fields are %q", got)
}
}
func keysIn(t *testing.T, body []byte) string {
t.Helper()
var m map[string]any
if err := json.Unmarshal(body, &m); err != nil {
t.Fatal(err)
}
var keys []string
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return strings.Join(keys, " ")
}
// A node's mesh-cli subject names the node, and no other subject does.
func TestTheMeshCLISubjectNamesItsNode(t *testing.T) {
if node, ok := CLINode(CLISubject("laptop")); !ok || node != "laptop" {
t.Fatalf("CLINode(%q) = %q %v", CLISubject("laptop"), node, ok)
}
for _, s := range []string{"mesh.control.laptop.report", "mesh.control..cli", "mesh.control.a.b.cli", "mesh.node.a.cli"} {
if _, ok := CLINode(s); ok {
t.Fatalf("%s was read as a mesh-cli subject", s)
}
}
}
// An answer larger than one bus message is cut to fit, and the cut is said.
func TestALargeMeshCLIAnswerIsCutAndSaysSo(t *testing.T) {
a := CLIAnswer{Stdout: []byte(strings.Repeat("o", 200000)), Stderr: []byte(strings.Repeat("e", 1000)), Why: "the terminal"}
body := FitCLIAnswer(a, 64<<10)
if len(body) > 64<<10 {
t.Fatalf("the answer is %d bytes, over the bound", len(body))
}
var got CLIAnswer
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
if !got.Cut || got.Why != "the terminal" || len(got.Stdout) == 0 || string(got.Stderr) != strings.Repeat("e", 1000) {
t.Fatalf("the cut answer is %+v", got)
}
if small := FitCLIAnswer(CLIAnswer{Stdout: []byte("ok")}, 64<<10); strings.Contains(string(small), `"cut"`) {
t.Fatal("an answer that fits was said to be cut")
}
}
// cliBus serves mesh-cli on a bus of the test's own with a call log of its own, and returns a connection to ask on.
func cliBus(t *testing.T, l *CallLog, atOnce int, handle CLIHandler) *nats.Conn {
t.Helper()
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
stop, err := OverNATS{Conn: conn}.serveCLI(l, atOnce, handle, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(stop)
return conn
}
// askCLI asks one request on a node's subject and reads the envelope.
func askCLI(t *testing.T, conn *nats.Conn, node string, asked CLIAsked) (CLIAnswer, map[string]any, string) {
t.Helper()
body, _ := json.Marshal(asked)
msg, err := conn.Request(CLISubject(node), body, 5*time.Second)
if err != nil {
t.Fatal(err)
}
var env struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
if err := json.Unmarshal(msg.Data, &env); err != nil {
t.Fatalf("not an envelope: %s", msg.Data)
}
var a CLIAnswer
var raw map[string]any
_ = json.Unmarshal(env.Result, &a)
_ = json.Unmarshal(env.Result, &raw)
return a, raw, env.Error
}
// **A line that outlasts the bus's window for an answer is followed to its answer, never lost** (review of ADR
// 0272): the first answer says it is running and names its call, and following the call by the same account on
// the same node gives what the line printed once it ends. Another account, or another node, is told no such call.
func TestALongMeshCLILineIsFollowedToItsAnswer(t *testing.T) {
was := AnswerWithin
AnswerWithin = 50 * time.Millisecond
t.Cleanup(func() { AnswerWithin = was })
release := make(chan struct{})
conn := cliBus(t, NewCallLog(), 4, func(ctx context.Context, node string, asked CLIAsked) CLIAnswer {
<-release
return CLIAnswer{Stdout: []byte("done on " + node), Terminal: true}
})
_, first, failed := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000})
call, _ := first["call"].(string)
if failed != "" || first["running"] != true || call == "" {
t.Fatalf("a long line was not answered as running with its call: %v %q", first, failed)
}
_, still, _ := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000})
if still["running"] != true {
t.Fatalf("following a running line answered %v", still)
}
close(release)
deadline := time.Now().Add(5 * time.Second)
for {
a, raw, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000})
if failed != "" {
t.Fatalf("following answered %q", failed)
}
if raw["running"] != true {
if string(a.Stdout) != "done on laptop" || !a.Terminal {
t.Fatalf("followed to %+v", a)
}
break
}
if time.Now().After(deadline) {
t.Fatal("the line never finished for its follower")
}
time.Sleep(20 * time.Millisecond)
}
if _, _, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "agent", UID: 1001}); failed == "" {
t.Fatal("another account followed the operator's line")
}
if _, _, failed := askCLI(t, conn, "desktop", CLIAsked{Follow: call, Account: "op", UID: 1000}); failed == "" {
t.Fatal("another node followed the line")
}
}
// Lines running at once are bounded: one over the bound is answered busy at once, and nothing ran.
func TestMeshCLILinesAtOnceAreBounded(t *testing.T) {
was := AnswerWithin
AnswerWithin = 50 * time.Millisecond
t.Cleanup(func() { AnswerWithin = was })
release := make(chan struct{})
t.Cleanup(func() { close(release) })
ran := make(chan struct{}, 4)
conn := cliBus(t, NewCallLog(), 1, func(context.Context, string, CLIAsked) CLIAnswer {
ran <- struct{}{}
<-release
return CLIAnswer{}
})
if _, first, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}); first["running"] != true {
t.Fatalf("the first line answered %v", first)
}
a, _, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"})
if !strings.Contains(a.Refused, "busy") || a.Exit == 0 {
t.Fatalf("a line over the bound answered %+v", a)
}
if len(ran) != 1 {
t.Fatalf("%d lines ran, one was bound", len(ran))
}
}
// A mesh-cli line's record keeps its first word, never the rest of its line, and its answer is never kept on the
// bus, where `calls` answers anyone who may call the seat.
func TestAMeshCLIRecordKeepsNeitherItsLineNorItsAnswerOnTheBus(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
writes := make(chan Call, 4)
l.writes = writes
asked, _ := json.Marshal(CLIAsked{Line: []string{"settings", "set", "x", `{"password":"s3cret"}`}, Account: "op"})
l.serveCall(CLISeat, "laptop", asked, "_INBOX.node.laptop.abcdefghijklmnopqrstuv",
func(context.Context, json.RawMessage) (any, error) {
return CLIAnswer{Stdout: []byte("s3cret-join")}, nil
},
a.respond, nil)
_ = a.only()
close(writes)
for c := range writes {
if carries(c.Args, "s3cret") || carries(c.Answer, "s3cret-join") {
t.Fatalf("the bus was sent %s / %s", c.Args, c.Answer)
}
if !strings.Contains(string(c.Args), "settings") || !strings.Contains(string(c.Args), `"op"`) {
t.Fatalf("the record does not say what was asked and by whom: %s", c.Args)
}
}
}
// `calls` answers anyone who may call the seat, agents among them: a mesh-cli line's answer is never shown there,
// even from the memory of the controller that ran it; every other call's is (review of ADR 0272).
func TestCallsNeverShowsAMeshCLILinesAnswer(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
asked, _ := json.Marshal(CLIAsked{Line: []string{"token", "issue", "x"}, Account: "operator"})
l.serveCall(CLISeat, "control", asked, "_INBOX.node.control.abcdefghijklmnopqrstuv",
func(context.Context, json.RawMessage) (any, error) {
return CLIAnswer{Stdout: []byte("s3cret-join")}, nil
},
a.respond, nil)
_ = a.only()
recent, _ := l.Recent()
shown, found, err := l.Shown(recent[0].ID)
if err != nil || !found {
t.Fatalf("the line is not shown at all: %v %v", found, err)
}
if carries(shown.Answer, "s3cret-join") {
t.Fatalf("calls shows a mesh-cli line's answer: %s", shown.Answer)
}
b := newAnswers(t)
l.serveCall("mesh-controller", "status", nil, "_INBOX.x.abcdefghijklmnopqrstuv",
func(context.Context, json.RawMessage) (any, error) { return "all well", nil }, b.respond, nil)
_ = b.only()
recent, _ = l.Recent()
if shown, _, _ := l.Shown(recent[0].ID); !strings.Contains(string(shown.Answer), "all well") {
t.Fatalf("another call's answer is withheld: %s", shown.Answer)
}
}
// carries says whether a record holds a secret as text or as the base64 JSON writes bytes in: a line's output is
// bytes, so a search for its text alone finds nothing whatever the record keeps (review of ADR 0272).
func carries(body []byte, secret string) bool {
return strings.Contains(string(body), secret) ||
strings.Contains(string(body), base64.StdEncoding.EncodeToString([]byte(secret)))
}
// The two tests above hold what they claim: each fails when the protection it names is taken away.
func TestTheWithholdingTestsHoldSomething(t *testing.T) {
// The answer as a mesh-cli line's record would carry it, were it kept: the search finds it.
body, _ := json.Marshal(map[string]any{"result": CLIAnswer{Stdout: []byte("s3cret-join")}})
if !carries(body, "s3cret-join") {
t.Fatalf("a record carrying the answer is not found carrying it: %s", body)
}
}