Compare commits

...
9 Commits
Author SHA1 Message Date
jochen a1a7cf451a Answer a panicking verb with an error, say flag errors in the answer, and leave refused logins out of D15
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery-group group fix/327-a-verb-reads-on-the-serving-connection delivering: 1 of 4 delivered
Read verbs now run in the serving process, where a panic would end every
call; refused logins are nobody's reconnect loop (review of hq issue 327).
2026-10-08 18:45:04 +02:00
jochen be5c4ca5bc Serve the read verbs on the serving controller's own connection, and name every connection
Each verb ran as a process that dialled the bus, so hundreds of short
connections an hour, all named mesh-controller, hid any client reconnecting
in a loop (hq issue 327). D15 now says a user whose connections keep dropping.
2026-10-08 18:43:49 +02:00
mesh-admin e3ec15f707 Merge pull request 'Fill a preference's ${setting:} from its manifest default (hq ADR 0262)' (#153) from feat/setting-defaults into main 2026-10-08 15:54:34 +00:00
mesh-admin ac91357a53 Merge pull request 'Promise unlink-dangling on the service manager, optional (hq issue 332)' (#160) from feat/service-manager-unlink-dangling into main 2026-10-08 15:52:53 +00:00
jochen b5f2c3b961 Promise unlink-dangling on the service manager, optional (hq issue 332)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer delivery to the same trunk took over its walk
disable cannot remove an enable link whose unit file is gone, so a
leftover unit stays wanted at every login with no verb to end it. Optional
until the systemd module serves it (ADR 0246 step 1).
2026-10-08 17:39:08 +02:00
jochen af63b233db Leave out a module whose stored manifest has an unknown field, and raise it (hq ADR 0262 review)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group feat/setting-defaults merged: every member is merged
A key dropped silently ran a module without what its manifest says, and a key inside a block still
failed the whole catalogue. Judge a key by what it is about, and narrow the listing to one machine.
2026-10-08 17:34:53 +02:00
jochen f5680ba8da List every module's preferences in the settings verb (hq ADR 0262)
One verb is the interface to every preference, so no module builds a settings tool of its own: each
key, its default and why, and every assigned machine's value with its source.
2026-10-08 17:24:32 +02:00
jochen 76babaea52 Read stored manifests leniently and mark the defaults layer (hq ADR 0262 review)
A strict read of the stored catalogue fails every plan and send once a manifest uses a field an older
controller lacks; registration stays strict. A node named default lost its layer to the name check.
Judge the operator's keys by whole words, and scan a default under any key.
2026-10-08 17:24:32 +02:00
jochen e41b78cd77 Fill a preference's ${setting:} from its manifest default (hq ADR 0262)
Without a default, a running module could never gain a setting: the file asking for it
failed to compose until set, and the key was refused as stray until a file asked for it.
Defaults sit under the mesh's and the node's settings, never merge into a JSON file, are
refused for the operator's own values, and settings shows each value's source.
2026-10-08 17:24:32 +02:00
45 changed files with 2027 additions and 116 deletions
+5 -1
View File
@@ -27,6 +27,8 @@ import (
"syscall"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/link"
@@ -175,7 +177,9 @@ func dialFor(credential Credential) (*broker.JetStream, string, error) {
if !credential.onTheNewBus() {
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
}
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
// Named for what it is, not the controller whose code dials it (novox/hq issue 327).
host, _ := os.Hostname()
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint, nats.Name("build agent on "+host))
if err != nil {
return nil, "", err
}
+3 -1
View File
@@ -8,6 +8,7 @@ import (
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
@@ -187,7 +188,8 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
a.mu.Lock()
a.serving = true
a.mu.Unlock()
js, err := broker.Dial(address)
// Its own connection, held as long as the lease, and named so (novox/hq issue 327).
js, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease"))
if err != nil {
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
"lease: %w", broker.BareAddress(address), err)
+5 -5
View File
@@ -879,6 +879,10 @@ func heldBy(ctx context.Context) map[string]string {
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
// being built it dials, because a build request is a one-shot and holds nothing else.
func askOverOn(seat string) (link.Builders, error) {
// The serving controller asks on its own connection (novox/hq issue 327).
if serving := servingBus.Load(); serving != nil {
return link.BuildsOn(serving, seat), nil
}
address, err := broker.BusAddress()
if err != nil {
return nil, err
@@ -937,11 +941,7 @@ func buildSeatAmong(entries []inventory.Entry) string {
// with a consumer of its own that is gone when this returns, so nothing accumulates in the server
// for the reading, and filtered by subject, so one build's lines are all that travel.
func buildLog(ctx context.Context, id string) error {
address, err := broker.BusAddress()
if err != nil {
return err
}
js, err := broker.Dial(address)
js, err := aBus()
if err != nil {
return fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
}
+29 -25
View File
@@ -5,6 +5,7 @@ import (
"errors"
"flag"
"fmt"
"io"
"os"
"slices"
"strconv"
@@ -107,19 +108,20 @@ func conditionsCommand(ctx context.Context, args []string) error {
}
switch sub {
case "list":
return listConditions(ctx, args)
return listConditions(ctx, args, os.Stdout)
case "show":
return showCondition(ctx, args)
return showCondition(ctx, args, os.Stdout)
case "silence":
return silenceCondition(ctx, args)
case "history":
return conditionHistory(ctx, args)
return conditionHistory(ctx, args, os.Stdout)
}
return errors.New(conditionsUsage)
}
func listConditions(ctx context.Context, args []string) error {
func listConditions(ctx context.Context, args []string, w io.Writer) error {
set := flag.NewFlagSet("conditions", flag.ContinueOnError)
usageTo(set, w)
scope := set.String("scope", "", "only this scope: "+strings.Join(conditions.Scopes, ", "))
severity := set.String("severity", "", "only urgent, or only warning")
machine := set.String("machine", "", "only those about this machine")
@@ -144,20 +146,20 @@ func listConditions(ctx context.Context, args []string) error {
}
}
if *asJSON {
return printJSON(map[string]any{"conditions": inBrief(out), "open": len(open), "counted": counted(out),
return printJSONTo(w, map[string]any{"conditions": inBrief(out), "open": len(open), "counted": counted(out),
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; " +
"each with its newest evidence — `conditions key=<key>` gives one whole"})
}
if len(out) == 0 {
if len(open) == 0 {
fmt.Println("no open conditions")
fmt.Fprintln(w, "no open conditions")
} else {
fmt.Printf("none of the %d open condition(s) is about that\n", len(open))
fmt.Fprintf(w, "none of the %d open condition(s) is about that\n", len(open))
}
return nil
}
for _, line := range conditionLines(out, time.Now()) {
fmt.Println(line)
fmt.Fprintln(w, line)
}
return nil
}
@@ -233,8 +235,9 @@ func conditionLines(list []conditions.Condition, now time.Time) []string {
return out
}
func showCondition(ctx context.Context, args []string) error {
func showCondition(ctx context.Context, args []string, w io.Writer) error {
set := flag.NewFlagSet("conditions show", flag.ContinueOnError)
usageTo(set, w)
asJSON := set.Bool("json", false, "as data")
rest, err := parseAround(set, args)
if err != nil {
@@ -266,34 +269,34 @@ func showCondition(ctx context.Context, args []string) error {
"history --key %s` what became of it", key, key)
}
if *asJSON {
return printJSON(c)
return printJSONTo(w, c)
}
now := time.Now()
fmt.Printf("%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
fmt.Printf(" kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
fmt.Fprintf(w, "%s %s\n %s\n\n", strings.ToUpper(string(c.Severity)), c.Key, c.Summary)
fmt.Fprintf(w, " kind %s\n about %s %s", c.Kind, c.Subject.Scope, c.Subject.ID)
if c.Subject.Machine != "" {
fmt.Printf(", on %s", c.Subject.Machine)
fmt.Fprintf(w, ", on %s", c.Subject.Machine)
}
fmt.Printf("\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
fmt.Fprintf(w, "\n raised by %s\n since %s (%s ago), observed %d time(s), last %s ago\n",
c.Source, c.Raised.Local().Format("2006-01-02 15:04:05"), roughly(now.Sub(c.Raised)), c.Observations,
now.Sub(c.LastObserved).Round(time.Second))
if c.Count > 1 {
fmt.Printf(" raised %d times, each within ten minutes of clearing\n", c.Count)
fmt.Fprintf(w, " raised %d times, each within ten minutes of clearing\n", c.Count)
}
fmt.Printf(" resolved by %s\n", resolverWords(c.Resolver))
fmt.Fprintf(w, " resolved by %s\n", resolverWords(c.Resolver))
if c.Silenced != nil {
fmt.Printf(" silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
fmt.Fprintf(w, " silenced until %s by %s: %s\n", c.Silenced.Until.Local().Format("2006-01-02 15:04"),
c.Silenced.By, c.Silenced.Why)
}
if len(c.Tried) > 0 {
fmt.Println("\n tried:")
fmt.Fprintln(w, "\n tried:")
for _, t := range c.Tried {
fmt.Printf(" %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
fmt.Fprintf(w, " %s %s — %s: %s\n", t.At.Local().Format("2006-01-02 15:04"), orHealer(t.By), t.What, t.Outcome)
}
}
fmt.Println("\n evidence, newest first:")
fmt.Fprintln(w, "\n evidence, newest first:")
for _, e := range c.Evidence {
fmt.Printf(" %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
fmt.Fprintf(w, " %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), e.Said)
}
return nil
}
@@ -378,8 +381,9 @@ func parseFor(s string) (time.Duration, error) {
return d, nil
}
func conditionHistory(ctx context.Context, args []string) error {
func conditionHistory(ctx context.Context, args []string, w io.Writer) error {
set := flag.NewFlagSet("conditions history", flag.ContinueOnError)
usageTo(set, w)
days := set.Int("days", 7, "how many days back, at most 90")
key := set.String("key", "", "only this condition")
asJSON := set.Bool("json", false, "as data")
@@ -420,10 +424,10 @@ func conditionHistory(ctx context.Context, args []string) error {
if out == nil {
out = []conditions.Event{}
}
return printJSON(map[string]any{"history": out, "days": *days})
return printJSONTo(w, map[string]any{"history": out, "days": *days})
}
if len(out) == 0 {
fmt.Printf("nothing was raised, changed or cleared in the last %d day(s)\n", *days)
fmt.Fprintf(w, "nothing was raised, changed or cleared in the last %d day(s)\n", *days)
return nil
}
for _, e := range out {
@@ -440,7 +444,7 @@ func conditionHistory(ctx context.Context, args []string) error {
line += " — " + e.Why
}
}
fmt.Println(line)
fmt.Fprintln(w, line)
}
return nil
}
+5
View File
@@ -116,6 +116,11 @@ var probeRegistry = []probe{
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
// connections, which the bus's own module reads.
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
"last hour: the bus module's nats_closed_connections", From: "issue 327", Kind: kindBusReconnects,
Phase: 1, run: probeReconnects},
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
// The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is
+15 -13
View File
@@ -6,6 +6,7 @@ import (
"errors"
"flag"
"fmt"
"io"
"os"
"slices"
"sort"
@@ -14,7 +15,6 @@ import (
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
@@ -149,14 +149,10 @@ func onTheBus(f func(*nats.Conn) error) error {
if handActConn != nil {
return f(handActConn)
}
address, err := broker.BusAddress()
js, err := aBus()
if err != nil {
return err
}
js, err := broker.Dial(address)
if err != nil {
return fmt.Errorf("cannot reach the bus: %w", err)
}
defer js.Close()
return f(js.Conn())
}
@@ -247,7 +243,13 @@ func handActCommand(ctx context.Context, args []string) error {
if len(args) > 0 && args[0] == "list" {
args = args[1:]
}
return listHandActs(ctx, args, os.Stdout)
}
// listHandActs is `hand-acts`: what was done by hand lately, and the causes done more than once.
func listHandActs(ctx context.Context, args []string, w io.Writer) error {
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
usageTo(set, w)
days := set.Int("days", 14, "how many days back")
asJSON := set.Bool("json", false, "as data")
if _, err := parseAround(set, args); err != nil {
@@ -265,27 +267,27 @@ func handActCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
fmt.Println(string(body))
fmt.Fprintln(w, string(body))
return nil
}
if len(acts) == 0 {
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
fmt.Fprintf(w, "nothing was done by hand in the last %d day(s)\n", *days)
return nil
}
for i := len(acts) - 1; i >= 0; i-- {
a := acts[i]
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
fmt.Fprintf(w, "%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
if a.Condition != "" {
fmt.Printf(", condition %s", a.Condition)
fmt.Fprintf(w, ", condition %s", a.Condition)
}
fmt.Println(")")
fmt.Fprintln(w, ")")
if pushedRecorded(a) {
carried := strings.Join(a.Carried, "; ")
if carried == "" {
carried = recordedBefore[a.ID]
}
fmt.Printf(" a push of recorded builds, no repair: %s\n", carried)
fmt.Fprintf(w, " a push of recorded builds, no repair: %s\n", carried)
}
}
if len(repeated) > 0 {
@@ -294,7 +296,7 @@ func handActCommand(ctx context.Context, args []string) error {
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
}
sort.Strings(causes)
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
fmt.Fprintf(w, "\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
strings.Join(causes, ", "))
}
return nil
+19
View File
@@ -15,6 +15,7 @@ import (
"os/signal"
"syscall"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/identity"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/licences"
@@ -53,6 +54,9 @@ func run() error {
return fmt.Errorf("no command given")
}
// Every connection this process dials says what it is (novox/hq issue 327).
broker.ConnectionName = connectionName(args[0], os.Getenv(verbVar))
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
@@ -416,3 +420,18 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
statusFrom.nudge()
return nil
}
// verbVar carries the seat verb a command runs for, from the serving controller to the process it starts.
const verbVar = "MESH_VERB"
// connectionName is what this process's connections say they are in the bus's list (novox/hq issue 327):
// the serving controller, a verb's own process and which verb, or a command run at a shell and which.
func connectionName(command, verb string) string {
switch {
case command == "serve":
return "mesh-controller serving"
case verb != "":
return "mesh-controller verb " + verb
}
return "mesh-controller command " + command
}
+163 -5
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"net"
"os"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/broker"
@@ -396,7 +397,8 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
func settingsCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("settings show <module> [--node <node>] [--history], settings set <module> <file> " +
"[--node <node>] [--replace], or settings clear <module> [--node <node>]")
"[--node <node>] [--replace], settings clear <module> [--node <node>], or settings preferences " +
"[<module>] [--node <node>]")
}
open, err := openStores(ctx)
if err != nil {
@@ -502,13 +504,92 @@ func settingsCommand(ctx context.Context, args []string) error {
}
if !has {
fmt.Printf("%s on %s: no layer — the module's definition says\n", positionals[0], where)
return nil
} else {
shown, err := json.MarshalIndent(values, "", " ")
if err != nil {
return err
}
fmt.Println(string(shown))
}
shown, err := json.MarshalIndent(values, "", " ")
// Every value the module gives a default or a layer sets, and where it came from (novox/hq
// ADR 0262): the default, the mesh's layer, or this node's. Said after the layer, which stays
// the first thing printed because a caller reads it before replacing it (ADR 0217).
known, err := inv.Catalogue(ctx)
if err != nil {
return err
}
fmt.Println(string(shown))
m, ok := known[positionals[0]]
if !ok || len(m.Settings) == 0 {
return nil
}
var layers []catalogue.Layer
if *node != "" {
if mesh, has, err := inv.Layer(ctx, "", positionals[0]); err != nil {
return err
} else if has {
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: mesh})
}
if has {
layers = append(layers, catalogue.Layer{From: *node, Values: values})
}
} else if has {
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: values})
}
fmt.Print(describeEffective(positionals[0], where, catalogue.Effective(m, layers)))
return nil
case "preferences":
// Every module's preferences, and each machine's value with its source (novox/hq ADR 0262).
if len(positionals) > 1 {
return errors.New("settings preferences [<module>] [--node <node>]")
}
only := ""
if len(positionals) == 1 {
only = positionals[0]
}
if *node != "" {
// A machine the mesh does not know is refused, never answered with an empty listing.
if _, err := inv.NodeByName(ctx, *node); err != nil {
return err
}
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
var listed []preferencesOf
for _, e := range entries {
m := e.Manifest
if len(m.Settings) == 0 || (only != "" && m.Module != only) {
continue
}
if *node != "" && !containsString(e.On, *node) {
// Asked for one machine: a module not on it has no value there to say.
continue
}
p := preferencesOf{Manifest: m, On: map[string][]catalogue.SettingSource{}}
for _, n := range e.On {
if *node != "" && n != *node {
continue
}
p.Nodes = append(p.Nodes, n)
layers, err := inv.SettingsFor(ctx, n, m.Module)
if err != nil {
return err
}
p.On[n] = catalogue.Effective(m, layers)
}
listed = append(listed, p)
}
if only != "" && len(listed) == 0 {
fmt.Printf("%s declares no preferences%s\n", only, onNode(*node))
return nil
}
if len(listed) == 0 && *node != "" {
fmt.Printf("no module on %s declares a preference\n", *node)
return nil
}
fmt.Print(describePreferences(listed))
return nil
case "clear":
@@ -522,10 +603,87 @@ func settingsCommand(ctx context.Context, args []string) error {
return nil
default:
return fmt.Errorf("settings has no %q; it has show, set and clear", args[0])
return fmt.Errorf("settings has no %q; it has show, set, clear and preferences", args[0])
}
}
// describeEffective says each setting's value on a machine or the whole mesh, where it came from, and
// the module's default when a layer overrides it.
func describeEffective(module, where string, values []catalogue.SettingSource) string {
var b strings.Builder
fmt.Fprintf(&b, "%s on %s, every value and where it comes from:\n", module, where)
for _, v := range values {
value, _ := json.Marshal(v.Value)
fmt.Fprintf(&b, " %s = %s (%s", v.Key, value, v.From)
if v.HasDefault && !v.FromDefault {
d, _ := json.Marshal(v.Default)
fmt.Fprintf(&b, "; the default is %s", d)
}
b.WriteString(")\n")
}
return b.String()
}
// onNode is ` on <node>` for one machine, nothing for the whole mesh.
func onNode(node string) string {
if node == "" {
return ""
}
return " on " + node
}
// preferencesOf is one module's preferences and its value on each machine it is assigned to.
type preferencesOf struct {
Manifest catalogue.Manifest
Nodes []string
On map[string][]catalogue.SettingSource
}
// describePreferences lists each module's preferences — key, default and why — and, per machine it is
// assigned to, the value and where it comes from (novox/hq ADR 0262).
func describePreferences(modules []preferencesOf) string {
if len(modules) == 0 {
return "no module declares a preference\n"
}
var b strings.Builder
for i, p := range modules {
if i > 0 {
b.WriteString("\n")
}
on := "assigned nowhere"
if len(p.Nodes) > 0 {
on = "on " + strings.Join(p.Nodes, ", ")
}
fmt.Fprintf(&b, "%s (%s)\n", p.Manifest.Module, on)
keys := make([]string, 0, len(p.Manifest.Settings))
for k := range p.Manifest.Settings {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
d := p.Manifest.Settings[k]
def, _ := json.Marshal(d.Default)
fmt.Fprintf(&b, " %s, default %s: %s\n", k, def, d.Why)
for _, n := range p.Nodes {
for _, s := range p.On[n] {
if s.Key != k {
continue
}
v, _ := json.Marshal(s.Value)
from := s.From
if s.FromDefault {
from = catalogue.DefaultLayer
} else if s.From != catalogue.MeshWideLayer {
from = "the node"
}
fmt.Fprintf(&b, " %s: %s (%s)\n", n, v, from)
}
}
}
}
return b.String()
}
// nodeFlag is ` --node <node>` for a machine's layer, nothing for the whole mesh's.
func nodeFlag(node string) string {
if node == "" {
+7 -2
View File
@@ -7,7 +7,9 @@ import (
"errors"
"flag"
"fmt"
"io"
"net"
"os"
"strings"
"time"
@@ -684,11 +686,14 @@ func orNotReported(s string) string {
}
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
func printJSON(v any) error {
func printJSON(v any) error { return printJSONTo(os.Stdout, v) }
// printJSONTo is printJSON to a writer of the caller's.
func printJSONTo(w io.Writer, v any) error {
body, err := json.MarshalIndent(v, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
fmt.Fprintln(w, string(body))
return nil
}
+3
View File
@@ -478,6 +478,9 @@ func settlingPending(ctx context.Context, open *stores) {
for _, line := range settlePending(ctx, open, time.Now()) {
fmt.Println(line)
}
for _, line := range raiseUnknownFields(ctx, open.inventory) {
fmt.Println(line)
}
select {
case <-ctx.Done():
return
+6
View File
@@ -543,6 +543,12 @@ var plainWordings = map[string]func(conditions.Observation) words{
Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.",
Resolved: "The listener keeps up again"}
}),
kindBusReconnects: worded(func(o conditions.Observation) words {
return words{Headline: "A client keeps losing the bus",
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " +
"While it reconnects, what it says and what it is asked waits.",
Resolved: "Resolved: the client stays connected"}
}),
"max-deliveries": worded(func(o conditions.Observation) words {
return words{Headline: "A message could not be handled",
Explanation: "The bus gave up on a message after trying to hand it over too many times.",
+3 -4
View File
@@ -515,9 +515,8 @@ func sortedKeysOf(m map[string]string) []string {
// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out.
func reportLeftOut(node string, declared sendable) {
for _, m := range declared.LeftOut {
fmt.Printf("%s: %s left out — a setting stored for it cannot compose with its definition; "+
"what the machine holds for it is kept and its containers are untouched. %s\n",
node, m, declared.leftOutWhy[m])
fmt.Printf("%s: %s left out — what the machine holds for it is kept and its containers are "+
"untouched. %s\n", node, m, declared.leftOutWhy[m])
}
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
// 0225): the machine is sent everything else, and the consumer is named.
@@ -1425,7 +1424,7 @@ func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
}
serves := catalogue.ServedOn(m, provision, ports)
if len(serves) > 0 {
serves, err = catalogue.Settle(serves, layers)
serves, err = catalogue.Settle(serves, catalogue.WithDefaults(m, layers))
if err != nil {
return nil, err
}
+3 -6
View File
@@ -9,7 +9,6 @@ import (
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -98,11 +97,9 @@ func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inven
if len(holders) == 0 {
return pauseView{}
}
address, err := broker.BusAddress()
if err != nil {
return pauseView{}
}
js, err := broker.Dial(address)
// On the serving controller's own connection when this is it: a watchdog tick while a walk waits for a
// build dialled one every 30 seconds (novox/hq issue 327).
js, err := aBus()
if err != nil {
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
return pauseView{}
+2
View File
@@ -219,6 +219,8 @@ func serve(ctx context.Context) (err error) {
}
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
handActConn = bus.Conn
// And everything else this controller does on the bus for a moment (novox/hq issue 327).
servingBus.Store(server.JetStream())
// And says when it replaced a value given by hand (novox/hq ADR 0228).
givenEvents = bus
// And a pull request's merge check, asked when the forge announces its head and said when judged
+12 -12
View File
@@ -6,6 +6,7 @@ import (
"errors"
"flag"
"fmt"
"io"
"os"
"sort"
"strings"
@@ -37,22 +38,21 @@ const (
killAnswer = 75 * time.Second
)
// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
// dialTheBus is the controller's connection, for a command that reads or changes the queue: the serving
// controller's own, lent, when this process is it (novox/hq issue 327).
func dialTheBus() (*broker.JetStream, error) {
address, err := broker.BusAddress()
if err != nil {
return nil, err
}
js, err := broker.Dial(address)
if err != nil {
return nil, fmt.Errorf("cannot reach the bus: %w", err)
}
return js, nil
return aBus()
}
// queueCommand prints every ask in the build seat's work queue.
func queueCommand(ctx context.Context, args []string) error {
return listQueue(ctx, args, os.Stdout)
}
// listQueue is `queue`: the build queue, as a person reads it or as JSON.
func listQueue(ctx context.Context, args []string, w io.Writer) error {
set := flag.NewFlagSet("queue", flag.ContinueOnError)
usageTo(set, w)
asJSON := set.Bool("json", false, "the queue as JSON")
if _, err := parseAround(set, args); err != nil {
return err
@@ -72,10 +72,10 @@ func queueCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
fmt.Println(string(body))
fmt.Fprintln(w, string(body))
return nil
}
fmt.Print(queueText(q, time.Now()))
fmt.Fprint(w, queueText(q, time.Now()))
return nil
}
+185
View File
@@ -0,0 +1,185 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// D15: no client of the bus reconnects in a loop (novox/hq issue 327).
//
// The server's connection total was the one number that would show a client reconnecting, and every verb
// the controller served opened and closed a connection of its own, hundreds an hour, so a loop was
// invisible in it. The bus's own module reads the server's record of closed connections
// (`nats_closed_connections`); this asks it every run, and says each user whose connections were dropped —
// closed by anything but the client itself: a read or write error, a stale connection, a slow consumer, a
// refused login — more often than reconnectBound in the last hour.
const (
probeReconnectsID = "D15"
kindBusReconnects = "bus-reconnects"
// reconnectBound is how many dropped connections in an hour one user may have before it is said:
// a client that loses its connection every five minutes. Provisional.
reconnectBound = 12
// busModule is the module that is the bus, and closedTool its tool that reads closed connections.
busModule = "nats"
closedTool = "nats_closed_connections"
closedAsk = 20 * time.Second
)
// closedConnections is what the bus's module answers.
type closedConnections struct {
Hours float64 `json:"hours"`
Reaches bool `json:"reaches"`
Users []struct {
User string `json:"user"`
Closed int `json:"closed"`
Dropped int `json:"dropped"`
DroppedPerHour float64 `json:"dropped_per_hour"`
Names []struct {
Name string `json:"name"`
Closed int `json:"closed"`
Dropped int `json:"dropped"`
Reasons map[string]int `json:"reasons"`
} `json:"names"`
} `json:"users"`
}
// probeReconnects is D15.
func probeReconnects(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
if d.js == nil {
return nil, fmt.Errorf("no bus to ask the bus's module over")
}
on, err := d.open.inventory.Running(ctx, busModule)
if err != nil {
return nil, err
}
if len(on) == 0 {
return nil, nil // no bus module assigned: a mesh whose bus is not the mesh's module
}
return reconnectsOn(ctx, d.js.Conn(), on[0])
}
// reconnectsOn asks the bus module on its machine and says who reconnects in a loop.
func reconnectsOn(ctx context.Context, conn *nats.Conn, node string) ([]conditions.Observation, error) {
read, err := askClosed(ctx, conn, node)
if isNothingServes(err) {
// A bus module older than its tool has nothing it can say, which is not a failure of the probe.
return nil, nil
}
if err != nil {
return nil, err
}
return reconnecting(read), nil
}
// askClosed asks the bus's module, on its machine, who closed connections in the last hour.
func askClosed(ctx context.Context, conn *nats.Conn, node string) (closedConnections, error) {
var read closedConnections
answer, err := link.AskModuleToolOn(ctx, conn, busModule, closedTool, node, map[string]any{"hours": 1}, closedAsk)
if err != nil {
return read, err
}
if answer.Error != "" {
return read, fmt.Errorf("%s on %s answered %s with an error: %s", busModule, node, closedTool, answer.Error)
}
if err := unmarshalAnswer(answer, &read); err != nil {
return read, fmt.Errorf("%s on %s answered %s with something unreadable: %w", busModule, node, closedTool, err)
}
return read, nil
}
// reconnecting is one observation per user whose connections were dropped more than reconnectBound times
// an hour.
func reconnecting(read closedConnections) []conditions.Observation {
hours := read.Hours
if hours <= 0 {
hours = 1
}
var out []conditions.Observation
for _, u := range read.Users {
if u.User == "" || strings.HasPrefix(u.User, "(") {
// Refused before it logged in: no client of the mesh's, so nobody's reconnect loop. A login
// refused again and again is a question of its own, not this probe's.
continue
}
perHour := float64(u.Dropped) / hours
if perHour <= reconnectBound {
continue
}
reasons := map[string]int{}
var names []string
for _, n := range u.Names {
if n.Dropped == 0 {
continue
}
names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped))
for r, c := range n.Reasons {
if r != "Client Closed" {
reasons[r] += c
}
}
}
var why []string
for r, c := range reasons {
why = append(why, fmt.Sprintf("%s ×%d", r, c))
}
sort.Strings(why)
partial := ""
if !read.Reaches {
partial = " (at least: the server's record of closed connections does not reach back the whole hour)"
}
who, machine := busUserWords(u.User)
out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: u.User, Kind: kindBusReconnects,
Machine: machine, Severity: conditions.Warning,
Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a "+
"client reconnecting in a loop", u.User, u.Dropped, partial, reconnectBound),
Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped,
u.Closed, hours, strings.Join(names, ", "), strings.Join(why, ", ")),
Headline: clipWords(conditions.Capital(who)+" keeps losing the bus", 60),
Explanation: conditions.Capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour "+
"and connected again each time. While it reconnects, what it says and what it is asked waits.", who,
u.Dropped)),
Resolved: "Resolved: " + who + " stays connected"})
}
return out
}
// busUserWords is a bus user as the operator says it, and the machine it is on: `node.<machine>` the
// node-engine, `<machine>.node-tools` the tool runner, `controller` the controller, `<machine>.<module>` a
// module.
func busUserWords(user string) (string, string) {
switch {
case user == "controller":
return "the controller", ""
case strings.HasPrefix(user, "node."):
m := strings.TrimPrefix(user, "node.")
return "the node-engine on " + m, m
}
if m, module, ok := strings.Cut(user, "."); ok {
if module == "node-tools" {
return "the tool runner on " + m, m
}
return module + " on " + m, m
}
return "a client of the bus", ""
}
// clipWords is words at most n characters long, cut at a word.
func clipWords(s string, n int) string {
if len(s) <= n {
return s
}
cut := s[:n]
if i := strings.LastIndex(cut, " "); i > 0 {
cut = cut[:i]
}
return cut
}
+189
View File
@@ -0,0 +1,189 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// The serving controller's own connection serves what it does for a moment: no connection is opened,
// and closing what it was lent leaves the serving one open (novox/hq issue 327).
func TestTheServingControllerLendsItsOwnConnection(t *testing.T) {
bus := testbus.Start(t)
serving, err := broker.Dial(bus.ClientURL())
if err != nil {
t.Fatal(err)
}
defer serving.Close()
if err := serving.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
was := servingBus.Load()
servingBus.Store(serving)
defer servingBus.Store(was)
t.Setenv(broker.NATSVar, bus.ClientURL())
before, _ := bus.Varz(nil)
lent, err := aBus()
if err != nil {
t.Fatal(err)
}
lent.Close()
if !serving.Conn().IsConnected() {
t.Fatal("closing a lent connection closed the serving controller's")
}
if err := onTheBus(func(*nats.Conn) error { return nil }); err != nil {
t.Fatal(err)
}
handlers, _, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
answer, err := handlers["hand-acts"](context.Background(), json.RawMessage(`{}`))
if err != nil {
t.Fatal(err)
}
if a := answer.(verbAnswer); !a.OK || a.Answer == nil {
t.Fatalf("hand-acts answered %+v", a)
}
_, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`))
after, _ := bus.Varz(nil)
if opened := after.TotalConnections - before.TotalConnections; opened != 0 {
t.Fatalf("the serving controller opened %d connection(s) of its own", opened)
}
}
// A verb that still runs as a process of its own says which in the bus's list of connections.
func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) {
bus := testbus.Start(t)
setup, err := broker.Dial(bus.ClientURL())
if err != nil {
t.Fatal(err)
}
if err := setup.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
setup.Close()
asAProcess(t, bus.ClientURL())
handlers, _, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
// A silence acts, so it is still its own process; it dials, and finds no such condition.
_, _ = handlers["conditions"](context.Background(),
json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`))
names := closedNames(t, bus)
found := false
for _, n := range names {
found = found || n == "mesh-controller verb conditions"
}
if !found {
t.Fatalf("the verb's connection was named %q", names)
}
}
func TestEachProcessNamesItsConnectionsForWhatItIs(t *testing.T) {
for _, c := range []struct{ command, verb, want string }{
{"serve", "", "mesh-controller serving"},
{"conditions", "conditions", "mesh-controller verb conditions"},
{"delivery", "delivery-check", "mesh-controller verb delivery-check"},
{"push", "", "mesh-controller command push"},
} {
if got := connectionName(c.command, c.verb); got != c.want {
t.Errorf("%s/%s: %q", c.command, c.verb, got)
}
}
}
// D15: a user whose connections are dropped more than twelve times an hour is said, in plain words; one
// whose client closes its own short connections is not.
func TestAClientReconnectingInALoopIsSaid(t *testing.T) {
var read closedConnections
if err := json.Unmarshal([]byte(`{"hours":1,"reaches":true,"users":[
{"user":"ace.node-tools","closed":40,"dropped":40,"dropped_per_hour":40,"names":[
{"name":"ace.node-tools","closed":40,"dropped":40,"reasons":{"Stale Connection":30,"Read Error":10}}]},
{"user":"controller","closed":300,"dropped":0,"names":[
{"name":"mesh-controller verb delivery-check","closed":300,"dropped":0,"reasons":{"Client Closed":300}}]},
{"user":"(no user: refused before it logged in)","closed":90,"dropped":90,"names":[{"name":"","closed":90,
"dropped":90,"reasons":{"Authentication Failure":90}}]},
{"user":"node.anchor","closed":5,"dropped":5,"names":[{"name":"mesh-host/anchor","closed":5,"dropped":5,
"reasons":{"Read Error":5}}]}]}`), &read); err != nil {
t.Fatal(err)
}
found := reconnecting(read)
if len(found) != 1 {
t.Fatalf("%+v", found)
}
o := found[0]
if o.ID != "ace.node-tools" || o.Machine != "ace" || o.Kind != kindBusReconnects ||
o.Headline != "The tool runner on ace keeps losing the bus" || !strings.Contains(o.Said, "Stale Connection ×30") {
t.Fatalf("%+v", o)
}
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
Resolved: o.Resolved}, "ace"); !ok {
t.Fatalf("not plain: %s", why)
}
}
// The bus's module is asked on its machine, over the bus.
func TestTheBusModuleIsAskedWhoClosedConnections(t *testing.T) {
bus := testbus.Start(t)
conn, err := nats.Connect(bus.ClientURL())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
sub, err := conn.Subscribe(link.ModuleToolOn("nats", "nats_closed_connections", "anchor"), func(m *nats.Msg) {
_ = m.Respond([]byte(`{"result":{"hours":1,"reaches":true,"users":[{"user":"controller","closed":2,"dropped":0}]}}`))
})
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
read, err := askClosed(context.Background(), conn, "anchor")
if err != nil {
t.Fatal(err)
}
if len(read.Users) != 1 || read.Users[0].Closed != 2 {
t.Fatalf("%+v", read)
}
}
// A bus module older than the tool answers nothing to the question: the probe passes over it quietly.
func TestAnOlderBusModuleIsPassedOverQuietly(t *testing.T) {
bus := testbus.Start(t)
conn, err := nats.Connect(bus.ClientURL())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
found, err := reconnectsOn(context.Background(), conn, "anchor")
if err != nil || len(found) != 0 {
t.Fatalf("a bus module without the tool: %v %v", found, err)
}
}
// A verb answered in the serving controller says its flag errors in its answer, not in the controller's log.
func TestAFlagErrorIsSaidInTheAnswer(t *testing.T) {
bus := testbus.Start(t)
serving, err := broker.Dial(bus.ClientURL())
if err != nil {
t.Fatal(err)
}
defer serving.Close()
was := servingBus.Load()
servingBus.Store(serving)
defer servingBus.Store(was)
answer, read := readHere(context.Background(), []string{"hand-acts", "--bogus"})
if !read || answer.OK || !strings.Contains(answer.Output, "flag provided but not defined") {
t.Fatalf("answered %+v", answer)
}
}
+106
View File
@@ -0,0 +1,106 @@
package main
import (
"context"
"encoding/json"
"os/exec"
"path/filepath"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
// novox/hq issue 327, replayed with only what the controller had before its fix, so it can be laid over
// the older commit. On 2026-10-08 the bus's connection total rose by 5.2 a minute while the same 16
// connections stayed open, and three calls of `conditions` in one second added three: each verb ran as a
// process of the controller's own binary, which dialled the bus, logged in and left. The operator's
// channel reads `conditions` at least once a minute. A verb that only reads, served by the serving
// controller, opens no connection of its own.
func TestReplay327(t *testing.T) {
bus := testbus.Start(t)
serving, err := broker.Dial(bus.ClientURL())
if err != nil {
t.Fatal(err)
}
defer serving.Close()
if err := serving.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
keeper, err := keeperOn(context.Background(), serving.Conn())
if err != nil {
t.Fatal(err)
}
// The serving controller: its keeper and its connection, as serve sets them.
keptBefore, connBefore := conditionsFrom, handActConn
conditionsFrom, handActConn = keeper, serving.Conn()
defer func() { conditionsFrom, handActConn = keptBefore, connBefore }()
// A verb that runs as a process of its own runs this controller's binary, on this bus.
asAProcess(t, bus.ClientURL())
handlers, _, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
accepted := func() uint64 {
v, err := bus.Varz(nil)
if err != nil {
t.Fatal(err)
}
return v.TotalConnections
}
before := accepted()
for i := 0; i < 3; i++ {
answer, err := handlers["conditions"](context.Background(), json.RawMessage(`{}`))
if err != nil {
t.Fatal(err)
}
if a, ok := answer.(verbAnswer); !ok || !a.OK {
t.Fatalf("conditions answered %+v", answer)
}
}
if opened := accepted() - before; opened != 0 {
t.Fatalf("three conditions calls opened %d connection(s) to the bus; the serving controller is on it already",
opened)
}
}
// asAProcess makes a verb that runs as a process of its own run this package's binary, on the bus at url:
// built into the test's own directory, which goes with the test.
func asAProcess(t *testing.T, url string) {
t.Helper()
path := filepath.Join(t.TempDir(), "mesh-controller")
if out, err := exec.Command("go", "build", "-o", path, ".").CombinedOutput(); err != nil {
t.Fatalf("the controller could not be built to run a verb as its own process: %v: %s", err, out)
}
was := ownImage
ownImage = func() string { return path }
t.Cleanup(func() { ownImage = was })
t.Setenv(broker.NATSVar, url)
t.Setenv(broker.CertificateVar, "")
}
// closedNames are the names of the connections the bus saw closed.
func closedNames(t *testing.T, bus *server.Server) []string {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
var names []string
for time.Now().Before(deadline) {
connz, err := bus.Connz(&server.ConnzOptions{State: server.ConnClosed})
if err != nil {
t.Fatal(err)
}
names = names[:0]
for _, c := range connz.Conns {
names = append(names, c.Name)
}
if len(names) > 0 {
return names
}
time.Sleep(50 * time.Millisecond)
}
return names
}
+4 -1
View File
@@ -131,7 +131,10 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
// **Dialled the way the mesh dials it** — credential and pin — because a bare connect to a
// bus that requires TLS and a user fails at the handshake, and the check then reported a
// standing server as absent (seen live, 2026-09-28).
if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
// The serving controller's own connection answers it without a second (novox/hq issue 327).
if serving := servingBus.Load(); serving != nil && serving.Conn().IsConnected() {
state.ServerStanding = true
} else if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
state.ServerStanding = true
js.Close()
}
+97 -12
View File
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"github.com/nats-io/nats.go/micro"
"io"
"os"
"os/exec"
"slices"
@@ -28,6 +29,9 @@ import (
// answer. It also means a refusal is the same refusal in the same words, because it is the same
// output.
// verbKey carries the verb a call is for, to the process it runs.
type verbKey struct{}
// verbAnswer is what a verb answers: what the command printed, whether it succeeded, and — where the
// command speaks JSON — the same as data.
type verbAnswer struct {
@@ -761,6 +765,28 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return argv, nil
case "settings":
// Every module's preferences, their defaults and each machine's value (novox/hq ADR 0262):
// the one interface for them, so no module builds a settings tool of its own. Asked for by
// name, or by naming no module, since a layer is always some module's.
if str("module") == "" && str("values") != "" {
return nil, errors.New("settings: a module is needed to set values; name it with module")
}
if str("module") == "" && on("clear") {
return nil, errors.New("settings: a module is needed to clear a layer; name it with module")
}
if list := str("list"); list != "" || str("module") == "" {
if list != "" && list != "preferences" {
return nil, fmt.Errorf("settings lists %q only; %q is not a listing", "preferences", list)
}
argv := []string{"settings", "preferences"}
if m := str("module"); m != "" {
argv = append(argv, m)
}
if n := str("node"); n != "" {
argv = append(argv, "--node", n)
}
return argv, nil
}
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
// because a tool has no file to hand the command; the command reads either.
if err := need("module"); err != nil {
@@ -897,6 +923,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
caller = "a seat call whose caller the bus did not name"
}
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
// And which verb, so the connection it dials says so in the bus's list (novox/hq issue 327).
verb, _ := ctx.Value(verbKey{}).(string)
if verb == "" {
verb = argv[0]
}
cmd.Env = append(cmd.Env, verbVar+"="+verb)
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
// prints its warnings beside the document, and a JSON parsed from the two together parsed
@@ -905,18 +937,7 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
cmd.Stdout = &stdout
cmd.Stderr = &stderr
runErr := cmd.Run()
answer := verbAnswer{Output: stdout.String() + stderr.String(), OK: runErr == nil}
if jsonVerbs[argv[0]] && runErr == nil {
var parsed any
if json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &parsed) == nil {
answer.Answer = parsed
if overviewVerbs[argv[0]] {
// Once, as data: the same document again as text doubled an answer that already
// outgrew one message of the bus (novox/hq issue 314).
answer.Output = stderr.String() + "its answer, as data, is `answer`\n"
}
}
}
answer := answerOf(argv, stdout.Bytes(), stderr.String(), runErr == nil)
var exit *exec.ExitError
if runErr != nil && !errors.As(runErr, &exit) {
// Not the command refusing — the command not running at all, which is this process's fault.
@@ -931,6 +952,66 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
return answer, nil
}
// answerOf is what a command said, as a verb answers it: both streams as text, and standard output as data
// where the command speaks JSON.
func answerOf(argv []string, stdout []byte, stderr string, ok bool) verbAnswer {
answer := verbAnswer{Output: string(stdout) + stderr, OK: ok}
if jsonVerbs[argv[0]] && ok {
var parsed any
if json.Unmarshal(bytes.TrimSpace(stdout), &parsed) == nil {
answer.Answer = parsed
if overviewVerbs[argv[0]] {
// Once, as data: the same document again as text doubled an answer that already
// outgrew one message of the bus (novox/hq issue 314).
answer.Output = stderr + "its answer, as data, is `answer`\n"
}
}
}
return answer
}
// readHere answers a verb that only reads the bus in the serving controller itself, on its own connection
// and keeper (novox/hq issue 327): the same command, writing to the answer rather than to a process's
// output, so the answer is the one the command prints. False for any other command line, which runs as a
// command of its own. Every `conditions` call — the operator's channel reads it at least once a minute —
// was a process that dialled the bus, logged in and left.
func readHere(ctx context.Context, argv []string) (verbAnswer, bool) {
var read func(context.Context, []string, io.Writer) error
args := argv[1:]
// Each where this process holds what it reads: the serving keeper, the hand-act log's connection, the
// serving connection.
switch argv[0] {
case "conditions":
sub := "list"
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
if conditionsFrom != nil {
read = map[string]func(context.Context, []string, io.Writer) error{
"list": listConditions, "show": showCondition, "history": conditionHistory}[sub]
}
case "hand-acts":
if handActConn != nil || servingBus.Load() != nil {
read = listHandActs
}
case "queue":
if servingBus.Load() != nil {
read = listQueue
}
}
if read == nil {
return verbAnswer{}, false
}
var out bytes.Buffer
stderr := ""
err := read(ctx, args, &out)
if err != nil {
// As the command says it when it fails (main).
stderr = "mesh-controller: " + err.Error() + "\n"
}
return answerOf(argv, out.Bytes(), stderr, err == nil), true
}
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
@@ -1011,6 +1092,7 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
if err != nil {
return nil, err
}
ctx = context.WithValue(ctx, verbKey{}, verb)
if verb == "status" && statusFrom != nil {
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
return statusFrom.answer(ctx)
@@ -1019,6 +1101,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
// Whatever it did, `status` is composed again once it has.
defer statusFrom.nudge()
}
if answer, read := readHere(ctx, argv); read {
return answer, nil
}
if answersFirst(argv) {
// Before anything is sent: a push sends the bus's own machine first, and a broker
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
+49
View File
@@ -0,0 +1,49 @@
package main
import (
"flag"
"fmt"
"io"
"os"
"sync/atomic"
"github.com/novox/mesh-controller/internal/broker"
)
// The serving controller's own connection, lent to whatever it does for a moment (novox/hq issue 327).
//
// Every place that needed the bus for a moment dialled it: a verb's own process, and in the serving
// controller a watchdog tick reading whether the build seat paused, a walk's step reading readiness, a
// queue read. Each paid a connection, a TLS handshake and a login on the control node, and hundreds an
// hour hid in the server's connection total the one thing it would show: a client reconnecting in a loop.
// The serving controller is on the bus already; what it does is done on that connection.
// servingBus is the serving controller's connection, set when it starts serving; nil in every other
// process, which dials its own.
var servingBus atomic.Pointer[broker.JetStream]
// aBus is a connection for something done for a moment: the serving controller's own, lent — so its
// Close closes nothing — when this process is it, and otherwise one dialled for it, named for this
// process (broker.ConnectionName), which its Close closes.
func aBus() (*broker.JetStream, error) {
if serving := servingBus.Load(); serving != nil {
return broker.Borrow(serving), nil
}
address, err := broker.BusAddress()
if err != nil {
return nil, err
}
js, err := broker.Dial(address)
if err != nil {
return nil, fmt.Errorf("cannot reach the bus: %w", err)
}
return js, nil
}
// usageTo sends a command's flag errors and usage to where its answer goes when that is not this process's
// output: a verb answered in the serving controller says them in its answer, not in the controller's log.
func usageTo(set *flag.FlagSet, w io.Writer) {
if w != os.Stdout {
set.SetOutput(w)
}
}
@@ -0,0 +1,109 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// `settings` says each value and where it came from, and the default a layer overrides (novox/hq ADR 0262).
func TestSettingsSayWhereEachValueComesFrom(t *testing.T) {
got := describeEffective("dunst", "laptop", []catalogue.SettingSource{
{Key: "font-size", Value: float64(13), From: "laptop", Default: float64(10), HasDefault: true},
{Key: "width", Value: float64(250), From: catalogue.DefaultLayer, FromDefault: true, Default: float64(250), HasDefault: true},
})
want := "dunst on laptop, every value and where it comes from:\n" +
" font-size = 13 (laptop; the default is 10)\n" +
" width = 250 (default)\n"
if got != want {
t.Fatalf("said\n%s\nwant\n%s", got, want)
}
}
// `settings` with list "preferences" is the one listing of every module's preferences; module and node
// narrow it, and no other listing is taken.
func TestSettingsListPreferences(t *testing.T) {
for _, c := range []struct {
args map[string]any
want string
}{
{map[string]any{"list": "preferences"}, "settings preferences"},
{map[string]any{"list": "preferences", "module": "dunst"}, "settings preferences dunst"},
{map[string]any{"list": "preferences", "node": "laptop"}, "settings preferences --node laptop"},
} {
argv, err := argvFor("settings", c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%v: %v %v, want %s", c.args, argv, err, c.want)
}
}
for args, want := range map[string]map[string]any{
"a module is needed to set values": {"values": `{"width": 300}`},
"a module is needed to clear a layer": {"clear": "true"},
} {
if _, err := argvFor("settings", want); err == nil || !strings.Contains(err.Error(), args) {
t.Errorf("%v: %v, want %q", want, err, args)
}
}
if _, err := argvFor("settings", map[string]any{"list": "everything"}); err == nil {
t.Error("a listing other than preferences was taken")
}
if argv, err := argvFor("settings", map[string]any{}); err != nil || strings.Join(argv, " ") != "settings preferences" {
t.Errorf("settings naming no module is the listing: %v %v", argv, err)
}
}
func TestPreferencesSayEachMachinesValueAndItsSource(t *testing.T) {
m := catalogue.Manifest{Module: "dunst", Settings: map[string]catalogue.SettingDeclaration{
"font-size": {Kind: catalogue.KindPreference, Default: float64(10), Why: "readable at 100 DPI"},
"width": {Kind: catalogue.KindPreference, Default: float64(250), Why: "forty characters"},
}}
on := map[string][]catalogue.SettingSource{
"laptop": catalogue.Effective(m, []catalogue.Layer{{From: "laptop", Values: map[string]any{"font-size": float64(16)}}}),
"desk": catalogue.Effective(m, []catalogue.Layer{{From: catalogue.MeshWideLayer, Values: map[string]any{"width": float64(300)}}}),
}
got := describePreferences([]preferencesOf{{Manifest: m, Nodes: []string{"desk", "laptop"}, On: on}})
want := "dunst (on desk, laptop)\n" +
" font-size, default 10: readable at 100 DPI\n" +
" desk: 10 (default)\n" +
" laptop: 16 (the node)\n" +
" width, default 250: forty characters\n" +
" desk: 300 (the mesh)\n" +
" laptop: 250 (default)\n"
if got != want {
t.Fatalf("said\n%s\nwant\n%s", got, want)
}
if describePreferences(nil) != "no module declares a preference\n" {
t.Fatal("an empty listing")
}
}
// The listing over the real stores: each machine's value with its source; a machine names only the
// modules on it; a machine the mesh does not know is refused (novox/hq ADR 0262).
func TestPreferencesListedFromTheStores(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
Settings: map[string]catalogue.SettingDeclaration{
"font-size": {Kind: catalogue.KindPreference, Default: float64(10), Why: "readable at 100 DPI"},
},
Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644",
"content": "font = ${setting:font-size}\n"}}})
if _, err := assign(ctx, open, "laptop", "notes"); err != nil {
t.Fatal(err)
}
if err := open.inventory.SetSettings(ctx, "laptop", "notes", map[string]any{"font-size": float64(16)}); err != nil {
t.Fatal(err)
}
all := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences"}) })
if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (the node)") ||
!strings.Contains(all, "font-size, default 10: readable at 100 DPI") {
t.Fatalf("the listing:\n%s", all)
}
if got := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences", "--node", "anchor"}) }); got != "no module on anchor declares a preference\n" {
t.Fatalf("a machine without the module:\n%s", got)
}
if err := settingsCommand(ctx, []string{"preferences", "--node", "nowhere"}); err == nil {
t.Fatal("a machine the mesh does not know was answered")
}
}
+64
View File
@@ -0,0 +1,64 @@
package main
import (
"context"
"fmt"
"sort"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// A stored manifest with a key this controller does not know (novox/hq ADR 0262). The module is left out
// of every machine's declaration by name; this is the loud half: a condition per module until the
// controller is updated, or the module is registered again in a shape this controller reads.
const (
sourceUnknownFields = "the catalogue"
kindUnknownField = "unknown-field"
)
// unknownFieldObservations is one condition for each module of the catalogue whose stored manifest has
// a key this controller does not know.
func unknownFieldObservations(known map[string]catalogue.Manifest) []conditions.Observation {
names := make([]string, 0, len(known))
for name, m := range known {
if m.UnknownField() != "" {
names = append(names, name)
}
}
sort.Strings(names)
var out []conditions.Observation
for _, name := range names {
m := known[name]
out = append(out, conditions.Observation{
Scope: conditions.ScopeMesh, ID: name, Kind: kindUnknownField, Severity: conditions.Warning,
Resolver: conditions.ResolverOperator, Source: sourceUnknownFields,
Summary: catalogue.UnknownFieldReason(m),
Said: m.UnknownField(),
Headline: name + " is left out until the controller is updated",
Explanation: name + " uses a field this controller does not know, so it is left out of every machine it is on, and nothing of it changes there until the controller is updated.",
Needs: "update the controller, or register " + name + " again at a version this controller knows.",
Resolved: "the controller reads " + name + " again",
})
}
return out
}
// raiseUnknownFields raises those conditions and clears the ones no longer true, on the controller's
// tick. A catalogue that could not be read raises and clears nothing: "none" is not said for "could not
// tell" (ADR 0227 rule 4).
func raiseUnknownFields(ctx context.Context, inv *inventory.Inventory) []string {
if conditionsFrom == nil {
return nil
}
known, err := inv.Catalogue(ctx)
if err != nil {
return []string{fmt.Sprintf("the catalogue could not be read to say which modules it cannot read: %v", err)}
}
if err := conditionsFrom.Reconcile(ctx, sourceUnknownFields, unknownFieldObservations(known)); err != nil {
return []string{fmt.Sprintf("the modules with a field this controller does not know could not be kept as conditions: %v", err)}
}
return nil
}
@@ -0,0 +1,50 @@
package main
import (
"encoding/json"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
)
// A module whose stored manifest has a key this controller does not know is a condition, in plain
// words, until it is read again; every other module raises nothing (novox/hq ADR 0262).
func TestAModuleWithAnUnknownFieldIsACondition(t *testing.T) {
var later, now catalogue.Manifest
if err := json.Unmarshal([]byte(`{"module": "dunst", "version": "2", "settings": {}, "a-field-from-later": 1}`), &later); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(`{"module": "xorg", "version": "1"}`), &now); err != nil {
t.Fatal(err)
}
observed := unknownFieldObservations(map[string]catalogue.Manifest{"dunst": later, "xorg": now})
if len(observed) != 1 || observed[0].ID != "dunst" || observed[0].Kind != kindUnknownField {
t.Fatalf("observed: %+v", observed)
}
o := observed[0]
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
Resolved: o.Resolved, Needs: o.Needs}); !ok {
t.Fatalf("not plain: %s", why)
}
k, _ := withConditionsInMemory(t)
if err := k.Reconcile(t.Context(), sourceUnknownFields, observed); err != nil {
t.Fatal(err)
}
if _, open, _ := k.Get(t.Context(), o.Key()); !open {
t.Fatal("not raised")
}
if err := k.Reconcile(t.Context(), sourceUnknownFields, unknownFieldObservations(map[string]catalogue.Manifest{"xorg": now})); err != nil {
t.Fatal(err)
}
still, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
for _, c := range still {
if c.Key == o.Key() {
t.Fatalf("not cleared once read again: %+v", c)
}
}
}
+21 -2
View File
@@ -28,6 +28,8 @@ import (
type JetStream struct {
conn *nats.Conn
js nats.JetStreamContext
// borrowed is a connection lent by its owner (Borrow): closing it is the owner's.
borrowed bool
// Note is how this says something it decided not to fail over. Nil is silent, which is only
// right for a caller that has no way to report; the controller sets it.
Note func(string, ...any)
@@ -40,9 +42,17 @@ func (j *JetStream) note(format string, args ...any) {
}
}
// ConnectionName is what a connection this process dials says it is, in the server's list of
// connections: the controller's role and what it is doing (novox/hq issue 327). Every connection was
// named `mesh-controller` — the serving controller's, each verb's own process, the build agents' — so the
// server's list could not say which was which. The process sets it once, at its start; an option a
// caller passes to Dial names one connection otherwise.
var ConnectionName = "mesh-controller"
// Dial connects and returns the controller's JetStream handle.
func Dial(url string, opts ...nats.Option) (*JetStream, error) {
opts = append(opts, nats.Name("mesh-controller"), nats.Timeout(10*time.Second))
// The name first, so a name the caller gives is the one that stands.
opts = append([]nats.Option{nats.Name(ConnectionName), nats.Timeout(10 * time.Second)}, opts...)
// **Pinned, not named.** The bus presents the mesh's own certificate, which names nothing a
// public verifier would accept (design 25 §4: a host pins the server's exact certificate and
// checks nothing else, and so does this). Without this, the first connection failed with
@@ -130,11 +140,20 @@ func (j *JetStream) Conn() *nats.Conn { return j.conn }
func (j *JetStream) Context() nats.JetStreamContext { return j.js }
func (j *JetStream) Close() {
if j.conn != nil {
if j.conn != nil && !j.borrowed {
j.conn.Close()
}
}
// Borrow is the same connection for a caller that will close what it was handed when it is done: its
// Close closes nothing, and the connection stays its owner's (novox/hq issue 327). How the serving
// controller lends its own connection to work that would otherwise dial one of its own.
func Borrow(j *JetStream) *JetStream {
lent := *j
lent.borrowed = true
return &lent
}
// EnsureStream creates the stream if it is absent and brings it to match if it is present.
//
// **Idempotent, because the controller asserts on every start** rather than creating once at
+31
View File
@@ -3,6 +3,8 @@ package broker
import (
"testing"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
@@ -66,3 +68,32 @@ func TestAgainstARealServer(t *testing.T) {
}
})
}
// A connection says what it is in the server's list (novox/hq issue 327): the process's name, unless the
// caller names this one; and a lent connection's Close leaves its owner's open.
func TestAConnectionIsNamedAndALentOneIsNotClosed(t *testing.T) {
url := testbus.URL(t)
was := ConnectionName
ConnectionName = "mesh-controller verb conditions"
defer func() { ConnectionName = was }()
named, err := Dial(url)
if err != nil {
t.Fatal(err)
}
defer named.Close()
if got := named.Conn().Opts.Name; got != "mesh-controller verb conditions" {
t.Errorf("named %q", got)
}
lease, err := Dial(url, nats.Name("mesh-controller serving lease"))
if err != nil {
t.Fatal(err)
}
defer lease.Close()
if got := lease.Conn().Opts.Name; got != "mesh-controller serving lease" {
t.Errorf("a name the caller gave became %q", got)
}
Borrow(named).Close()
if !named.Conn().IsConnected() {
t.Error("closing a lent connection closed its owner's")
}
}
+1 -1
View File
@@ -238,7 +238,7 @@ func (r Resolution) derivedFor(provision, as, consumer, local string, settings S
"different things and nothing would compare them (novox/hq ADR 0202)",
consumer, local, provision, m.Module, orNothing(sortedAnyKeys(names)))
}
settled, err := Settle(names, settings[m.Module])
settled, err := Settle(names, WithDefaults(m, settings[m.Module]))
if err != nil {
return nil, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
}
+1 -1
View File
@@ -159,7 +159,7 @@ func (b *DataBackup) UnmarshalJSON(raw []byte) error {
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", err)
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", typedUnknown(err))
}
*b = DataBackup{Dump: full.Dump, Into: full.Into}
return nil
+7 -3
View File
@@ -318,6 +318,10 @@ func (e *NotMadeError) Error() string {
func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string {
out := map[string]string{}
for _, m := range r.Modules {
if why := UnknownFieldReason(m); why != "" {
out[m.Module] = why
continue
}
if err := JudgeSettings(m, settings[m.Module], adopted); err != nil {
out[m.Module] = err.Error()
}
@@ -907,7 +911,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a
// definition may not carry because it is true of one installation only. Filled from
// the same layers a mergeable file takes, and refused when no layer set it.
if err := settingInto(copied, with.Settings[m.Module], m.Module); err != nil {
if err := settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module); err != nil {
return nil, err
}
// **Placed before anything reads a path.** A pathless directory receives the path
@@ -1495,7 +1499,7 @@ func (r Resolution) composed(m Manifest, to string, raw map[string]any, layers [
// Overridden, not merged: a setting changes a key the contribution declares and adds none.
// The provider reads the contribution as a contract, and a setting made for one of this
// module's files is no part of it (novox/hq 04-ISSUES/173).
values, err := overridden(raw, layers, what)
values, err := overridden(raw, WithDefaults(m, layers), what)
if err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
@@ -1926,7 +1930,7 @@ func (r Resolution) servedOnThisMachine(provision string, with Rendering) (map[s
// one, so keep looking rather than concluding from the first.
continue
}
settled, err := Settle(serves, with.Settings[m.Module])
settled, err := Settle(serves, WithDefaults(m, with.Settings[m.Module]))
if err != nil {
return nil, false, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
}
+4 -1
View File
@@ -153,7 +153,10 @@ func walk(node any, at string, meant map[string]bool, visit func(at, value strin
}
sort.Strings(keys)
for _, k := range keys {
if prose[k] || k == NamesOnPurpose || (at == "" && k == "module") {
// Prose is a string a person reads. A key that is called `why` or `description` and holds
// anything else — a setting of that name, whose default the mesh writes — is walked like any
// other (novox/hq ADR 0262).
if _, isString := v[k].(string); (prose[k] && isString) || k == NamesOnPurpose || (at == "" && k == "module") {
continue
}
child := at + "." + k
+51 -6
View File
@@ -197,7 +197,7 @@ func (i *OfferIdentity) UnmarshalJSON(raw []byte) error {
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return fmt.Errorf("an offer's identity is false or {max, in}: %w", err)
return fmt.Errorf("an offer's identity is false or {max, in}: %w", typedUnknown(err))
}
*i = OfferIdentity{Max: full.Max, In: full.In}
return nil
@@ -316,7 +316,7 @@ func (o *Offer) UnmarshalJSON(raw []byte) error {
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach, identity, "+
"keeps-consumer-data}: %w", err)
"keeps-consumer-data}: %w", typedUnknown(err))
}
o.Name, o.Scope, o.Credential, o.Reach, o.Identity = full.Name, full.Scope, full.Credential, full.Reach, full.Identity
o.KeepsConsumerData = full.Keeps
@@ -458,6 +458,19 @@ type Manifest struct {
// (novox/hq ADR 0201). Not history — that is an event — and never a secret, sealed or not.
State []StateDeclaration `json:"state,omitempty"`
// Settings are the defaults this module gives its settings (novox/hq ADR 0262): each key a file,
// a contribution or a served fact asks for as `${setting:<key>}`, its default, and why that
// default. Only a preference has one — a font size, a width, a number of workers — and a value
// that is inherently the operator's (a domain, an identity, a secret) is declared nowhere here and
// stays refused by name until a layer sets it. The mesh's layer, then the node's, override it.
Settings map[string]SettingDeclaration `json:"settings,omitempty"`
// unknown is the first key this manifest has that this controller does not know, at any depth, when
// it was read from the store (novox/hq ADR 0262): a manifest a newer controller registered.
// ParseManifest refuses it; the store's catalogue still loads, and the module is left out of every
// machine's declaration by name until the controller is updated (LeftOut).
unknown string
// Data is every kind of data this module keeps — its own, by directory, and what it keeps for
// its consumers, by provision — each with a class the mesh protects and watches it by (novox/hq
// ADR 0233). One list: the backup holder's lines, the bindings that do not move, what an
@@ -1178,7 +1191,13 @@ func ReceivedID(requirement string) string { return "received-" + requirement }
type manifestFields Manifest
// UnmarshalJSON reads `secrets` in both of its shapes — a path, or an object of local names to
// paths (ADR 0094) — and everything else exactly as the fields declare, unknown keys refused.
// paths (ADR 0094) — and everything else exactly as the fields declare.
//
// **An unknown key is kept aside, not refused here** (novox/hq ADR 0262). Registration and the module
// check refuse it, through ParseManifest. Reading the catalogue the store already holds does not: a
// manifest registered under a newer controller carries a field an older one does not know, and a
// strict read there failed the whole catalogue, and with it every plan and every send, the moment a
// controller was rolled back. UnknownField says what was set aside.
func (m *Manifest) UnmarshalJSON(raw []byte) error {
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
@@ -1259,10 +1278,25 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
decoder := json.NewDecoder(bytes.NewReader(rest))
decoder.DisallowUnknownFields()
var fields manifestFields
unknown := ""
if err := decoder.Decode(&fields); err != nil {
return err
if asUnknownField(err) == nil {
return err
}
// Read without it where the key is at the top; where it is inside a block, the block's own
// decoder refuses it again, and the manifest keeps its name and version alone. Either way the
// module is left out of every declaration by name (LeftOut), so nothing runs on a part-read
// manifest.
unknown = err.Error()
fields = manifestFields{}
if json.Unmarshal(rest, &fields) != nil {
fields = manifestFields{}
_ = json.Unmarshal(keys["module"], &fields.Module)
_ = json.Unmarshal(keys["version"], &fields.Version)
}
}
*m = Manifest(fields)
m.unknown = unknown
if len(plain) > 0 {
m.Secrets = plain
}
@@ -1366,6 +1400,10 @@ func SecretLocal(to, local string) string {
return local
}
// UnknownField is the first key a leniently read manifest had that this controller does not know, as
// the JSON decoder words it, or "" when it had none (novox/hq ADR 0262).
func (m Manifest) UnknownField() string { return m.unknown }
func ParseManifest(raw []byte) (Manifest, error) {
var m Manifest
// Strictly. **An unknown key is refused**, which is the discipline the host's declaration
@@ -1377,7 +1415,12 @@ func ParseManifest(raw []byte) (Manifest, error) {
// checking whether something is restricted will find that it is, and be wrong.
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&m); err != nil {
err := decoder.Decode(&m)
if err == nil && m.unknown != "" {
// Kept aside by UnmarshalJSON for the stored catalogue's sake; registration refuses it.
err = errors.New(m.unknown)
}
if err != nil {
// A key that used to mean something says what it became. Refusing a renamed field with
// "unknown field" is correct and unhelpful: whoever wrote it knew what they meant, and
// the mesh knows what it is called now.
@@ -1541,6 +1584,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, EventProblems(m)...)
// And what it may call its state, and whose it may read (state.go, novox/hq ADR 0201).
problems = append(problems, StateProblems(m)...)
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262).
problems = append(problems, SettingProblems(m)...)
wellFormed := true
for _, c := range m.Claims {
if !name.MatchString(c.Name) {
@@ -2441,7 +2486,7 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error {
dec := json.NewDecoder(bytes.NewReader(body))
dec.DisallowUnknownFields()
if err := dec.Decode(&long); err != nil {
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\"}: %w", name, err)
return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\"}: %w", name, typedUnknown(err))
}
out[name] = OwnSecret{Path: long.Path, Taken: long.Taken, IssuedBy: long.IssuedBy}
}
+13 -3
View File
@@ -43,7 +43,7 @@ func TestTheServiceManagerSeatServesTheUnitVerbs(t *testing.T) {
if seat.Scope != ScopeNode {
t.Fatalf("the service manager is a role each machine has once, and the seat is %s-scoped", seat.Scope)
}
want := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "failed", "reset-failed", "wanted-by"}
want := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "failed", "reset-failed", "wanted-by", "unlink-dangling"}
var got []string
for _, v := range seat.Serves {
got = append(got, v.Name)
@@ -81,7 +81,7 @@ func TestFailedIsAnOptionalVerbOfTheServiceManager(t *testing.T) {
if err := CanHold(holder(eight[1:]...), seat); err == nil || !strings.Contains(err.Error(), "does not serve units") {
t.Fatalf("a holder missing a required verb was accepted: %v", err)
}
optional := map[string]bool{"failed": true, "reset-failed": true, "wanted-by": true}
optional := map[string]bool{"failed": true, "reset-failed": true, "wanted-by": true, "unlink-dangling": true}
for _, v := range seat.Serves {
if v.Optional != optional[v.Name] {
t.Errorf("%s optional: %v", v.Name, v.Optional)
@@ -105,10 +105,14 @@ func TestResetFailedAndWantedByAreOptionalVerbsOfTheServiceManager(t *testing.T)
if err := CanHold(holder(append(nine, "reset-failed", "wanted-by")...), seat); err != nil {
t.Fatalf("a holder serving both is refused: %v", err)
}
if err := CanHold(holder(append(nine, "reset-failed", "wanted-by", "unlink-dangling")...), seat); err != nil {
t.Fatalf("a holder serving unlink-dangling too is refused: %v", err)
}
if err := CanHold(holder(append(nine, "why-started")...), seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
t.Fatalf("a verb the seat does not promise was accepted: %v", err)
}
replaces := map[string]string{"reset-failed": "systemctl reset-failed", "wanted-by": "systemctl list-dependencies --reverse"}
replaces := map[string]string{"reset-failed": "systemctl reset-failed", "wanted-by": "systemctl list-dependencies --reverse",
"unlink-dangling": "rm ~/.config/systemd/user/*.wants/<unit>"}
for _, v := range seat.Serves {
want, ok := replaces[v.Name]
if !ok {
@@ -121,6 +125,12 @@ func TestResetFailedAndWantedByAreOptionalVerbsOfTheServiceManager(t *testing.T)
if _, has := props["unit"]; !has {
t.Errorf("%s takes no unit", v.Name)
}
if v.Name == "unlink-dangling" {
// Removing is the person's act: their reason is required, and kept with the record.
if required, _ := v.Input["required"].([]string); !slices.Contains(required, "why") {
t.Errorf("unlink-dangling does not require why: %v", v.Input["required"])
}
}
delete(replaces, v.Name)
}
if len(replaces) > 0 {
+14 -2
View File
@@ -620,8 +620,8 @@ func SeatsWithAProtocol() []Seat {
// 0177): the units on the machine in both scopes, read and acted on by name. Every verb takes an
// optional scope — "system" when absent, "user" for the operator account's own manager — so a
// caller asks for a user unit the way it asks for a system one; `failed` alone reads both managers
// when none is named. `failed`, `reset-failed` and `wanted-by` are optional (Verb.Optional) until every
// holder serves them (ADR 0246).
// when none is named. `failed`, `reset-failed`, `wanted-by` and `unlink-dangling` are optional
// (Verb.Optional) until every holder serves them (ADR 0246).
func serviceManagerVerbs() []Verb {
scoped := func(more map[string]string, required []string) map[string]any {
props := map[string]string{"scope": "\"system\" (the default) or \"user\": the operator account's own manager"}
@@ -685,6 +685,18 @@ func serviceManagerVerbs() []Verb {
Input: scoped(unit, []string{"unit"}),
Replaces: []string{"systemctl list-dependencies --reverse", "systemctl show -p WantedBy",
"ls ~/.config/systemd/user/*.wants"}},
// **A dangling enable link removed on the person's word** (novox/hq issue 332): `systemctl disable`
// leaves a link whose unit file is gone, so the manager keeps asking for the unit. Optional until its
// holders serve it (ADR 0246 step 1).
{Name: "unlink-dangling", Optional: true, Description: "Remove the dangling enable links named for one " +
"unit — links in a *.wants, *.requires or *.upholds directory of the account's or the machine's own " +
"configuration whose target does not exist, which `disable` leaves once the unit's file is gone — on " +
"the person's word: `why` is required. Each is recorded (its path, its target, the reason) before it is " +
"removed, and the manager reloaded; a link whose target exists, or one a package, a generator or the " +
"runtime placed, is left and said.",
Input: scoped(map[string]string{"unit": unit["unit"],
"why": "the person's reason for removing the links, kept in the record"}, []string{"unit", "why"}),
Replaces: []string{"rm ~/.config/systemd/user/*.wants/<unit>", "unlink /etc/systemd/system/*.wants/<unit>"}},
}
}
+217
View File
@@ -0,0 +1,217 @@
package catalogue
import (
"fmt"
"sort"
"strings"
)
// A preference has a default; the operator's own value has none (novox/hq ADR 0262, extending ADR
// 0112 and taking the default half of ADR 0164).
//
// `${setting:<key>}` was refused whenever no layer set the key, and `settings set` refused a key no
// file asked for yet. Together they meant a running module could never gain a setting: the file that
// asks for it fails to compose until somebody sets it, and nobody can set it until the file asks.
// A definition may now give a key a default, with why, when the value is a **preference** — a font
// size, a width, a number of workers: something true of the software that a machine may tune. A
// value that is inherently the operator's — a domain, a public name, an identity, a secret — gets no
// default and stays refused by name until a layer sets it, which is what ADR 0112 and ADR 0155 were
// written for.
//
// The default is the lowest layer. The mesh's layer, then the node's, override it. It fills only
// `${setting:<key>}`: a mergeable file's content already is its defaults, and laying a default over
// it would reach every mergeable file of the module (novox/hq issue 168).
// SettingDeclaration is one key's default, as the definition gives it.
type SettingDeclaration struct {
// Kind says what sort of value this is. `preference` is the only kind with a default; it is
// stated rather than assumed so a reviewer sees the claim being made.
Kind string `json:"kind"`
// Default is the value when no layer sets the key: a string, a number or a boolean.
Default any `json:"default"`
// Why this default: one sentence for whoever wonders whether to change it.
Why string `json:"why"`
}
// KindPreference is the kind of a setting that may have a default.
const KindPreference = "preference"
// DefaultLayer is what the layer of a module's own defaults is called where a value's source is said.
// Only said: the layer is recognised by Layer.Default, never by this name, which a node may also have.
const DefaultLayer = "default"
// meshWords are the settings keys the mesh reads itself; a module declares none of them.
var meshWords = map[string]bool{
PortsSetting: true, ExposeSetting: true, ReachSetting: true, EndpointsSetting: true,
PlacesSetting: true, AccessesSetting: true, NetworksSetting: true,
}
// operatorsOwn are the words that, as what a key's name is about, say its value is the operator's and
// never a preference: a default for one would be the literal ADR 0112 removed from definitions.
var operatorsOwn = map[string]bool{
"domain": true, "host": true, "hostname": true, "servername": true, "fqdn": true, "zone": true,
"realm": true, "tenant": true, "site": true, "timezone": true,
"issuer": true, "url": true, "uri": true, "webhook": true, "origin": true, "dsn": true,
"ip": true, "ipv4": true, "ipv6": true,
"email": true, "mail": true, "phone": true, "address": true,
"identity": true, "login": true, "user": true, "username": true, "account": true, "owner": true,
"uid": true, "gid": true, "puid": true, "pgid": true,
"password": true, "pass": true, "passwd": true, "passphrase": true, "secret": true, "token": true,
"key": true, "apikey": true, "bearer": true, "cert": true, "credential": true,
}
// operatorsCompounds are names of two words that are the operator's though neither word alone says so
// at the end of a key: a client's identifier, and a name the world knows a site or server by.
var operatorsCompounds = map[string]bool{
"client-id": true, "site-name": true, "server-name": true, "public-name": true, "smtp-relay": true,
}
// aboutAnAmount are first words that make a key about how many or whether, never about whom:
// `max-tokens` is a number, `show-hostname` a switch.
var aboutAnAmount = map[string]bool{
"max": true, "min": true, "num": true, "count": true, "show": true, "hide": true, "enable": true,
"disable": true, "use": true, "allow": true,
}
// operatorsWord is what in a key's name says its value is the operator's, or "". A key is about its
// last word — `url-timeout` is a timeout, `user-agent` an agent, `mail-domain` a domain — or its last two
// as one of operatorsCompounds. A plural is read as its singular.
func operatorsWord(key string) string {
words := strings.FieldsFunc(key, func(r rune) bool { return r == '-' || r == '_' || r == '.' })
if len(words) == 0 || (len(words) > 1 && aboutAnAmount[words[0]]) {
return ""
}
for i, w := range words {
if !operatorsOwn[w] && strings.HasSuffix(w, "s") && operatorsOwn[strings.TrimSuffix(w, "s")] {
words[i] = strings.TrimSuffix(w, "s")
}
}
if n := len(words); n > 1 {
if pair := words[n-2] + "-" + words[n-1]; operatorsCompounds[pair] {
return pair
}
}
if last := words[len(words)-1]; operatorsOwn[last] {
return last
}
return ""
}
// SettingProblems is every way a definition's setting defaults are wrong, in its own words.
func SettingProblems(m Manifest) []string {
if len(m.Settings) == 0 {
return nil
}
used := settingKeysUsedBy(m)
var problems []string
for _, key := range sortedSettingKeys(m.Settings) {
d := m.Settings[key]
say := func(format string, args ...any) {
problems = append(problems, fmt.Sprintf("%s's setting %q: ", m.Module, key)+fmt.Sprintf(format, args...))
}
if !settingRef.MatchString("${setting:" + key + "}") {
say("not a usable key: lower-case letters, digits, dots, dashes and underscores")
continue
}
if meshWords[key] {
say("the mesh reads %q itself, and a module gives it no default", key)
continue
}
if d.Kind != KindPreference {
say("kind is %q, and only a %q has a default (novox/hq ADR 0262)", d.Kind, KindPreference)
}
if word := operatorsWord(key); word != "" {
say("a key naming %q is the operator's value, and has no default — it is the "+
"assignment's, never the definition's (novox/hq ADR 0112, ADR 0262)", word)
}
switch v := d.Default.(type) {
case string:
if strings.TrimSpace(v) == "" {
say("an empty default is no default; give the value, or declare nothing")
}
case float64, bool:
case nil:
say("no default: a key without one is the operator's, and is not declared here")
default:
say("a default is a string, a number or a boolean, and this is %T", d.Default)
}
if strings.TrimSpace(d.Why) == "" {
say("no why: say in one sentence why this default")
}
if !used[key] {
say("nothing asks for ${setting:%s}, so the default reaches nothing", key)
}
}
return problems
}
// Defaults is the layer a module's own defaults make, or nothing when it gives none.
func Defaults(m Manifest) (Layer, bool) {
if len(m.Settings) == 0 {
return Layer{}, false
}
values := map[string]any{}
for key, d := range m.Settings {
if d.Default != nil {
values[key] = d.Default
}
}
return Layer{From: DefaultLayer, Values: values, Default: true}, len(values) > 0
}
// WithDefaults is a module's layers with its defaults under them, for filling `${setting:<key>}`.
func WithDefaults(m Manifest, layers []Layer) []Layer {
d, has := Defaults(m)
if !has {
return layers
}
return append([]Layer{d}, layers...)
}
// SettingSource is one key's effective value and the layer it came from.
type SettingSource struct {
Key string
Value any
// From is DefaultLayer, MeshWideLayer or the node's name.
From string
// FromDefault is whether the value is the module's default, whatever From reads.
FromDefault bool
// Default is the module's default, when it gives one.
Default any
HasDefault bool
}
// Effective is every key a module gives a default or a layer sets, with its value and where it came
// from: the default, then the mesh's layer, then the node's — later wins.
func Effective(m Manifest, layers []Layer) []SettingSource {
byKey := map[string]*SettingSource{}
for key, d := range m.Settings {
byKey[key] = &SettingSource{Key: key, Value: d.Default, From: DefaultLayer, FromDefault: true,
Default: d.Default, HasDefault: true}
}
for _, layer := range layers {
for key, v := range layer.Values {
s, ok := byKey[key]
if !ok {
s = &SettingSource{Key: key}
byKey[key] = s
}
s.Value, s.From, s.FromDefault = v, layer.From, layer.Default
}
}
out := make([]SettingSource, 0, len(byKey))
for _, s := range byKey {
out = append(out, *s)
}
sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
return out
}
func sortedSettingKeys(in map[string]SettingDeclaration) []string {
keys := make([]string, 0, len(in))
for k := range in {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
+334
View File
@@ -0,0 +1,334 @@
package catalogue
import (
"encoding/json"
"strings"
"testing"
)
// A preference has a default in the definition; the mesh's layer, then the node's, override it; a key
// with a default is not stray; and a value that is the operator's still has none (novox/hq ADR 0262).
func notifier() Manifest {
return Manifest{Module: "notifier",
Settings: map[string]SettingDeclaration{
"font-size": {Kind: KindPreference, Default: float64(10), Why: "readable at a scale of one"},
"width": {Kind: KindPreference, Default: float64(250), Why: "fits a title of forty characters"},
},
Resources: []map[string]any{{"id": "configuration", "type": "file", "path": "/x/notifierrc",
"content": "font = Inter ${setting:font-size}\nwidth = ${setting:width}\n"}},
}
}
func parsed(t *testing.T, m Manifest) error {
t.Helper()
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
_, err = ParseManifest(raw)
return err
}
func TestAnUnsetPreferenceTakesItsDefault(t *testing.T) {
m := notifier()
for _, layers := range [][]Layer{nil, {{From: MeshWideLayer, Values: map[string]any{}}}} {
file := map[string]any{}
for k, v := range m.Resources[0] {
file[k] = v
}
if err := settingInto(file, WithDefaults(m, layers), m.Module); err != nil {
t.Fatal(err)
}
if file["content"] != "font = Inter 10\nwidth = 250\n" {
t.Fatalf("filled as %q", file["content"])
}
}
if err := JudgeSettings(m, nil, false); err != nil {
t.Fatalf("a module whose every key has a default does not compose with no layer: %v", err)
}
}
func TestTheNodeOverTheMeshOverTheDefault(t *testing.T) {
m := notifier()
layers := []Layer{
{From: MeshWideLayer, Values: map[string]any{"width": float64(300)}},
{From: "laptop", Values: map[string]any{"font-size": float64(13), "width": float64(340)}},
}
file := map[string]any{"type": "file", "content": m.Resources[0]["content"]}
if err := settingInto(file, WithDefaults(m, layers), m.Module); err != nil {
t.Fatal(err)
}
if file["content"] != "font = Inter 13\nwidth = 340\n" {
t.Fatalf("filled as %q", file["content"])
}
file = map[string]any{"type": "file", "content": m.Resources[0]["content"]}
if err := settingInto(file, WithDefaults(m, layers[:1]), m.Module); err != nil {
t.Fatal(err)
}
if file["content"] != "font = Inter 10\nwidth = 300\n" {
t.Fatalf("the mesh's layer over the default filled as %q", file["content"])
}
}
// Composed for a machine, the way a push writes it: the default reaches the file, and the node's
// layer overrides it.
func TestAComposedMachineGetsTheDefaultAndTheNodesValue(t *testing.T) {
r := anAdoptedAnchor()
r.Modules = append(r.Modules, notifier())
with := anchorRendering(false)
composed, err := r.Compose(with)
if err != nil {
t.Fatal(err)
}
if why, left := composed.LeftOut["notifier"]; left {
t.Fatalf("the notifier was left out: %s", why)
}
if c := byID(composed.Resources)["notifier.configuration"]["content"]; c != "font = Inter 10\nwidth = 250\n" {
t.Fatalf("composed with no layer as %q", c)
}
with.Settings["notifier"] = []Layer{{From: "anchor", Values: map[string]any{"font-size": float64(13)}}}
if composed, err = r.Compose(with); err != nil {
t.Fatal(err)
}
if c := byID(composed.Resources)["notifier.configuration"]["content"]; c != "font = Inter 13\nwidth = 250\n" {
t.Fatalf("composed with the node's font size as %q", c)
}
}
// Setting a key the module gives a default is not refused as reaching nothing: it overrides the
// default, which is how a running module gains a setting with no gap between.
func TestAKeyWithADefaultIsNotStray(t *testing.T) {
m := notifier()
m.Resources[0]["content"] = "font = Inter ${setting:font-size}\nwidth = ${setting:width}\n"
stray := UnusedSettings(m, []Layer{{From: "laptop", Values: map[string]any{"font-size": float64(13), "colour": "red"}}})
joined := strings.Join(stray, "; ")
if strings.Contains(joined, `"font-size"`) || !strings.Contains(joined, `"colour"`) {
t.Fatalf("stray: %s", joined)
}
}
// A default fills ${setting:…} only: it never becomes a key of a mergeable file (issue 168).
func TestADefaultIsNotMergedIntoAJSONFile(t *testing.T) {
m := notifier()
m.Resources = append(m.Resources, map[string]any{"id": "other", "type": "file", "path": "/x/other.json",
"merge": MergeJSON, "content": `{"keep": 1}`})
out, err := ApplySettings(m.Resources[1], WithDefaults(m, nil))
if err != nil {
t.Fatal(err)
}
if strings.Contains(out["content"].(string), "font-size") {
t.Fatalf("a default reached a mergeable file: %s", out["content"])
}
}
// A value that is the operator's has no default: no layer setting it is refused by name, as before.
func TestAnOperatorsValueWithoutADefaultIsStillRefused(t *testing.T) {
m := notifier()
m.Resources[0]["content"] = "font = Inter ${setting:font-size}\nwidth = ${setting:width}\nfrom = ${setting:domain}\n"
err := JudgeSettings(m, nil, false)
if err == nil || !strings.Contains(err.Error(), "${setting:domain}") {
t.Fatalf("judged %v", err)
}
if strings.Contains(err.Error(), "font-size") {
t.Fatalf("a default was named as set today: %v", err)
}
}
func TestTheParserTakesAPreferenceAndRefusesTheRest(t *testing.T) {
if err := parsed(t, notifier()); err != nil {
t.Fatalf("a preference with a default was refused: %v", err)
}
for _, c := range []struct {
name string
key string
d SettingDeclaration
refuse string
}{
{"no kind", "font-size", SettingDeclaration{Default: float64(10), Why: "x"}, `only a "preference" has a default`},
{"another kind", "font-size", SettingDeclaration{Kind: "operator", Default: float64(10), Why: "x"}, `only a "preference"`},
{"no default", "font-size", SettingDeclaration{Kind: KindPreference, Why: "x"}, "no default"},
{"an empty default", "font-size", SettingDeclaration{Kind: KindPreference, Default: " ", Why: "x"}, "an empty default"},
{"an object", "font-size", SettingDeclaration{Kind: KindPreference, Default: map[string]any{"a": 1.0}, Why: "x"}, "a string, a number or a boolean"},
{"no why", "font-size", SettingDeclaration{Kind: KindPreference, Default: float64(10)}, "no why"},
{"the operator's", "mail-domain", SettingDeclaration{Kind: KindPreference, Default: "example.tld", Why: "x"}, "is the operator's value"},
{"a secret", "api-token", SettingDeclaration{Kind: KindPreference, Default: "x", Why: "x"}, "is the operator's value"},
{"the mesh's word", PortsSetting, SettingDeclaration{Kind: KindPreference, Default: float64(1), Why: "x"}, "the mesh reads"},
{"read by nothing", "height", SettingDeclaration{Kind: KindPreference, Default: float64(300), Why: "x"}, "reaches nothing"},
} {
m := notifier()
m.Resources[0]["content"] = m.Resources[0]["content"].(string) + "x = ${setting:" + c.key + "}\n"
if c.name == "read by nothing" {
m.Resources[0]["content"] = "font = Inter ${setting:font-size}\nwidth = ${setting:width}\n"
}
m.Settings[c.key] = c.d
err := parsed(t, m)
if err == nil || !strings.Contains(err.Error(), c.refuse) {
t.Errorf("%s: parsed %v, want %q", c.name, err, c.refuse)
}
}
}
func TestEveryValueSaysWhereItCameFrom(t *testing.T) {
m := notifier()
m.Resources[0]["content"] = m.Resources[0]["content"].(string) + "x = ${setting:position}\n"
got := Effective(m, []Layer{
{From: MeshWideLayer, Values: map[string]any{"width": float64(300), "position": "top-right"}},
{From: "laptop", Values: map[string]any{"font-size": float64(13)}},
})
want := map[string]string{"font-size": "laptop", "position": MeshWideLayer, "width": MeshWideLayer}
if len(got) != 3 {
t.Fatalf("effective: %+v", got)
}
for _, s := range got {
if s.From != want[s.Key] {
t.Errorf("%s from %q, want %q", s.Key, s.From, want[s.Key])
}
}
if got[0].Key != "font-size" || got[0].Value != float64(13) || got[0].Default != float64(10) {
t.Fatalf("font-size: %+v", got[0])
}
only := Effective(m, nil)
if only[0].From != DefaultLayer || only[0].Value != float64(10) {
t.Fatalf("with no layer: %+v", only)
}
}
// A node may be called `default`. Its layer is a node's like any other: it overrides the module's
// default, it merges into a mergeable file, and it is named among what is set.
func TestANodeCalledDefaultIsANodesLayer(t *testing.T) {
m := notifier()
node := []Layer{{From: DefaultLayer, Values: map[string]any{"font-size": float64(13)}}}
file := map[string]any{"type": "file", "content": m.Resources[0]["content"]}
if err := settingInto(file, WithDefaults(m, node), m.Module); err != nil {
t.Fatal(err)
}
if file["content"] != "font = Inter 13\nwidth = 250\n" {
t.Fatalf("the node called default was dropped: %q", file["content"])
}
out, err := ApplySettings(map[string]any{"id": "j", "type": "file", "merge": MergeJSON, "content": `{}`}, node)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(out["content"].(string), `"font-size": 13`) {
t.Fatalf("the node called default did not merge: %s", out["content"])
}
if got := Effective(m, node); got[0].FromDefault || got[0].Value != float64(13) {
t.Fatalf("the node's value read as the default: %+v", got[0])
}
}
// The operator's own value is told by what a key's name is about: its last word, or its last two as
// a known compound; a plural as its singular; and never a number or a switch.
func TestAKeyIsTheOperatorsByWhatItIsAbout(t *testing.T) {
for _, key := range []string{"max-tokens", "show-hostname", "ghost-opacity", "users-per-page", "mailbox-size",
"font-size", "width", "keyboard-delay", "ipv6-preferred", "client-width", "user-agent", "url-timeout",
"site-title", "cert-renewal-days", "name", "font-name"} {
if w := operatorsWord(key); w != "" {
t.Errorf("%s read as the operator's (%s)", key, w)
}
}
for key, word := range map[string]string{"mail-domain": "domain", "api-key": "key", "admin-password": "password",
"oauth-client-id": "client-id", "db-dsn": "dsn", "public-ip": "ip", "puid": "puid", "dns-zone": "zone",
"webhook": "webhook", "notify_phone": "phone", "cors.origin": "origin", "smtp-pass": "pass",
"backup-passphrase": "passphrase", "data-owner": "owner", "fqdn": "fqdn", "tenant": "tenant", "host": "host",
"allowed-hosts": "host", "admin-emails": "email", "tokens": "token", "hostname": "hostname",
"apikey": "apikey", "servername": "servername", "tls-cert": "cert", "site": "site", "timezone": "timezone",
"bearer": "bearer", "bind-ipv4": "ipv4", "listen-ipv6": "ipv6", "site-name": "site-name",
"server-name": "server-name", "public-name": "public-name", "smtp-relay": "smtp-relay"} {
if w := operatorsWord(key); w != word {
t.Errorf("%s: read %q, want %q", key, w, word)
}
}
}
// A default is a value the mesh writes, so the installation check reads it, even under a setting
// called `description` or `why`; a setting's own why is prose and is not read.
func TestTheInstallationCheckReadsADefault(t *testing.T) {
m := notifier()
m.Settings["relay"] = SettingDeclaration{Kind: KindPreference, Default: "relay.acme.be", Why: "as at relay.acme.be"}
m.Settings["description"] = SettingDeclaration{Kind: KindPreference, Default: "notes.acme.be", Why: "x"}
problems := strings.Join(InstallationProblems(m), "; ")
for _, want := range []string{"relay.acme.be at settings.relay.default", "notes.acme.be at settings.description.default"} {
if !strings.Contains(problems, want) {
t.Errorf("not reported: %q in %s", want, problems)
}
}
if strings.Contains(problems, "settings.relay.why") {
t.Errorf("a why was read as a value: %s", problems)
}
}
// A default fills ${setting:…} in what a module contributes and serves, and never replaces a value a
// contribution states itself.
func TestADefaultFillsAContributionAndAServedFactButNotALiteral(t *testing.T) {
m := notifier()
m.Settings["site-title"] = SettingDeclaration{Kind: KindPreference, Default: "Notes", Why: "x"}
contribution := map[string]any{"title": "${setting:site-title}", "width": "fixed", "port": float64(8080)}
got, err := overridden(contribution, WithDefaults(m, nil), "a contribution")
if err != nil {
t.Fatal(err)
}
if got["title"] != "Notes" || got["width"] != "fixed" {
t.Fatalf("contribution: %v", got)
}
got, err = overridden(contribution, WithDefaults(m, []Layer{{From: "laptop", Values: map[string]any{"width": "wide"}}}), "a contribution")
if err != nil || got["width"] != "wide" {
t.Fatalf("a node's value did not override a contribution's own key: %v %v", got, err)
}
served, err := Settle(map[string]any{"name": "${setting:site-title}"}, WithDefaults(m, nil))
if err != nil || served["name"] != "Notes" {
t.Fatalf("served: %v %v", served, err)
}
if _, err := Settle(map[string]any{"name": "${setting:site-title}"}, nil); err == nil {
t.Fatal("a served fact without the defaults was filled")
}
}
// A stored manifest with a key this controller does not know, at the top or inside any block, is read
// and its module is left out of every declaration by name; the rest of the catalogue is read; the module
// check refuses it. One case per block whose decoder wraps the decoder's words in its own.
func TestAStoredManifestWithAnUnknownKeyIsLeftOutAndRegistrationRefusesIt(t *testing.T) {
for where, raw := range map[string]string{
"the top": `{"module": "later", "version": "2", "a-field-from-later": {"x": 1}}`,
"a state": `{"module": "later", "version": "2", "state": [{"name": "s", "a-field-from-later": 1}]}`,
"a provided name": `{"module": "later", "version": "2", "provides": [{"name": "p", "a-field-from-later": 1}]}`,
"an offer's identity": `{"module": "later", "version": "2", "provides": [{"name": "p", "identity": {"in": "x", "a-field-from-later": 1}}]}`,
"a backup": `{"module": "later", "version": "2", "data": {"own": [{"id": "d", "path": "${dir:d}", "class": "valuable", "backup": {"dump": "x", "into": "y", "a-field-from-later": 1}}]}}`,
"an own secret": `{"module": "later", "version": "2", "own-secrets": {"s": {"path": "/x", "a-field-from-later": 1}}}`,
"a seat's verb": `{"module": "later", "version": "2", "seats": [{"name": "later-seat", "serves": [{"name": "v", "a-field-from-later": 1}]}]}`,
} {
var m Manifest
if err := json.Unmarshal([]byte(raw), &m); err != nil {
t.Errorf("%s: the stored manifest was not read: %v", where, err)
continue
}
if m.Module != "later" || m.Version != "2" || !strings.Contains(m.UnknownField(), `"a-field-from-later"`) {
t.Errorf("%s: read as %q %q, unknown %q", where, m.Module, m.Version, m.UnknownField())
continue
}
if where != "the top" && !strings.Contains(m.UnknownField(), ": json: unknown field") {
t.Errorf("%s: the block's decoder did not say it: %q", where, m.UnknownField())
}
left := Resolution{Node: "laptop", Modules: []Manifest{m, notifier()}}.LeftOut(nil, false)
if why := left["later"]; !strings.Contains(why, "uses a field this controller does not know") ||
!strings.Contains(why, "a-field-from-later") {
t.Errorf("%s: not left out by name: %v", where, left)
}
if _, notifierLeft := left["notifier"]; notifierLeft {
t.Errorf("%s: another module was left out with it: %v", where, left)
}
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "a-field-from-later") {
t.Errorf("%s: the module check took it: %v", where, err)
}
}
var known Manifest
if err := json.Unmarshal([]byte(`{"module": "now", "state": [{"name": "s"}]}`), &known); err != nil || known.UnknownField() != "" {
t.Fatalf("a known manifest: %v %q", err, known.UnknownField())
}
// A malformed manifest is still refused: only an unknown key is read past.
var bad Manifest
if err := json.Unmarshal([]byte(`{"module": "bad", "state": [{"name": 3}]}`), &bad); err == nil {
t.Fatal("a malformed stored manifest was read")
}
}
+8 -3
View File
@@ -40,8 +40,9 @@ func settingsUsed(content string) []string {
// settingInto fills a file's ${setting:…} placeholders from the layers over a module.
//
// The last layer setting a key wins, which is the node's over the mesh's — the same order settle
// applies to a mergeable file. A value that is not a string is written the way a program would read
// The last layer setting a key wins, which is the node's over the mesh's over the module's own
// default (novox/hq ADR 0262) — the same order settle applies to a mergeable file. The caller lays
// the defaults under the layers with WithDefaults. A value that is not a string is written the way a program would read
// it (a number without a trailing .000000, a boolean as true/false).
func settingInto(resource map[string]any, layers []Layer, module string) error {
if fmt.Sprint(resource["type"]) != "file" {
@@ -56,7 +57,8 @@ func settingInto(resource map[string]any, layers []Layer, module string) error {
if !set {
return fmt.Errorf(
"%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
"value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+
"preference has a default in the definition (ADR 0262): "+
"`settings set %s <file>` with {%q: …}%s",
module, key, key, module, key, orNoSettings(layers))
}
@@ -80,6 +82,9 @@ func settingValue(layers []Layer, key string) (any, bool) {
func orNoSettings(layers []Layer) string {
var keys []string
for _, l := range layers {
if l.Default {
continue
}
for k := range l.Values {
keys = append(keys, k)
}
+16 -2
View File
@@ -30,6 +30,9 @@ type Layer struct {
// Where these came from, for saying which layer set a value.
From string
Values map[string]any
// Default marks the layer a module's own defaults make (novox/hq ADR 0262). Marked rather than
// recognised by From, which is a node's name for a node's layer, and a node may be called anything.
Default bool
}
// ApplySettings produces a resource's final content from the module's own and the layers over it.
@@ -114,7 +117,7 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an
values[key] = value
}
}
kept = append(kept, Layer{From: layer.From, Values: values})
kept = append(kept, Layer{From: layer.From, Values: values, Default: layer.Default})
}
merged, err := settle(base, kept, nil, what)
if err != nil {
@@ -151,6 +154,12 @@ func settle(base map[string]any, layers []Layer, protected map[string]bool, what
map[string]any, error) {
merged := deepCopy(base)
for _, layer := range layers {
if layer.Default {
// A module's own defaults fill ${setting:<key>} only (novox/hq ADR 0262). A mergeable
// file's content already is its defaults, and a contribution or served fact declares its
// own; laid on here, a default would reach every mergeable file of its module (issue 168).
continue
}
for key, value := range layer.Values {
if key == PortsSetting {
// Where the machine puts a port is the mesh's to apply, not a value for a file or
@@ -224,6 +233,11 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
}
lands := settingKeysUsedBy(m)
// A key the module gives a default reaches what asks for it (novox/hq ADR 0262): setting it
// overrides the default, which is how a running module gains a setting without a gap between.
for key := range m.Settings {
lands[key] = true
}
for _, values := range m.Contributes {
for key := range values {
lands[key] = true
@@ -420,7 +434,7 @@ func JudgeSettings(m Manifest, layers []Layer, adopted bool) error {
for k, v := range settled {
copied[k] = v
}
if err := settingInto(copied, layers, m.Module); err != nil {
if err := settingInto(copied, WithDefaults(m, layers), m.Module); err != nil {
return err
}
}
+1 -1
View File
@@ -49,7 +49,7 @@ func (s *StateDeclaration) UnmarshalJSON(raw []byte) error {
dec := json.NewDecoder(bytes.NewReader(trimmed))
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return fmt.Errorf("a state is either a name or {name, history, ttl-seconds, per-machine}: %w", err)
return fmt.Errorf("a state is either a name or {name, history, ttl-seconds, per-machine}: %w", typedUnknown(err))
}
*s = StateDeclaration(full)
return nil
+58
View File
@@ -0,0 +1,58 @@
package catalogue
import (
"errors"
"regexp"
)
// A key this controller does not know, in a manifest (novox/hq ADR 0262).
//
// Registration refuses it, as it always has. A manifest the store already holds was registered by a newer
// controller, and is read by this one after a rollback: refusing it there failed the whole catalogue, and
// with it every plan and every send. Dropping the key silently would be worse — a module running without
// something its manifest says. So the manifest is read, the module is left out of every machine's
// declaration by name, and the controller raises a condition until it is updated.
// UnknownFieldError is a key a manifest has that this controller does not know, wherever it is: at the
// top of the manifest or inside a block (a state, an offer, a data item's backup, an own secret, a verb).
// Typed, because the blocks' decoders wrap the decoder's words in their own.
type UnknownFieldError struct {
Field string
err error
}
func (e *UnknownFieldError) Error() string { return e.err.Error() }
func (e *UnknownFieldError) Unwrap() error { return e.err }
// unknownFieldText is how the JSON decoder words a key its target does not have.
var unknownFieldText = regexp.MustCompile(`^json: unknown field "([^"]*)"$`)
// typedUnknown is err as an UnknownFieldError when it is the decoder refusing an unknown key, else err.
func typedUnknown(err error) error {
if err == nil {
return nil
}
if m := unknownFieldText.FindStringSubmatch(err.Error()); m != nil {
return &UnknownFieldError{Field: m[1], err: err}
}
return err
}
// asUnknownField is the unknown key err is about, at any depth, or nil.
func asUnknownField(err error) *UnknownFieldError {
var u *UnknownFieldError
if errors.As(typedUnknown(err), &u) {
return u
}
return nil
}
// UnknownFieldReason is why a module whose stored manifest has a key this controller does not know is
// left out of a machine's declaration, or "" when it has none.
func UnknownFieldReason(m Manifest) string {
if m.unknown == "" {
return ""
}
return m.Module + " uses a field this controller does not know (" + m.unknown + "); it is left out " +
"until the controller is updated (novox/hq ADR 0262)"
}
+9 -4
View File
@@ -56,7 +56,7 @@ func (v *Verb) UnmarshalJSON(raw []byte) error {
decoder := json.NewDecoder(bytes.NewReader(trimmed))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&p); err != nil {
return fmt.Errorf("a served verb is a name or {name, description, input, output}: %w", err)
return fmt.Errorf("a served verb is a name or {name, description, input, output}: %w", typedUnknown(err))
}
if p.Name == "" {
return fmt.Errorf("a served verb has no name: %s", trimmed)
@@ -249,19 +249,24 @@ var ControllerVerbs = []Verb{
}, nil, "adopted"),
Replaces: []string{"mesh-controller token issue", "wg set"}},
{Name: "settings", Description: "Read or set what an assignment is configured with: a module's settings for the whole " +
"mesh, or for one machine. Without values or clear, answers the layer as it stands — read it before setting it; " +
"mesh, or for one machine. Without values or clear, answers the layer as it stands — read it before setting it — " +
"then every value the module gives a default or a layer sets, with where it comes from: the module's default, " +
"the mesh, or the machine (novox/hq ADR 0262); with list \"preferences\", or with no module, every " +
"module's preferences and each machine's value; " +
"with history, the layers it replaced. Setting replaces that layer whole and answers each key it adds (+), " +
"changes (~) and removes (-); a set that would remove a key is refused unless replace says it is meant " +
"(novox/hq ADR 0217). Takes effect at the next push. With clear, removes the layer and the module is back to " +
"what its definition says; a cleared or replaced layer is kept in the history.",
Input: schema(map[string]string{
"module": "the module's name",
"module": "the module's name; with list, only that module's",
"values": "the settings as a JSON object, for set",
"node": "one machine; the whole mesh when absent",
"clear": "\"true\" to remove the layer instead of setting it; not with values",
"replace": "\"true\": with values, the set is meant to remove the keys the layer had and it does not name",
"history": "\"true\": without values or clear, the layers this one replaced, the latest first",
}, []string{"module"}, "clear", "replace", "history")},
"list": "\"preferences\": every module's preferences — key, default and why — and the value on each " +
"machine it is assigned to with where it comes from; module and node narrow it (novox/hq ADR 0262)",
}, nil, "clear", "replace", "history")},
{Name: "command", Description: "Run one command line of the controller's own, as you would type it at its " +
"shell — `node account g14 jochen`, `node show ace`, `module list` — and answer what it printed. The " +
"generic verb beside the named ones (novox/hq ADR 0154): everything the binary can do, without a verb " +
+24
View File
@@ -5,15 +5,37 @@ import (
"encoding/json"
"errors"
"fmt"
"log"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/catalogue"
)
// noted is each stored manifest's unknown key already said, so a catalogue read on every plan says it once.
var noted sync.Map
// noteUnknown says, once per module and key, that a stored manifest has a key this controller does not
// know (novox/hq ADR 0262). A manifest registered under a newer controller is read by an older one after
// a rollback; refusing it here failed the whole catalogue. The module is left out of every machine by name
// (catalogue.LeftOut), and the controller's tick raises a condition for it.
func noteUnknown(m catalogue.Manifest) {
u := m.UnknownField()
if u == "" {
return
}
if _, said := noted.LoadOrStore(m.Module+"\x00"+u, true); said {
return
}
log.Printf("the stored manifest of %s has a key this controller does not know (%s): a newer controller "+
"registered it, and %s is left out of every machine until this controller is updated (novox/hq ADR 0262)",
m.Module, u, m.Module)
}
// ErrNoSuchModule is what the mesh says about a module it has never been told about.
var ErrNoSuchModule = errors.New("no module of that name")
@@ -259,6 +281,7 @@ func (i *Inventory) Catalogue(ctx context.Context) (map[string]catalogue.Manifes
if err := json.Unmarshal(raw, &m); err != nil {
return nil, err
}
noteUnknown(m)
out[m.Module] = m
}
return out, rows.Err()
@@ -1217,6 +1240,7 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
if err := json.Unmarshal(raw, &m); err != nil {
return nil, err
}
noteUnknown(m)
entry := Entry{Manifest: m, Source: source, On: on}
if source.Repository == providedBy {
// It came with the control plane. Not a repository, and showing it as one would have
+34
View File
@@ -0,0 +1,34 @@
package inventory
import (
"strings"
"testing"
)
// A manifest a newer controller registered, with a key this one does not know, does not stop the
// catalogue from loading: it is read, marked, and every other module is read as before (novox/hq ADR 0262).
func TestTheCatalogueLoadsAroundAManifestWithAnUnknownKey(t *testing.T) {
inv := fresh(t)
if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil), Source{}); err != nil {
t.Fatal(err)
}
if _, err := inv.store.Pool().Exec(t.Context(),
`insert into module (name, manifest) values ($1, $2)`, "later",
`{"module": "later", "version": "2", "state": [{"name": "s", "a-field-from-later": 1}]}`); err != nil {
t.Fatal(err)
}
known, err := inv.Catalogue(t.Context())
if err != nil {
t.Fatalf("one manifest with an unknown key failed the whole catalogue: %v", err)
}
if known["thing"].Module != "thing" || known["thing"].UnknownField() != "" {
t.Fatalf("the other module: %+v", known["thing"])
}
if !strings.Contains(known["later"].UnknownField(), "a-field-from-later") {
t.Fatalf("the newer manifest is not marked: %q", known["later"].UnknownField())
}
entries, err := inv.Catalogued(t.Context())
if err != nil || len(entries) < 2 {
t.Fatalf("the listing: %v %d", err, len(entries))
}
}
+6
View File
@@ -47,6 +47,12 @@ func BuildsOverNATSOn(address, seat string) (Builders, error) {
return &natsBuilds{js: js, owned: true, seat: seat}, nil
}
// BuildsOn is the asking side on a connection the caller holds, which Close leaves open: the serving
// controller asks on its own rather than dialling one per build (novox/hq issue 327).
func BuildsOn(js *broker.JetStream, seat string) Builders {
return &natsBuilds{js: js, seat: seat}
}
// role is the seat asked: what the asker was made for, or the current build role for one made
// without saying (a test building the struct by hand).
func (b *natsBuilds) role() string {
+14
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"log"
"regexp"
"runtime/debug"
"sort"
"strconv"
"strings"
@@ -575,6 +576,19 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply
done := make(chan outcome, 1)
go func() {
var body []byte
// **A handler that panics is an answer, not a dead controller.** A verb answered in the serving
// process (novox/hq issue 327: conditions, hand-acts, queue, dead-letters) runs on this goroutine,
// where a panic would take every other call and the controller with it.
defer func() {
if p := recover(); p != nil {
if logger != nil {
logger.Printf("%s.%s: call %s panicked: %v\n%s", seat, verb, c.ID, p, debug.Stack())
}
body, _ = json.Marshal(map[string]any{"error": fmt.Sprintf("%s failed inside the controller and "+
"answered nothing: %v. It is in the controller's log", verb, p)})
done <- outcome{body, true}
}
}()
result, err := handle(ctx, args)
failed := err != nil
if errors.Is(err, ErrHandingOver) {
+30
View File
@@ -11,6 +11,10 @@ import (
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// answers collects what a call answered, and fails a second answer: the bus permits one.
@@ -223,3 +227,29 @@ func TestAJoinTokenIsShownToItsCallerAndNotKept(t *testing.T) {
}
}
}
// A verb whose handler panics answers an error, and the controller serves the next call (review of
// novox/hq issue 327: read verbs are answered in the serving process now).
func TestAHandlerThatPanicsAnswersAnError(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
stop, err := OverNATS{Conn: conn}.ServeSeatTools("panicky", map[string]ToolHandler{
"boom": func(context.Context, json.RawMessage) (any, error) { panic("nil map") },
"fine": func(context.Context, json.RawMessage) (any, error) { return "ok", nil },
}, nil)
if err != nil {
t.Fatal(err)
}
defer stop()
answer, err := AskMeshSeatTool(context.Background(), conn, "panicky", "boom", map[string]any{}, 5*time.Second)
if err != nil || !strings.Contains(answer.Error, "failed inside the controller") {
t.Fatalf("a panic answered %+v (%v)", answer, err)
}
answer, err = AskMeshSeatTool(context.Background(), conn, "panicky", "fine", map[string]any{}, 5*time.Second)
if err != nil || answer.Error != "" {
t.Fatalf("the call after a panic answered %+v (%v)", answer, err)
}
}