novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 3 Releases Wiki Activity

Compare commits

merge into: novox/mesh-controller:main
Branches Tags
novox/mesh-controller:main novox/mesh-controller:feat/a-core-that-cannot-fail-silently-phase-0 novox/mesh-controller:feat/retire-the-networking-bundle novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module novox/mesh-controller:fix/the-runtime-file-is-the-runtimes novox/mesh-controller:feat/no-change-takes-effect-unseen novox/mesh-controller:feat/build-edges-are-recorded novox/mesh-controller:fix/several-contributions-collide-in-grants-v2
..
pull from: novox/mesh-controller:fix/the-runtime-file-is-the-runtimes
Branches Tags
novox/mesh-controller:feat/a-core-that-cannot-fail-silently-phase-0 novox/mesh-controller:main novox/mesh-controller:feat/retire-the-networking-bundle novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module novox/mesh-controller:fix/the-runtime-file-is-the-runtimes novox/mesh-controller:feat/no-change-takes-effect-unseen novox/mesh-controller:feat/build-edges-are-recorded novox/mesh-controller:fix/several-contributions-collide-in-grants-v2
1 Commits
main .. fix/the-runtime-file-is-the-runtimes
Author SHA1 Message Date
jochen d20a077096 The runtime's file is the runtime module's: the resolver test expects docker to write live-restore (issue 190, ADR 0196) 
The catalogue moves daemon.json's live-restore and the reload from resolv-conf to the docker
module, so no module writes another software's configuration. The test composes docker beside
the resolver modules and refuses resolv-conf writing the runtime's file.
 2026-10-05 20:42:04 +02:00
89 changed files with 668 additions and 5892 deletions
Expand all files Collapse all files
cmd/mesh-controller/addresses_test.go
-8
View File
@@ -111,14 +111,6 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
t.Fatal(err)
}
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
// asks where this machine reaches the store, as the docker module does.
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
t.Fatal(err)
}
on := map[string]bool{"anchor": true, "laptop": true}
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
cmd/mesh-controller/adopting_test.go
+1 -1
View File
@@ -85,7 +85,7 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
t.Fatal(err)
}
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
cmd/mesh-controller/build.go
-7
View File
@@ -676,13 +676,6 @@ type answers struct {
// refused, until the switch — and while there is any, the mesh is not all well: the order the
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
unheld []catalogue.Unheld
// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
// journal was the only place that said so for a day (04-ISSUES/179).
failing []inventory.ProviderStanding
// overflowing is every module whose identity overflows the bound of a provision it requires
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
// this is the one place it is said across the mesh. Not well while there is any.
overflowing []catalogue.Overflow
}
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
cmd/mesh-controller/calls_verb_test.go
-81
View File
@@ -1,81 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
// RequestTimeout) — the shortest wait of a caller the mesh ships.
const consoleWaits = 30 * time.Second
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
if link.AnswerWithin >= consoleWaits/2 {
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
}
if link.AnswerWithin >= broker.ResponseTTL {
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
}
}
// A push — named or through command — answers before it runs: it sends the machine holding the bus
// first, and the broker reloading its user list forgets the answer it was about to permit.
func TestAPushAnswersBeforeItSends(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want bool
}{
{"push", map[string]any{"node": "anchor"}, true},
{"push", map[string]any{}, true},
{"command", map[string]any{"command": "push anchor"}, true},
{"command", map[string]any{"command": "push --behind"}, true},
{"command", map[string]any{"command": "builds"}, false},
{"status", map[string]any{}, false},
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil {
t.Fatalf("%s %v: %v", c.verb, c.args, err)
}
if got := answersFirst(argv); got != c.want {
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
}
}
}
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
handlers, behind, err := seatToolHandlers()
if err != nil || len(behind) != 0 {
t.Fatalf("%v %v", behind, err)
}
calls, ok := handlers["calls"]
if !ok {
t.Fatal("calls is not served")
}
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
!strings.Contains(err.Error(), `"node"`) {
t.Errorf("calls took an argument it does not declare: %v", err)
}
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
!strings.Contains(err.Error(), "not across a restart") {
t.Errorf("an unknown call was not said plainly: %v", err)
}
got, err := calls(context.Background(), json.RawMessage(`{}`))
if err != nil {
t.Fatal(err)
}
if _, listed := got.(map[string]any)["calls"]; !listed {
t.Errorf("calls answered %v", got)
}
}
cmd/mesh-controller/check.go
+1 -21
View File
@@ -25,17 +25,7 @@ import (
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
// refused what it could not see would teach people to ignore it.
//
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
// provider's machine when a real machine's name first meets the module's.
func moduleCheck(paths []string, out io.Writer) error {
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
}
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
if len(paths) == 0 {
return errors.New("module check <manifest.json>... — one file per module; pass every " +
"manifest of a repository together so the rules between them are checked too")
@@ -84,15 +74,6 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
}
failed += len(problems)
// Between the manifests too: an identity against the bounds of the provisions it wants, which
// only the provider's manifest states.
identities := catalogue.IdentityProblems(shelf, longestMachine)
sort.Strings(identities)
for _, p := range identities {
fmt.Fprintln(out, p)
}
failed += len(identities)
var names []string
for name := range shelf {
names = append(names, name)
@@ -128,8 +109,7 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
}
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
"against the bounds of the providers given here\n", len(paths), longestMachine)
"module not given here reads as unknown\n", len(paths))
return nil
}
cmd/mesh-controller/held_back.go
-241
View File
@@ -1,241 +0,0 @@
package main
import (
"context"
"fmt"
"io"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
// issue 259, ADR 0221).
//
// A named push ends by sending every other machine whose declaration differs from what it was last
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
// out makes every machine running it differ from the merge on, and so did a module whose plan was
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
// everywhere at once.
//
// What tells the two apart is which build of each module the machine was last sent, kept with every
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
// heldBack is why a push that did not name a machine must not send it, empty when it may.
//
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
// machine was last sent — including a module the machine was never sent at all. A move is held when
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
// is not known, so a held upgrade cannot be told from anything else.
func heldBack(node string, modules []string, sent map[string]string, known bool,
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
if !known {
return []string{"which builds it was last sent is not known — it was last sent before the " +
"mesh kept them, or sent a declaration by hand"}
}
var why []string
for _, m := range modules {
now := current[m]
was, carried := sent[m]
if carried && was == now.Commit {
continue
}
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
if !now.RollOut {
why = append(why, move+", which its upgrade policy records rather than rolls out")
continue
}
if id := planStillToSend(plans, m, node); id != "" {
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
}
}
sort.Strings(why)
return why
}
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
// one holding the module that has neither finished sending it nor sent it here first, and has not
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
// per machine.
func planStillToSend(plans []inventory.Plan, module, node string) string {
for _, p := range plans {
s, holds := p.Modules[module]
if !p.Open() || !holds {
continue
}
if s == nil {
return p.ID
}
if s.SentAt != nil || s.State == "failed" {
continue
}
first := false
for _, n := range s.First {
if n == node {
first = true
}
}
if !first {
return p.ID
}
}
return ""
}
func fromBuild(was string, carried bool) string {
if !carried {
return "(never sent it) "
}
return "from " + buildName(was) + " "
}
func buildName(commit string) string {
if commit == "" {
return "a build with no source"
}
return shortCommit(commit)
}
// heldMachines reads, for each machine named, why a push that did not name it must not send it
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
// to the send, which says why.
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
out := map[string][]string{}
if len(names) == 0 {
return out, nil
}
inv := open.inventory
current, err := inv.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
plans, err := inv.OpenPlans(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
plan, _, err := planFor(ctx, open, node)
if err != nil {
continue
}
modules := make([]string, 0, len(plan.Modules))
for _, m := range plan.Modules {
modules = append(modules, m.Module)
}
sent, known, err := inv.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
out[node] = why
}
}
return out, nil
}
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
func sayHeld(w io.Writer, node string, why []string) {
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
"grant from this push among it — waits with it. `push %s` sends it\n",
node, strings.Join(why, "; "), node)
}
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
//
// `handled` is every machine already sent or already said; it is not considered again. Answers the
// machines that could not be composed, as refusals.
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
w io.Writer) ([]string, error) {
inv := open.inventory
var refusals []string
// Bounded by the node count: a node is marked handled the round it is considered and is never
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
// cascade legitimately still converging, so it is said rather than passed over in silence.
rounds := 0
for {
would, err := wouldSend(ctx, open, nodes)
if err != nil {
return refusals, err
}
behind, err := inv.Waiting(ctx, would)
if err != nil {
return refusals, err
}
var also []string
for _, m := range behind {
if !handled[m.Node] {
also = append(also, m.Node)
}
}
if len(also) == 0 {
return refusals, nil
}
if rounds++; rounds > len(nodes) {
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
"should not happen; run `push --behind` to finish\n",
rounds-1, strings.Join(also, ", "))
return refusals, nil
}
sort.Strings(also)
held, err := heldMachines(ctx, open, also)
if err != nil {
return refusals, err
}
var sending []string
for _, name := range also {
// Every candidate this round is marked handled — the sent ones so they are not
// re-listed, the held ones because they stay held, and the refused ones so a machine
// that cannot be composed does not make the loop spin on it for ever.
handled[name] = true
if why, isHeld := held[name]; isHeld {
sayHeld(w, name, why)
continue
}
sending = append(sending, name)
}
if len(sending) == 0 {
continue
}
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
// Held for this round only, and after the last round's were given back, so two pushes
// cascading into each other's machines never each wait on the other.
refused, err := sendRound(ctx, open, sending, compose, d, holder)
refusals = append(refusals, refused...)
if err != nil {
return refusals, err
}
}
}
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
// is left out of it said, and its declaration allocated.
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
return func(held context.Context, node string) (sendable, error) {
plan, settings, err := planFor(held, open, node)
if err != nil {
return sendable{}, err
}
reportUnhostable(node, plan)
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
if err == nil {
reportLeftOut(node, declared)
}
return declared, err
}
}
cmd/mesh-controller/held_back_test.go
-335
View File
@@ -1,335 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/json"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
// else that moved is still a consequence the push sends (ADR 0083).
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
current := map[string]inventory.CurrentBuild{
"resolver": {Commit: "c2c2c2c2c2"},
"agent": {Commit: "a2", RollOut: true},
"network": {},
}
modules := []string{"network", "resolver", "agent"}
sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
// A module whose policy records moved: held, naming it, both builds and why.
why := heldBack("laptop", modules, sent, true, current, nil)
if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
!strings.Contains(why[0], "upgrade policy records") {
t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
}
// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
sent["resolver"] = "c2c2c2c2c2"
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
t.Fatalf("a machine whose builds are all current was held: %v", why)
}
// A module whose policy rolls out moved, and no plan holds it: sent, as before.
sent["agent"] = "a1"
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
}
// The last send's builds are not known: held whole.
if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
!strings.Contains(why[0], "not known") {
t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
}
// A module the machine was never sent, under a recording policy: held, and said so.
delete(sent, "resolver")
sent["agent"] = "a2"
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
t.Fatalf("a module never sent under a recording policy: %v", why)
}
}
// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
// naming some other machine must not send them for it.
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
at := time.Now()
current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
sent := map[string]string{"agent": "a1"}
waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
t.Fatalf("a machine the plan has not reached was not held: %v", why)
}
// The first machine itself was sent by the plan: not held by it.
if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
t.Fatalf("the plan's first machine was held: %v", why)
}
// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
Modules: map[string]*inventory.PlanModule{"agent": s}}}
if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
t.Errorf("%s: not held: %v", what, why)
}
}
// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
for what, plans := range map[string][]inventory.Plan{
"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "failed"}}}},
"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built"}}}},
} {
if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
t.Errorf("%s: held: %v", what, why)
}
}
}
// A send records the build of each module it carried; a module left out of it keeps the build it
// was last sent, since the machine keeps that one.
func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
current, map[string]string{"a": "a1", "b": "b1"})
if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
t.Fatalf("carried %v, wanted %v", got, want)
}
// Not known before: the left-out module is not recorded at all, so it reads as never sent.
got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
t.Fatalf("carried %v, wanted %v", got, want)
}
}
// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
// reads the machine as current — and writes down which machines it declared.
type recordedDelivery struct {
inv *inventory.Inventory
declared []string
}
func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
r.declared = append(r.declared, s.node)
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
}
// aResolver is a module built from a repository, at a commit, with something on the machine that
// says which build it is.
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
t.Helper()
m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
}}
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
t.Fatal(err)
}
}
// A third machine on the private network: once it is sent, every other machine's peers change with
// it, which is a consequence a push must still send — no build moved.
func aThirdMachine(t *testing.T, open *stores) {
t.Helper()
ctx := t.Context()
record, err := open.inventory.AddNode(ctx, "spare")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
t.Fatal(err)
}
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
{"name": "systemd", "present": true}}})
if err != nil {
t.Fatal(err)
}
var profile map[string]any
if err := json.Unmarshal(reported, &profile); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
t.Fatal(err)
}
}
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := time.Now().Add(-time.Hour)
aResolver(t, open, "c1c1c1c1c1", asked)
for _, node := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
t.Fatal(err)
}
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
compose := composeForPush(open, gens)
d := &recordedDelivery{inv: inv}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
}
digestOfLaptop := func() string {
sent, err := inv.Outstanding(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
return sent
}
before := digestOfLaptop()
// The change merges; the policy is the default, record. `push anchor` sends the anchor...
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
d.declared = nil
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
t.Fatal(err)
}
// ...and its cascade leaves the laptop, saying so.
var said bytes.Buffer
d.declared = nil
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
if err != nil || len(refused) != 0 {
t.Fatalf("the cascade failed: %v %v", refused, err)
}
if len(d.declared) != 0 {
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
}
if digestOfLaptop() != before {
t.Fatal("the laptop's last send moved: it was sent the held build")
}
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
"upgrade policy records", "`push laptop` sends it"} {
if !strings.Contains(said.String(), want) {
t.Errorf("the push did not say %q:\n%s", want, said.String())
}
}
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
// is held, and that what else it is owed waits with it.
aThirdMachine(t, open)
d.declared = nil
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
t.Fatal(err)
}
d.declared = nil
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil {
t.Fatal(err)
}
if len(d.declared) != 0 || digestOfLaptop() != before {
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
}
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
t.Fatalf("the push did not say both:\n%s", said.String())
}
// A policy that rolls out: the laptop is a consequence like any other, and sent.
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
t.Fatal(err)
}
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
}
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
t.Fatalf("the new send did not record the new build: %v", builds)
}
}
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
// for another reason is sent by a push that names someone else.
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
compose := composeForPush(open, gens)
d := &recordedDelivery{inv: inv}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
t.Fatal(err)
}
// `push spare`, the machine just placed: the others' peers change with it.
aThirdMachine(t, open)
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
t.Fatal(err)
}
d.declared = nil
var said bytes.Buffer
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
}
if strings.Contains(said.String(), "was not sent") {
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
}
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
record, err := inv.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
t.Fatal(err)
}
d.declared = nil
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
t.Fatal(err)
}
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
// question.
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
}
}
cmd/mesh-controller/holdings.go
-71
View File
@@ -6,8 +6,6 @@ import (
"sort"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
@@ -92,72 +90,3 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
}
return said, nil
}
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
//
// **The holders on record, and only the sole claimant when there are none** — the same answer the
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
// listed here. A holder off the private network is left out: a resolver named at an address nothing
// answers is a lookup that waits out its timeout on every name.
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
var replicated []catalogue.Seat
for _, s := range catalogue.Seats() {
if s.Replicated && s.Scope == catalogue.ScopeMesh {
replicated = append(replicated, s)
}
}
if len(replicated) == 0 {
return nil, nil
}
recorded, err := inv.Holdings(ctx)
if err != nil {
return nil, err
}
places, err := onTheNetwork(ctx, inv, shelf)
if err != nil {
return nil, err
}
address := map[string]string{}
for _, p := range places {
address[p.Name] = p.Address
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return nil, err
}
out := map[string]map[string]string{}
for _, seat := range replicated {
var nodes []string
for _, h := range recorded {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
nodes = append(nodes, h.Node)
}
}
if len(nodes) == 0 {
var derived []string
for _, e := range entries {
for _, c := range e.Manifest.Claims {
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
derived = append(derived, e.On...)
}
}
}
if len(derived) == 1 {
nodes = derived
}
}
at := map[string]string{}
for _, n := range nodes {
if address[n] != "" {
at[overlay.InternalName(n)] = address[n]
}
}
out[seat.Name] = at
}
return out, nil
}
cmd/mesh-controller/identity_bound_test.go
-150
View File
@@ -1,150 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
// one consumer's identity.
// `module check` refuses the pull request that introduces an overflow, naming the module.
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
dir := t.TempDir()
write := func(name, body string) string {
p := filepath.Join(dir, name+".json")
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
return p
}
objects := write("objects", `{"module":"objects","version":"1",
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
resolver := write("resolver", `{"module":"resolver","version":"1",
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
var out bytes.Buffer
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
}
out.Reset()
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
if err == nil {
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
}
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
strings.Contains(out.String(), "networkmanager wants") {
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
}
}
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
// resolver provision, and a second consumer there overflows an object store's access key. The
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
// by push and by `status`, and the keyless consumer is granted with its long name.
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
Requires: []string{"wildcard-resolution"}})
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
// The consumer's machine resolves, and says which of its modules no provider will grant.
consumer, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
over := consumer.Overflowing()
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
}
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
// networkmanager, and no push to the provider could go through.
plan, settings, err := planFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
if err != nil {
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
}
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
}
grants, _, err := grantsFor(ctx, open, "anchor")
if err != nil {
t.Fatal(err)
}
var keyless bool
for _, g := range grants {
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
}
if !keyless {
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
}
var granted []string
for _, c := range declared.Received["objects"]["s3-bucket"] {
granted = append(granted, c.From)
}
if strings.Join(granted, ",") != "files" {
t.Fatalf("the object store grants %v; files and only files fit", granted)
}
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
!strings.Contains(said, "left out of anchor's grants") {
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
}
// And `status` names it, and does not call the mesh well while it stands.
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
}
if asked.well() {
t.Fatal("a mesh with a consumer left out of its grants reads as well")
}
shown := printed(t, func() error { return printStatus(asked) })
if !strings.Contains(shown, "identified too long for a provision they require") ||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
t.Fatalf("status does not say it:\n%s", shown)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
Overflowing []catalogue.Overflow `json:"overflowing"`
}
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
doc.Overflowing[0].Bound.Max != 20 {
t.Fatalf("the document does not carry it: %v\n%s", err, body)
}
}
cmd/mesh-controller/issue_204_test.go
+1 -1
View File
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
}
cancel() // the sender is going away: its context is cancelled between the send and the record
body := []byte(`{"declaration":1,"resources":[]}`)
digest, err := recordSent(ctx, inv, "anchor", body, nil)
digest, err := recordSent(ctx, inv, "anchor", body)
if err != nil {
// NodeByName on the cancelled context may itself refuse; the record must still be possible
// through the detached context, so look the node up again on a live one.
cmd/mesh-controller/main.go
+1 -2
View File
@@ -194,7 +194,6 @@ func usage() {
seats [--json] every seat this mesh defines, what it delivers, and who holds it
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
board [--listen ADDR] the same three questions, as a page that holds nothing
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
assign <node> <module>... put modules on a node, judged together (ADR 0207)
@@ -232,7 +231,7 @@ func usage() {
licence add|list|use|key model access, under the name a person calls it
licence manager <name> <node> the node that holds a refreshable licence's refresh token
licence refresh <name> mint a new access token and seal it to every holder
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
pin <node> <provision> <from-node> <module>
which provider this one gets a provision from: the module, and its node
cmd/mesh-controller/missed_merges.go
-131
View File
@@ -1,131 +0,0 @@
package main
import (
"context"
"fmt"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
//
// The forge's poll announces every merge on the events stream, and the controller acts on what its
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
// server moved the consumer past it — a fault of consumers with several filters in the server the
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
// built from that repository stayed behind and were built by hand.
//
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
// would still move something was never acted on — it is said, and acted on now.
const (
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
mergeGrace = 10 * time.Minute
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
mergeLookBack = 24 * time.Hour
// mergeCatchUpEvery is how often the stream is read back.
mergeCatchUpEvery = 5 * time.Minute
)
// merges is what reads back the forge's announcements; the link server, or a test's list.
type merges interface {
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
}
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
f := following{open}
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
entries, err := open.inventory.Catalogued(ctx)
if err != nil {
return nil, nil, err
}
read, err := open.inventory.ReadRepositories(ctx)
return entries, read, err
}
failing := ""
tick := time.NewTicker(mergeCatchUpEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
}
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
fmt.Printf(format+"\n", args...)
})
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
why := ""
if err != nil {
why = err.Error()
}
if why != failing {
if why != "" {
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
} else {
fmt.Println("merges the bus may not have handed over are looked for again")
}
failing = why
}
}
}
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
// move something is said and acted on, oldest first.
//
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
// because acting on an earlier missed merge of the same repository may have moved what a later one
// would have.
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
if err != nil {
return err
}
entries, read, err := catalogued(ctx)
if err != nil {
return err
}
for _, a := range all {
if now.Sub(a.At) < mergeGrace {
continue
}
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
continue
}
if entries, read, err = catalogued(ctx); err != nil {
return err
}
moves := wouldMove(a.SourceMoved, entries, read)
if len(moves) == 0 {
continue
}
var names []string
for _, e := range moves {
names = append(names, e.Manifest.Module)
}
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
isAre(len(names)))
if err := act(ctx, a.SourceMoved); err != nil {
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
a.Owner, a.Repo, a.Commit, err)
continue
}
if entries, read, err = catalogued(ctx); err != nil {
return err
}
}
return nil
}
cmd/mesh-controller/missed_merges_test.go
-180
View File
@@ -1,180 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// announcedList is the events stream's merges as a test gives them.
type announcedList []link.AnnouncedMerge
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
var out []link.AnnouncedMerge
for _, m := range a {
if !m.At.Before(since) {
out = append(out, m)
}
}
return out, nil
}
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
type aCatalogue struct {
entries []inventory.Entry
acted []string
fail error
}
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
return append([]inventory.Entry(nil), c.entries...), nil, nil
}
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
return func(_ context.Context, m link.SourceMoved) error {
if c.fail != nil {
return c.fail
}
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
for _, moved := range wouldMove(m, c.entries, nil) {
for i := range c.entries {
if c.entries[i].Manifest.Module == moved.Manifest.Module {
c.entries[i].Source.Head = m.Commit
c.entries[i].Source.Seen = now()
}
}
}
return nil
}
}
func at(s string) time.Time {
t, err := time.Parse(time.RFC3339, s)
if err != nil {
panic(err)
}
return t
}
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
return link.AnnouncedMerge{
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
MergedAt: mergedAt, Paths: paths,
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
At: at(onTheBus),
}
}
func built(module, repo, path, commit, seen string) inventory.Entry {
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
return e
}
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
// events stream; the bus never handed it to the controller, which acted on the merges around it and
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
// move something — so it is said and acted on, once, and only after the controller's own consumer
// has had its time with it.
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
cat := &aCatalogue{entries: []inventory.Entry{
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
// Acted on when it was announced: looked at after it was merged.
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
}}
stream := announcedList{
// Nothing the mesh holds is built from the records repository.
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
// Acted on: its module was looked at since.
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
// Never handed over.
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
"node-tools/internal/console/console.go"),
}
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
clock := at("2026-10-05T22:50:00Z")
now := func() time.Time { return clock }
pass := func() {
t.Helper()
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
}
pass()
if len(cat.acted) != 0 {
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
}
clock = at("2026-10-05T22:58:00Z")
pass()
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
}
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
t.Fatalf("the missed merge was not said as one: %q", said)
}
clock = at("2026-10-05T23:03:00Z")
pass()
if len(cat.acted) != 1 || len(said) != 1 {
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
}
}
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
// that could not be acted on is said and tried again on the next pass.
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
cat := &aCatalogue{
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
fail: errors.New("the store is restarting"),
}
stream := announcedList{
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
"modules/plex/module.json", "modules/plex/x.ts"),
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
}
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
clock := at("2026-10-05T22:00:00Z")
now := func() time.Time { return clock }
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
t.Fatalf("a failed catch-up was not said: %q", said)
}
cat.fail = nil
clock = at("2026-10-05T22:05:00Z")
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
}
}
// A merge older than the look-back is left to the operator: a controller that did not run this
// missed it, and acting on it days later would be a surprise rebuild.
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
clock := at("2026-10-05T22:00:00Z")
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
func(string, ...any) {}); err != nil {
t.Fatal(err)
}
if len(cat.acted) != 0 {
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
}
}
cmd/mesh-controller/modules.go
+3 -14
View File
@@ -59,19 +59,8 @@ func moduleCommand(ctx context.Context, args []string) error {
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
if args[0] == "check" {
set := flag.NewFlagSet("module check", flag.ContinueOnError)
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
"judge each module's identity on a machine name this many characters long")
given, err := parseAround(set, args[1:])
if err != nil {
return err
}
if *longest < 1 {
return errors.New("--longest-machine-name is a length, at least 1")
}
var paths []string
for _, a := range given {
for _, a := range args[1:] {
if info, err := os.Stat(a); err == nil && info.IsDir() {
under, err := manifestsUnder(a)
if err != nil {
@@ -82,7 +71,7 @@ func moduleCommand(ctx context.Context, args []string) error {
}
paths = append(paths, a)
}
return moduleCheckFor(paths, *longest, os.Stdout)
return moduleCheck(paths, os.Stdout)
}
open, err := openStores(ctx)
if err != nil {
@@ -672,7 +661,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
// durable subscription nobody reads.
if consumer, needed := broker.ConsumerFor(broker.Principal{
Kind: broker.KindModule, Node: node, Module: m.Module,
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
}); needed {
if busAddress == "" {
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
cmd/mesh-controller/network.go
+18 -2
View File
@@ -275,9 +275,25 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
if err != nil {
return nil, err
}
// No registry trust is composed here any more: the container runtime's module states it, told
// where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR 0222, issue 190).
g, err := overlay.From(nodes, cidr, "")
if g != nil {
// The artifact store, as this network reaches it. Found rather than configured: the
// provider is whichever module offers it, on whichever machine holds that module — and if
// nothing does yet (genesis raises the registry before the catalogue knows it), there is
// no trust to write and nothing is written (novox/hq ADR 0082).
//
// Refused rather than composed without it when the question could not be answered: a
// declaration missing the trust because a lookup failed is a machine that cannot pull,
// delivered by a push that reported success — and nothing recomposes it until the next
// push (the shape of novox/hq issues 042/048, reappearing as a race).
at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
if storeErr != nil {
return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
}
if found {
g.TrustRegistry(overlay.InternalName(at) + ":" + port)
}
}
if err != nil && len(refused) > 0 {
// The network is missing something, and some machines could not be resolved at all. Those
// are almost always the same fact: a node that does not resolve contributes nothing, so
cmd/mesh-controller/nodes.go
-13
View File
@@ -505,19 +505,6 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
fmt.Printf(" public domain %s\n", domain)
}
// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
// capabilities, because it is something not working now and they are a description.
failing, err := failingProviders(ctx, inv)
if err != nil {
return err
}
if here := failingOn(failing, name); len(here) > 0 {
fmt.Printf("\n %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
for _, line := range failingLines(here, time.Now()) {
fmt.Printf(" %s\n", line)
}
}
held, err := inv.Profile(ctx, name)
if err != nil {
return err
cmd/mesh-controller/plan.go
+19 -107
View File
@@ -397,49 +397,9 @@ func declarationWith(ctx context.Context, open *stores, node string,
if err != nil {
return sendable{}, err
}
out := sendable{Resources: composed.Resources, Adoption: adoption,
return sendable{Resources: composed.Resources, Adoption: adoption,
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld}
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
// 0221). Read only on the send path: a question about what would be sent records nothing.
if choosing == Allocating {
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return sendable{}, err
}
before, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return sendable{}, err
}
if !known {
before = nil
}
names := make([]string, 0, len(plan.Modules))
for _, m := range plan.Modules {
names = append(names, m.Module)
}
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
}
return out, nil
}
// carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
// issue 259): the module's current build for each module in it, and for a module left out of it
// (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when
// that is not known. Never nil, so a send through here always records what it knows.
func carriedBuilds(modules []string, leftOut map[string]string, current map[string]inventory.CurrentBuild,
before map[string]string) map[string]string {
out := map[string]string{}
for _, m := range modules {
if _, left := leftOut[m]; left {
if was, kept := before[m]; kept {
out[m] = was
}
continue
}
out[m] = current[m].Commit
}
return out
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
}
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
@@ -464,11 +424,6 @@ func reportLeftOut(node string, declared sendable) {
"what the machine holds for it is kept and its containers are untouched. %s\n",
node, m, declared.leftOutWhy[m])
}
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
// 0225): the machine is sent everything else, and the consumer is named.
for _, o := range declared.withheld {
fmt.Printf("%s: %s\n", node, o)
}
}
// renderingFor is everything a node's declaration is composed with, and the node's record.
@@ -476,7 +431,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
plan catalogue.Resolution, settings catalogue.SettingsBy,
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
inv := open.inventory
grants, withheld, err := grantsFor(ctx, open, node)
grants, err := grantsFor(ctx, open, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
@@ -712,13 +667,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
return catalogue.Rendering{}, inventory.Node{}, err
}
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
// lists every holder of the mesh's resolver.
replicas, err := replicatedHolders(ctx, inv, shelf)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
// before it had an address on the private network. A machine joins through the tunnel now, and
@@ -784,22 +732,15 @@ func renderingFor(ctx context.Context, open *stores, node string,
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
// 0222): the artifact store as this network reaches it, which the container runtime is told to
// trust (ADR 0082). The same address composed into every reference the mesh built.
var reach map[string]string
if artifactStore != "" {
reach = map[string]string{"mesh-artifact-store": artifactStore}
}
return catalogue.Rendering{
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines, Zones: zones, Holders: replicas,
Machines: machines, Zones: zones,
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
BusUsers: busUsers, Withheld: withheld,
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
BusUsers: busUsers,
}, record, nil
}
@@ -935,34 +876,28 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
// The mirror of what a consumer is given, and the half that makes the credential real: a password
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
// the mesh hands over something it cannot itself use.
//
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, []catalogue.Overflow, error) {
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
inv := open.inventory
issued, err := inv.SecretsFrom(ctx, node)
if err != nil {
return nil, nil, err
return nil, err
}
// Where each consumer is, so a provider that must reach back to one does not have to know how
// the mesh names machines.
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, nil, err
return nil, err
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return nil, nil, err
return nil, err
}
// What each consumer actually asked for, taken from that machine's own resolution rather than
// from a record beside it. A provider told to create a password and not what to create it for
// can do nothing with it, and the name a consumer wants is the consumer's to say.
out := make([]catalogue.Grant, 0, len(issued))
var withheld []catalogue.Overflow
for _, s := range issued {
plan, settings, err := planFor(ctx, open, s.Consumer)
switch {
@@ -975,11 +910,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
// The mesh could not be asked what they wanted, which is not the same as their wanting
// nothing — and withholding a grant on that reading takes a consumer's access away
// (novox/hq 04-ISSUES/152).
return nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
}
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
if err != nil {
return nil, nil, err
return nil, err
}
// A port in there is the consumer's software port until this. The consumer is on another
// machine, so the assignment that moved it is that machine's — fetched here rather than
@@ -987,7 +922,7 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
// this case (novox/hq 04-ISSUES/038, the cross-node half).
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
if err != nil {
return nil, nil, err
return nil, err
}
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
from := s.ConsumerModule
@@ -998,10 +933,9 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
from = ""
}
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
// this provision, as the consumer's resolution states it from the provider's offer (ADR
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
// provider silently shortened — and rather than this whole machine refused for it.
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
// remedy a short slug rather than a login a provider silently shortened.
slug := ""
for _, mm := range plan.Modules {
if mm.Module == s.ConsumerModule {
@@ -1010,32 +944,15 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
}
}
if from != "" {
bound := boundOfGrant(plan, s, node)
source := catalogue.IdentitySource(slug, s.ConsumerModule)
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
Consumer: s.Consumer, Module: s.ConsumerModule,
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
continue
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
return nil, err
}
}
out = append(out, catalogue.Grant{
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
}
return out, withheld, nil
}
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
// than to none: what the provider keeps of it is not known here.
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
for _, n := range consumer.Needs {
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
return n.Identity
}
}
return catalogue.DefaultIdentityBound
return out, nil
}
// listensLines is what a person is told about what this module would open, and why — the same
@@ -1111,11 +1028,6 @@ func planCommand(ctx context.Context, args []string) error {
left := plan.LeftOut(settings, record.Adopted)
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
}
// And which of its modules no provider will grant, because the identity overflows the bound of
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
for _, o := range plan.Overflowing() {
fmt.Printf("%s: %s\n", args[0], o)
}
// And a setting that reaches nothing — refused where it is stored, and said here for one
// stored before its definition moved from under it.
for _, m := range plan.Modules {
cmd/mesh-controller/push.go
+75 -68
View File
@@ -127,19 +127,11 @@ func serve(ctx context.Context) error {
if err := server.Follows(following{open}); err != nil {
return err
}
// And the merges the bus announced and never handed over, read back on a timer and acted on late
// rather than never (novox/hq issue 266).
go catchingUpOnMerges(ctx, open, server)
// And a catalogue that has just started, asking for what it missed. The same type answers
// both: what a build meant and what the builds were are two questions about one record.
if err := server.Answers(following{open}); err != nil {
return err
}
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
// 0224): a provider's journal must not be the only place that says so.
if err := server.Watches(standings{inv}); err != nil {
return err
}
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
// from the store's row, so what the seat declares is what is answered.
@@ -158,8 +150,6 @@ func serve(ctx context.Context) error {
if !isNATS {
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
}
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
if err != nil {
return err
@@ -227,9 +217,8 @@ func declare(ctx context.Context, args []string) error {
}
// Written down like every other send (novox/hq issue 204): a declaration a person sent by hand
// is still what the machine was last told, and status must not read it as current for the one
// the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259):
// the mesh did not compose it, so a push that does not name this machine treats it as held.
if _, err := recordSent(ctx, inv, node, raw, nil); err != nil {
// the mesh would compose.
if _, err := recordSent(ctx, inv, node, raw); err != nil {
return err
}
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
@@ -327,7 +316,7 @@ func pushCommand(ctx context.Context, args []string) error {
if len(needsOne) == 0 {
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
// must never look the same.
fmt.Println("no machine named: a push of the whole mesh, and every machine is doing what it was told — nothing sent")
fmt.Println("every machine is doing what it was told")
return nil
}
}
@@ -368,18 +357,6 @@ func pushCommand(ctx context.Context, args []string) error {
}
asked = append(asked, n.Name)
}
// **A push that named no machine says so, first.** Through the console a machine the caller
// meant to name could be lost on the way (novox/hq issue 244): the call arrived empty, ran as
// `push --behind`, and every machine behind was pushed by someone who thought they had pushed one.
// Its whole-mesh reach is the first line of the answer, with the machines it is about to send.
if len(args) == 0 {
which := "every machine"
if *behind {
which = "every machine that is behind"
}
fmt.Printf("no machine named: this is a push of the WHOLE mesh — %s (%d): %s\n",
which, len(asked), strings.Join(asked, ", "))
}
// **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's
// user list, and a module's new grants are refused by the bus until that list says them. Among
@@ -390,23 +367,6 @@ func pushCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
// **Not when its own modules are held back** (novox/hq issue 259, ADR 0221): added rather than
// named, it is sent its whole declaration, and a build its policy records or a plan has not sent
// it yet would go with the user list. Named and left, with what that costs.
saidHeld := map[string]bool{}
if holderBehind && len(args) == 1 {
held, err := heldMachines(ctx, open, []string{holder})
if err != nil {
return err
}
if why, isHeld := held[holder]; isHeld {
sayHeld(os.Stdout, holder, why)
fmt.Printf("%s holds the bus, and the user list it would carry has changed: until it is "+
"sent, the bus may refuse what this push's machines were newly granted\n", holder)
holderBehind = false
saidHeld[holder] = true
}
}
asked = brokerFirst(asked, holder, holderBehind)
// Held from composing to sending, so a converge on one of them cannot send between the two
@@ -447,11 +407,7 @@ func pushCommand(ctx context.Context, args []string) error {
return err
}
release()
told := make([]string, 0, len(sending))
for _, r := range sending {
told = append(told, r.node)
}
fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", "))
fmt.Printf("\n%d node(s) told\n", len(sending))
reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
@@ -462,21 +418,76 @@ func pushCommand(ctx context.Context, args []string) error {
//
// Compared against what each machine was last SENT, not against a before/after of this push:
// the mint usually happened at `assign` or `module issue`, before this command ran, so the
// only durable signal is "what it should be" versus "what it last received". Bounded: a
// only durable signal is "what it should be" versus "what it last received". A machine behind
// for an unrelated reason is caught here too, which is not a cost — a named push that knew a
// machine was behind and left it so would be the very silence this removes. Bounded: a
// flushed send may itself mint, so this converges over a few rounds.
//
// **Except a machine a policy or a plan holds back** (novox/hq issue 259, ADR 0221): one whose
// modules would move to a build their upgrade policy records rather than rolls out, or that an
// open plan has not sent it yet. It is named, with why, and left for a push that names it.
if len(args) == 1 {
handled := map[string]bool{args[0]: true}
for n := range saidHeld {
handled[n] = true
}
refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, os.Stdout)
refusals = append(refusals, refused...)
if err != nil {
return err
flushed := map[string]bool{args[0]: true}
// Bounded by the node count: a node is marked flushed the round it is handled and is
// never handled twice, so the loop cannot run more than len(nodes) rounds. The bound is
// a guard against a logic error, not a real limit — if it were ever hit, that is a bug
// rather than a cascade legitimately still converging, so it is said rather than passed
// over in silence, unlike the earlier fixed cap that could stop a real cascade short.
rounds := 0
for {
would, err := wouldSend(ctx, open, nodes)
if err != nil {
return err
}
behind, err := inv.Waiting(ctx, would)
if err != nil {
return err
}
var also []string
for _, m := range behind {
if !flushed[m.Node] {
also = append(also, m.Node)
}
}
if len(also) == 0 {
break
}
if rounds++; rounds > len(nodes) {
fmt.Printf("\nstopped cascading after %d rounds with %s still behind — this "+
"should not happen; run `push --behind` to finish\n",
rounds-1, strings.Join(also, ", "))
break
}
sort.Strings(also)
fmt.Printf("\nthis push left %s behind — a provision granted from there, or a "+
"declaration since changed; sending it too\n", strings.Join(also, ", "))
// Tolerantly, exactly as the named send above: a machine that cannot be composed is
// collected as a refusal and reported at the end, and the others are still sent
// (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is
// all-or-nothing — so one swept machine's compose error failed the operator's named
// push and skipped its --wait, the very intolerance the main path exists to avoid.
// Held for this round only, and after the last round's were given back, so two pushes
// cascading into each other's machines never each wait on the other.
refused, err := sendRound(ctx, open, also,
func(held context.Context, node string) (sendable, error) {
plan, settings, err := planFor(held, open, node)
if err != nil {
return sendable{}, err
}
reportUnhostable(node, plan)
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
if err == nil {
reportLeftOut(node, declared)
}
return declared, err
},
bus, holder)
refusals = append(refusals, refused...)
if err != nil {
return err
}
// Every candidate this round is marked handled — the sent ones so they are not
// re-listed, and the refused ones so a machine that cannot be composed does not make
// the loop spin on it for ever. Its refusal is already in the report.
for _, name := range also {
flushed[name] = true
}
}
}
@@ -751,7 +762,7 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
}
// After it is away, not before. A digest recorded for something that failed to send would make
// the machine look current for a declaration it never received.
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds)
digest, err := recordSent(ctx, b.open.inventory, s.node, body)
if err != nil {
return "", err
}
@@ -1233,11 +1244,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, e
// never wrote it down: status read "applied, current" over a machine that had just been sent
// something else. What was sent was sent; the record of it must not depend on the sender living
// another second. Bounded, so a store that is away does not hold a dying process open for ever.
//
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
// what tells a machine held back by a policy or a plan from one a push left behind.
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
builds map[string]string) (string, error) {
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte) (string, error) {
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
record, err := inv.NodeByName(kept, node)
@@ -1245,7 +1252,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
return "", err
}
digest := digestOf(body)
if err := inv.RecordSent(kept, record.ID, digest, builds); err != nil {
if err := inv.RecordSent(kept, record.ID, digest); err != nil {
return "", err
}
return digest, nil
cmd/mesh-controller/readable.go
-10
View File
@@ -78,14 +78,6 @@ type meshStatus struct {
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
// dependency is met. Reported, not refused, until the switch.
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
// Failing is every consumer a provider says it keeps failing, with the class of error, since
// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
// document without this called the mesh well while the identity provider refused every consumer
// for a day (04-ISSUES/179).
Failing []inventory.ProviderStanding `json:"failing,omitempty"`
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
}
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
@@ -218,8 +210,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
}
}
out.Unheld = asked.unheld
out.Failing = asked.failing
out.Overflowing = asked.overflowing
for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]})
cmd/mesh-controller/rollout.go
+1 -1
View File
@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
// would bury the ones that matter under a list nobody can act on.
func speaksOnTheBus(m catalogue.Manifest) bool {
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
}
cmd/mesh-controller/rotate.go
+1 -34
View File
@@ -6,8 +6,6 @@ import (
"flag"
"fmt"
"sort"
"github.com/novox/mesh-controller/internal/inventory"
)
// rotateCommand replaces a credential and moves both ends together.
@@ -35,16 +33,12 @@ func rotateCommand(ctx context.Context, args []string) error {
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
// and rotating the other nine would be a great deal of disruption for one suspicion.
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
// One consuming module rather than every module on the machine. A machine runs many consumers
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
// issue 268) is no reason to restart every other one on the machine.
module := set.String("module", "", "only this consuming module's credential")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
return errors.New("rotate <provision> [--consumer <machine>]")
}
provision := positionals[0]
@@ -59,12 +53,6 @@ func rotateCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
holders = ofModule(holders, *module)
if len(holders) == 0 && *module != "" {
return fmt.Errorf(
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
"says what a machine holds", *module, onMachine(*only), provision)
}
if len(holders) == 0 {
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
// and a rotation somebody believes happened is worse than one they know did not.
@@ -128,27 +116,6 @@ func rotateCommand(ctx context.Context, args []string) error {
return nil
}
// ofModule is the holders whose consuming module is this one; all of them when none is named.
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
if module == "" {
return holders
}
var out []inventory.Holder
for _, h := range holders {
if h.ConsumerModule == module {
out = append(out, h)
}
}
return out
}
func onMachine(machine string) string {
if machine == "" {
return ""
}
return " on " + machine
}
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
func asLocal(local string) string {
if local == "" {
cmd/mesh-controller/rotate_test.go
-27
View File
@@ -1,27 +0,0 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// One consuming module's credential, and not its neighbours' on the same machine (novox/hq issue
// 268): a module that leaked its database password is no reason to restart every other consumer.
func TestARotationNarrowedToAModuleTouchesOnlyThatModulesCredential(t *testing.T) {
holders := []inventory.Holder{
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Provider: "ace"},
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Local: "reader", Provider: "ace"},
}
got := ofModule(holders, "letta")
if len(got) != 2 || got[0].ConsumerModule != "letta" || got[1].Local != "reader" {
t.Fatalf("narrowed to letta: %+v", got)
}
if len(ofModule(holders, "")) != 3 {
t.Fatal("no module named narrowed anyway")
}
if len(ofModule(holders, "absent")) != 0 {
t.Fatal("a module holding nothing matched")
}
}
cmd/mesh-controller/seats.go
+9 -35
View File
@@ -29,13 +29,11 @@ type seatHolder struct {
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
type seatRow struct {
Seat string `json:"seat"`
Scope string `json:"scope"`
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
Replicated bool `json:"replicated,omitempty"`
Holders []seatHolder `json:"holders"`
Seat string `json:"seat"`
Scope string `json:"scope"`
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
Holders []seatHolder `json:"holders"`
}
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -49,7 +47,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
rows := make([]seatRow, 0, len(seats))
for _, s := range seats {
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
Replicated: s.Replicated, Holders: []seatHolder{}}
Holders: []seatHolder{}}
seen := map[seatHolder]bool{}
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -106,13 +104,9 @@ func seatCommand(ctx context.Context, args []string) error {
return nil
}
if len(args) == 3 && args[1] == "--to" {
return handOver(ctx, args[0], args[2], false)
return handOver(ctx, args[0], args[2])
}
if len(args) == 3 && args[1] == "--add" {
return handOver(ctx, args[0], args[2], true)
}
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
"seat <name> --add <node>/<module>")
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
}
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
@@ -125,12 +119,7 @@ func seatCommand(ctx context.Context, args []string) error {
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
// and refusing to record a handover to a module the node has not started would make the handover
// impossible to do before the switch instead of as the switch.
//
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
// records the named assignment as a further holder and leaves every holder on record as it is. A
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
// the one named, as it always did.
func handOver(ctx context.Context, seatName, to string, adding bool) error {
func handOver(ctx context.Context, seatName, to string) error {
nodeName, module, ok := strings.Cut(to, "/")
if !ok || nodeName == "" || module == "" {
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
@@ -146,10 +135,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
if !known {
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
}
if adding && !seat.Replicated {
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
}
assigned, err := inv.Assigned(ctx, nodeName)
if err != nil {
return err
@@ -172,7 +157,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
}
var was string
var held []string
holdings, err := inv.Holdings(ctx)
if err != nil {
return err
@@ -180,7 +164,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
for _, h := range holdings {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
was = h.Node
held = append(held, h.Node)
}
}
@@ -204,15 +187,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
} else if err := catalogue.CanHold(*m, seat); err != nil {
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
}
if adding {
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
}
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
strings.Join(held, ", "))
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
return nil
}
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
}
cmd/mesh-controller/seatverbs.go
+37 -291
View File
@@ -36,194 +36,19 @@ type verbAnswer struct {
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
//
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
// the verb declares but did not use for the command line it composed — given beside another that
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
// not take that" would have stopped it before anything was sent.
func argvFor(verb string, args map[string]any) ([]string, error) {
a, err := readArguments(verb, args)
if err != nil {
return nil, err
str := func(key string) string {
v, _ := args[key].(string)
return strings.TrimSpace(v)
}
argv, err := a.commandLine()
if len(a.misread) > 0 {
// The table and the command line disagree: the verb reads an argument no caller can see
// in its schema, so no caller could ever pass it.
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
"table and its command lines disagree", verb, quoteAll(a.misread))
}
if err != nil {
return nil, err
}
if unused := a.unused(); len(unused) > 0 {
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
}
return argv, nil
}
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
// command line was composed from.
type verbArguments struct {
verb string
given map[string]string
used map[string]bool
declared map[string]bool
misread []string // arguments the command line read that the schema does not declare: a bug here
}
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
// wrote.
func controllerVerb(name string) (catalogue.Verb, bool) {
for _, v := range catalogue.ControllerVerbs {
if v.Name == name {
return v, true
}
}
return catalogue.Verb{}, false
}
// declaredArguments are a schema's properties, and which of them are switches.
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
switches = map[string]bool{}
props, _ := v.Input["properties"].(map[string]any)
for name, p := range props {
names = append(names, name)
desc, _ := p.(map[string]any)
switch enum := desc["enum"].(type) {
case []string:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
case []any:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
}
}
sort.Strings(names)
return names, switches
}
// readArguments refuses what the verb does not take, before anything is composed.
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
v, known := controllerVerb(verb)
if !known {
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
names, switches := declaredArguments(v)
declared := map[string]bool{}
for _, n := range names {
declared[n] = true
}
takes := "none"
if len(names) > 0 {
takes = quoteAll(names)
}
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
keys := make([]string, 0, len(args))
for k := range args {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if !declared[k] {
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
}
var value string
switch x := args[k].(type) {
case nil:
continue
case string:
value = strings.TrimSpace(x)
case bool:
if !switches[k] {
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
}
value = fmt.Sprint(x)
default:
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
}
if switches[k] {
switch value {
case "true":
case "false", "":
continue // said and off: the same as not given, and nothing passed over
default:
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
need := func(keys ...string) error {
for _, k := range keys {
if str(k) == "" {
return fmt.Errorf("%s needs %q", verb, k)
}
}
if value != "" {
a.given[k] = value
}
return nil
}
return a, nil
}
// str is one argument's value, marked as used.
func (a *verbArguments) str(key string) string {
if !a.declared[key] {
a.misread = append(a.misread, key)
}
a.used[key] = true
return a.given[key]
}
// on is a switch, marked as used.
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
// need refuses a call missing a required argument, in the verb's own words.
func (a *verbArguments) need(keys ...string) error {
for _, k := range keys {
if a.str(k) == "" {
return fmt.Errorf("%s needs %q", a.verb, k)
}
}
return nil
}
// unused are the arguments given that the command line was not composed from.
func (a *verbArguments) unused() []string {
var out []string
for k := range a.given {
if !a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
return out
}
func (a *verbArguments) usedGiven() []string {
var out []string
for k := range a.given {
if a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
if len(out) == 0 {
return []string{"nothing"}
}
return out
}
func quoteAll(xs []string) string {
q := make([]string, len(xs))
for i, x := range xs {
if x == "nothing" {
q[i] = x
continue
}
q[i] = fmt.Sprintf("%q", x)
}
return strings.Join(q, ", ")
}
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
func (a *verbArguments) commandLine() ([]string, error) {
verb, str, on, need := a.verb, a.str, a.on, a.need
switch verb {
case "command":
// The generic verb: the command line as given, split as a shell would split it, with
@@ -240,8 +65,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, errors.New("command names no command")
}
return argv, nil
case "tools":
return nil, errors.New("tools is answered from the records, not by a command")
case "status":
return []string{"status", "--json"}, nil
case "nodes":
@@ -259,14 +82,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
if id := str("log"); id != "" {
return []string{"builds", "--log", id}, nil
}
argv := []string{"builds"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
if m := str("module"); m != "" {
argv = append(argv, m)
return []string{"builds", m}, nil
}
return argv, nil
return []string{"builds"}, nil
case "plans":
if r := str("repository"); r != "" {
argv := []string{"plans", "--what-if", r}
@@ -278,19 +97,19 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return argv, nil
}
for _, act := range []string{"stop", "close", "retry"} {
if id := str(act); id != "" {
return []string{"plans", act, id}, nil
}
if id := str("stop"); id != "" {
return []string{"plans", "stop", id}, nil
}
if id := str("close"); id != "" {
return []string{"plans", "close", id}, nil
}
if id := str("retry"); id != "" {
return []string{"plans", "retry", id}, nil
}
if id := str("id"); id != "" {
return []string{"plans", id}, nil
}
argv := []string{"plans"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
return argv, nil
return []string{"plans"}, nil
// The build queue (novox/hq ADR 0219).
case "queue":
return []string{"queue"}, nil
@@ -300,7 +119,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return []string{verb, str("id")}, nil
case "clear":
if on("dead") {
if str("dead") == "true" {
return []string{"clear", "--dead"}, nil
}
return []string{"clear"}, nil
@@ -314,10 +133,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, err
}
argv := []string{"replay", str("id")}
if on("register") {
if str("register") == "true" {
argv = append(argv, "--register")
}
if on("older") {
if str("older") == "true" {
argv = append(argv, "--older")
}
return argv, nil
@@ -330,9 +149,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
if err := need("node"); err != nil {
return nil, err
}
if on("files") {
return []string{"plan", str("node"), "--files"}, nil
}
return []string{"plan", str("node"), "--json"}, nil
case "assign", "unassign":
if err := need("node", "module"); err != nil {
@@ -356,14 +172,8 @@ func (a *verbArguments) commandLine() ([]string, error) {
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
// time out on every machine that takes a minute, and say nothing about the ones that did not.
if n := str("node"); n != "" {
// behind is not read here: given with a machine, it is refused as passed over — naming
// a machine and asking for every machine behind are two requests, and guessing one
// would push a machine nobody named, or not push one somebody did.
return []string{"push", n, "--wait", "0"}, nil
}
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
// first, so a caller who meant one machine reads that it was not one.
on("behind")
return []string{"push", "--behind", "--wait", "0"}, nil
case "rotate":
if p := str("provision"); p != "" {
@@ -371,24 +181,13 @@ func (a *verbArguments) commandLine() ([]string, error) {
if c := str("consumer"); c != "" {
argv = append(argv, "--consumer", c)
}
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
// and secret stay the other shape's, and are refused as passed over.
if m := str("module"); m != "" {
argv = append(argv, "--module", m)
}
return argv, nil
}
_, node := a.given["node"]
_, module := a.given["module"]
_, secret := a.given["secret"]
if node || module || secret {
if err := need("node", "module", "secret"); err != nil {
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
}
if str("node") != "" && str("module") != "" && str("secret") != "" {
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
}
// Neither shape: the command says its usage, which names both, and that is the answer the
// caller needs.
// Half of either shape: the command says its usage, which names both shapes, and that is
// the answer the caller needs.
return []string{"rotate"}, nil
case "settings":
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
@@ -396,16 +195,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
if err := need("module"); err != nil {
return nil, err
}
var argv []string
if on("clear") {
argv := []string{"settings", "set", str("module")}
switch {
case str("clear") == "true":
argv = []string{"settings", "clear", str("module")}
} else {
argv = []string{"settings", "set", str("module")}
// Neither values nor clear: the command says its usage, which names both.
if v := str("values"); v != "" {
argv = append(argv, v)
}
case str("values") != "":
argv = append(argv, str("values"))
}
// Neither values nor clear: the command says its usage, which names both, and that is the
// answer the caller needs — the same as `rotate` given half of either shape.
if n := str("node"); n != "" {
argv = append(argv, "--node", n)
}
@@ -437,8 +235,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return argv, nil
}
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
verb, catalogue.ControllerSeatName)
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
@@ -490,22 +287,8 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
handlers := map[string]link.ToolHandler{}
for _, v := range seat.Serves {
verb := v.Name
if inProcess[verb] {
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
args := map[string]any{}
if len(bytes.TrimSpace(raw)) > 0 {
if err := json.Unmarshal(raw, &args); err != nil {
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
}
}
// Refused like any verb's: what a verb does not take is not ignored.
a, err := readArguments(verb, args)
if err != nil {
return nil, err
}
if verb == "calls" {
return callsAnswer(link.Calls, a.given["call"])
}
if verb == "tools" {
handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
return seatTools(), nil
}
continue
@@ -544,48 +327,12 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
if err != nil {
return nil, err
}
if answersFirst(argv) {
// Before anything is sent: a push sends the bus's own machine first, and a broker
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
link.Acknowledge(ctx)
}
return runVerb(ctx, argv)
}
}
return handlers, behind, nil
}
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
// the records, `calls` from what this process served.
var inProcess = map[string]bool{"tools": true, "calls": true}
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
// through `command`. A push sends the machine holding the bus first when its user list changed, the
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
func answersFirst(argv []string) bool {
return len(argv) > 0 && argv[0] == "push"
}
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
// one call whole.
func callsAnswer(log *link.CallLog, id string) (any, error) {
if id != "" {
c, ok := log.Get(id)
if !ok {
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
}
return c, nil
}
recent := log.Recent()
for i := range recent {
recent[i].Answer = nil
}
return map[string]any{"calls": recent, "kept": link.KeptCalls,
"note": "newest first; `calls` with a call's id gives its whole answer"}, nil
}
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
func seatTools() map[string]any {
@@ -606,10 +353,9 @@ func seatTools() map[string]any {
}
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
// more, since an argument a verb does not declare is refused.
// verb runnable rather than that it happens to want the arguments the check guessed.
func sampleArguments(v catalogue.Verb) map[string]any {
sample := map[string]any{}
sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
switch required := v.Input["required"].(type) {
case []string:
for _, k := range required {
cmd/mesh-controller/seatverbs_schema_test.go
-365
View File
@@ -1,365 +0,0 @@
package main
import (
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The schemas the controller serves, verb by verb, as the console receives them: from the
// announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244).
func servedSchemas(t *testing.T) map[string]catalogue.Verb {
t.Helper()
handlers, behind, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
if len(behind) != 0 {
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
}
served := map[string]catalogue.Verb{}
for _, e := range seatAnnouncement(handlers).Endpoints {
var input map[string]any
if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil {
t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err)
}
served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input}
}
if len(served) != len(catalogue.ControllerVerbs) {
t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs))
}
return served
}
// subsetsOf is every subset of the names, the empty one included.
func subsetsOf(names []string) [][]string {
var out [][]string
for mask := 0; mask < 1<<len(names); mask++ {
var s []string
for i, n := range names {
if mask&(1<<i) != 0 {
s = append(s, n)
}
}
out = append(out, s)
}
return out
}
func argumentsFor(subset []string, switches map[string]bool) map[string]any {
args := map[string]any{}
for _, n := range subset {
if switches[n] {
args[n] = "true"
} else {
args[n] = "x-" + n
}
}
return args
}
// saidOutright are the switches that say the default aloud, so the line is the same without them —
// on purpose, and only these.
var saidOutright = map[string]string{
"push": "behind", // the whole mesh is what no machine means; behind lets a caller say they meant it
}
// **No argument a caller gives is passed over** (novox/hq issue 244). For every verb served and
// every combination of the arguments its schema declares, the verb either refuses the call, or
// composes a command line that each given argument changed: taking any one away changes the line
// or makes it refused. An argument the line is the same without is one the verb ignored — the
// shape of the push that named a machine and pushed every machine behind.
func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
for name, v := range servedSchemas(t) {
if inProcess[name] {
continue
}
names, switches := declaredArguments(v)
for _, subset := range subsetsOf(names) {
args := argumentsFor(subset, switches)
argv, err := argvFor(name, args)
if err != nil {
if strings.Contains(err.Error(), "does not declare") {
t.Errorf("%s %v: %v", name, args, err)
}
continue
}
for _, dropped := range subset {
fewer := map[string]any{}
for k, val := range args {
if k != dropped {
fewer[k] = val
}
}
without, err := argvFor(name, fewer)
if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped {
t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv)
}
}
}
}
}
// **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with
// what it requires and one argument more; and `node` in particular, for every verb whose schema
// does not take a machine.
func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
for name, v := range servedSchemas(t) {
names, _ := declaredArguments(v)
strangers := []string{"no-such-argument"}
if !contains(names, "node") {
strangers = append(strangers, "node")
}
for _, stranger := range strangers {
args := sampleArguments(v)
args[stranger] = "x"
_, err := argvFor(name, args)
if inProcess[name] {
_, err = readArguments(name, args)
}
if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err)
}
}
}
if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil {
t.Error("a switch took a word that is neither true nor false, and would have read it as false")
}
if _, err := argvFor("node", map[string]any{"node": 7}); err == nil {
t.Error("a number was taken as a machine's name, or as no machine")
}
if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" {
t.Errorf("a switch given as JSON true: %v %v", argv, err)
}
}
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
// naming one beside behind is refused rather than one of the two guessed.
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
argv, err := argvFor("push", map[string]any{"node": "g1"})
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0" {
t.Fatalf("a named push: %v %v", argv, err)
}
for _, args := range []map[string]any{{}, {"behind": "true"}} {
argv, err := argvFor("push", args)
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0" {
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
}
}
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true"}); err == nil ||
!strings.Contains(err.Error(), `"behind"`) {
t.Fatalf("a named push with behind was taken: %v", err)
}
if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) {
t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err)
}
}
// commandFlags is every flag set this package's commands parse, by the name the set is made with,
// and the flags defined on it — read from the source, so a flag added to a command is seen here
// without anyone remembering to.
func commandFlags(t *testing.T) map[string][]string {
t.Helper()
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
fset := token.NewFileSet()
out := map[string][]string{}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
file, err := parser.ParseFile(fset, f, nil, 0)
if err != nil {
t.Fatal(err)
}
for _, decl := range file.Decls {
fn, ok := decl.(*ast.FuncDecl)
if !ok || fn.Body == nil {
continue
}
sets := map[string]string{} // variable → the set's name
ast.Inspect(fn.Body, func(n ast.Node) bool {
if assign, ok := n.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 && len(assign.Rhs) == 1 {
if call, ok := assign.Rhs[0].(*ast.CallExpr); ok && isSelector(call.Fun, "flag", "NewFlagSet") {
if id, ok := assign.Lhs[0].(*ast.Ident); ok {
if name, ok := stringLit(call.Args[0]); ok {
sets[id.Name] = name
out[name] = append(out[name], []string{}...)
}
}
}
}
call, ok := n.(*ast.CallExpr)
if !ok {
return true
}
sel, ok := call.Fun.(*ast.SelectorExpr)
if !ok {
return true
}
recv, ok := sel.X.(*ast.Ident)
if !ok || sets[recv.Name] == "" {
return true
}
arg := 0
switch sel.Sel.Name {
case "Bool", "String", "Int", "Int64", "Uint", "Uint64", "Float64", "Duration", "Func", "BoolFunc", "TextVar":
case "BoolVar", "StringVar", "IntVar", "Int64Var", "UintVar", "Uint64Var", "Float64Var", "DurationVar", "Var":
arg = 1
default:
return true
}
if arg < len(call.Args) {
if flagName, ok := stringLit(call.Args[arg]); ok {
out[sets[recv.Name]] = append(out[sets[recv.Name]], flagName)
}
}
return true
})
}
}
return out
}
func isSelector(e ast.Expr, pkg, name string) bool {
sel, ok := e.(*ast.SelectorExpr)
if !ok {
return false
}
id, ok := sel.X.(*ast.Ident)
return ok && id.Name == pkg && sel.Sel.Name == name
}
func stringLit(e ast.Expr) (string, bool) {
lit, ok := e.(*ast.BasicLit)
if !ok || lit.Kind != token.STRING {
return "", false
}
s, err := strconv.Unquote(lit.Value)
return s, err == nil
}
// accountedFlags are the flags of a verb's command that are not an argument of the same name:
// carried by an argument named otherwise ("=argument"), or set by the verb itself, or withheld from
// the named verb on purpose — each with why. `command` reaches every flag of the binary regardless.
var accountedFlags = map[string]map[string]string{
"status": {"json": "set by the verb: the answer is data"},
"seats": {"json": "set by the verb: the answer is data"},
"queue": {"json": "not set: the verb answers the table a person reads"},
"plan": {"json": "set by the verb unless files is asked"},
"push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"},
"build": {
"wait": "set by the verb to 0: the id follows the build (issue 176)",
"self": "set by the verb from the repository's form: a path on the forge, or a URL",
"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
"behind": "withheld: the named verb builds one named repository; `command` reaches the rest",
"on": "withheld: the named verb builds one named repository; `command` reaches the rest",
},
"builds": {"n": "=limit"},
"plans": {"n": "=limit", "what-if": "=repository"},
}
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
// from the source, so a flag added to a command — or a verb added whose command takes flags —
// fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And
// a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody
// reads.
func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
flags := commandFlags(t)
if len(flags["push"]) == 0 || len(flags["plan"]) == 0 {
t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags)
}
reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
served := servedSchemas(t)
for name, v := range served {
if inProcess[name] || name == "command" {
continue
}
names, switches := declaredArguments(v)
for _, subset := range subsetsOf(names) {
argv, err := argvFor(name, argumentsFor(subset, switches))
if err != nil {
continue
}
// The flag set is named by the longest run of leading words that names one.
var words []string
for _, w := range argv {
if strings.HasPrefix(w, "-") {
break
}
words = append(words, w)
}
for n := len(words); n > 0; n-- {
if _, has := flags[strings.Join(words[:n], " ")]; has {
if reached[name] == nil {
reached[name] = map[string]bool{}
}
reached[name][strings.Join(words[:n], " ")] = true
break
}
}
}
}
used := map[string]map[string]bool{}
verbs := make([]string, 0, len(reached))
for verb := range reached {
verbs = append(verbs, verb)
}
sort.Strings(verbs)
for _, verb := range verbs {
declared, _ := declaredArguments(served[verb])
for set := range reached[verb] {
if used[set] == nil {
used[set] = map[string]bool{}
}
for _, flagName := range flags[set] {
why, accounted := accountedFlags[set][flagName]
switch {
case accounted && strings.HasPrefix(why, "="):
used[set][flagName] = true
if !contains(declared, strings.TrimPrefix(why, "=")) {
t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare",
verb, flagName, set, strings.TrimPrefix(why, "="))
}
case accounted:
used[set][flagName] = true
case contains(declared, flagName):
default:
t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+
"or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName)
}
}
}
}
for set, fs := range accountedFlags {
for flagName := range fs {
if !used[set][flagName] {
t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set)
}
}
}
}
// Every verb's required arguments are properties of its schema: a schema that requires what it
// does not describe is the uncallable verb of issue 244 from the other side.
func TestEveryRequiredArgumentIsDescribed(t *testing.T) {
for name, v := range servedSchemas(t) {
names, _ := declaredArguments(v)
for k := range sampleArguments(v) {
if !contains(names, k) {
t.Errorf("%s requires %q and does not describe it", name, k)
}
}
}
}
cmd/mesh-controller/seatverbs_test.go
+26 -12
View File
@@ -10,6 +10,29 @@ import (
"github.com/novox/mesh-controller/internal/catalogue"
)
// Every verb the mesh-controller seat declares is one this binary can run, with the arguments the
// schema names and no other (novox/hq ADR 0154, ADR 0035).
func TestEveryDeclaredVerbHasACommandLine(t *testing.T) {
for _, v := range catalogue.ControllerVerbs {
if v.Name == "tools" {
continue
}
args := map[string]any{}
props, _ := v.Input["properties"].(map[string]any)
for name := range props {
args[name] = "x"
}
argv, err := argvFor(v.Name, args)
if err != nil {
t.Errorf("%s: %v", v.Name, err)
continue
}
if argv[0] == "" {
t.Errorf("%s: empty command", v.Name)
}
}
}
// `builds` given a build's id reads that build's log from the bus rather than listing builds
// (novox/hq ADR 0157).
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
@@ -43,22 +66,13 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
t.Fatalf("a pair credential: %v", argv)
}
argv, _ = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "module": "letta"})
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace --module letta" {
t.Fatalf("one consuming module's pair credential: %v", argv)
}
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
t.Fatalf("an own secret: %v", argv)
}
if _, err := argvFor("rotate", map[string]any{"node": "ace"}); err == nil || !strings.Contains(err.Error(), `"module"`) {
t.Fatalf("half an own secret is refused, naming what it lacks: %v", err)
}
if argv, _ := argvFor("rotate", nil); strings.Join(argv, " ") != "rotate" {
t.Fatalf("neither shape falls to the command's usage: %v", argv)
}
if _, err := argvFor("rotate", map[string]any{"provision": "p", "node": "ace", "module": "m", "secret": "s"}); err == nil {
t.Fatal("both shapes at once were taken, and one of them passed over")
argv, _ = argvFor("rotate", map[string]any{"node": "ace"})
if strings.Join(argv, " ") != "rotate" {
t.Fatalf("half an own secret falls to the command's usage: %v", argv)
}
}
cmd/mesh-controller/sendable.go
-7
View File
@@ -43,13 +43,6 @@ type sendable struct {
LeftOut []string
// leftOutWhy is why each was, for push and plan to say; never on the wire.
leftOutWhy map[string]string
// withheld is every consumer this machine's grants leave out, because its identity overflows the
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
withheld []catalogue.Overflow
// Builds is the build of each module this declaration carries — module to the commit its build
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
// Composed only on the send path; nil records that it is not known.
Builds map[string]string
}
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
cmd/mesh-controller/standing.go
-120
View File
@@ -1,120 +0,0 @@
package main
import (
"context"
"fmt"
"strings"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
//
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
// provider now announces a consumer it has failed for minutes; the controller keeps it until the
// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
// standings keeps what providers say, in the inventory.
type standings struct{ inv *inventory.Inventory }
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
Consumer: st.Consumer, ConsumerNode: st.Node,
Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
})
}
// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
//
// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
// consumer clears it.
func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
all, err := inv.FailingProviders(ctx)
if err != nil {
return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
}
assigned := map[string]map[string]bool{}
var out []inventory.ProviderStanding
for _, s := range all {
on, asked := assigned[s.ProviderNode]
if !asked {
modules, err := inv.Assigned(ctx, s.ProviderNode)
if err != nil {
// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
modules = nil
}
on = map[string]bool{}
for _, m := range modules {
on[m] = true
}
assigned[s.ProviderNode] = on
}
if on[s.Module] {
out = append(out, s)
}
}
return out, nil
}
// failingLines is how status says them: one consumer per entry, the error under it, and a provider
// that stopped repeating itself said so.
func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
var out []string
for _, s := range list {
where := s.Module
if s.ProviderNode != "" {
where += " on " + s.ProviderNode
}
whom := s.Consumer
if s.ConsumerNode != "" {
whom += " (" + s.ConsumerNode + ")"
}
out = append(out, fmt.Sprintf(" %-24s fails %s: %s, for %s (%d attempts since %s)",
where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
s.Since.Local().Format("2006-01-02 15:04")))
if e := strings.TrimSpace(s.Error); e != "" {
out = append(out, fmt.Sprintf(" %-24s %s", "", firstLine(e)))
}
if s.Quiet(now) {
out = append(out, fmt.Sprintf(" %-24s not said again for %s — the provider has stopped "+
"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
}
}
return out
}
func orUnclassed(class string) string {
if class == "" {
return "failing"
}
return class
}
// printFailing is the status section, said when there is anything to say.
func printFailing(list []inventory.ProviderStanding, now time.Time) {
if len(list) == 0 {
return
}
fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
for _, line := range failingLines(list, now) {
fmt.Println(line)
}
fmt.Printf("\n the provider's journal has every attempt; it says recovered on its next success\n\n")
}
// failingOn is the standings that concern one machine: a provider running there, or a consumer.
func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
var out []inventory.ProviderStanding
for _, s := range list {
if s.ProviderNode == node || s.ConsumerNode == node {
out = append(out, s)
}
}
return out
}
cmd/mesh-controller/standing_test.go
-115
View File
@@ -1,115 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
t.Fatal(err)
}
kept := standings{open.inventory}
since := time.Now().Add(-23 * time.Hour)
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
if _, err := kept.Stood(ctx, failing); err != nil {
t.Fatal(err)
}
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if asked.well() {
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
}
said := printed(t, func() error { return printStatus(asked) })
for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
if !strings.Contains(said, want) {
t.Fatalf("status does not say %q:\n%s", want, said)
}
}
if strings.Contains(said, "all doing what they were told") {
t.Fatalf("status said all well beside a failing provider:\n%s", said)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
Failing []inventory.ProviderStanding `json:"failing"`
}
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
t.Fatalf("the document does not carry it: %v\n%s", err, body)
}
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
for _, node := range []string{"anchor", "laptop"} {
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
t.Fatalf("node show %s does not name it:\n%s", node, shown)
}
}
// Recovered: gone, and the mesh may be well again as far as this is concerned.
failing.Failing = false
if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
t.Fatalf("%v %v", cleared, err)
}
asked, err = theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if len(asked.failing) != 0 {
t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
}
}
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
// not a problem.
func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
t.Fatal(err)
}
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if len(asked.failing) != 0 {
t.Fatalf("%+v", asked.failing)
}
}
func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
now := time.Now()
lines := strings.Join(failingLines([]inventory.ProviderStanding{{
Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
}}, now), "\n")
for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
if !strings.Contains(lines, want) {
t.Fatalf("%q not in:\n%s", want, lines)
}
}
if strings.Contains(lines, "more") {
t.Fatalf("more than the first line of an error:\n%s", lines)
}
}
cmd/mesh-controller/status.go
+1 -28
View File
@@ -129,10 +129,6 @@ func printStatus(asked answers) error {
fmt.Println()
}
// A provider failing a consumer, beside machines failing what they were told: both are something
// not working now (novox/hq ADR 0224).
printFailing(asked.failing, time.Now())
if len(quiet) > 0 {
var said []string
for _, n := range quiet {
@@ -302,19 +298,6 @@ func printStatus(asked answers) error {
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
}
if len(asked.overflowing) > 0 {
// **Reported, and the provider still pushed** (novox/hq ADR 0225, issue 263). Each is left
// out of its provider's grants, so the module holds a login nothing created; the provider's
// machine is sent everything else rather than refused for one consumer elsewhere.
fmt.Printf("%d module(s) identified too long for a provision they require, and not granted it:\n",
len(asked.overflowing))
for _, o := range asked.overflowing {
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
}
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
}
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
// Said, because nothing forces the flip: a node left adopted is visible here rather than
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -432,16 +415,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
return answers{}, err
}
out.unheld = append(out.unheld, plan.Unheld...)
// And which of its modules a provider leaves out of its grants, for an identity too long
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
// resolution, as the provider's composition judges it.
out.overflowing = append(out.overflowing, plan.Overflowing()...)
}
// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
// providers announced: nothing else in the mesh knows whether a provision is being made.
out.failing, err = failingProviders(ctx, inv)
if err != nil {
return answers{}, err
}
out.plans, err = inv.RecentPlans(ctx, 5)
if err != nil {
@@ -555,7 +528,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0 && len(a.overflowing) == 0
len(a.filtered) == 0 && len(a.unheld) == 0
}
// hostSplit is which machines report which host version, for every version more than one machine
cmd/mesh-controller/upgrades.go
+28 -58
View File
@@ -8,7 +8,6 @@ import (
"path"
"regexp"
"strings"
"sync"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
@@ -267,11 +266,6 @@ func notNow(err error) error {
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
// running the module is the upgrade's decision, taken when the catalogue announces it.
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
// what the bus did not hand over (novox/hq issue 266). Each judges against what the other wrote.
actingOnMerges.Lock()
defer actingOnMerges.Unlock()
inv := f.open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
@@ -282,7 +276,34 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
return notNow(err)
}
from, packaging, already := mergeCandidates(m, entries, read)
// Two kinds of module are affected by one merge, and they are affected differently.
//
// A module **built from** this repository and branch has moved: the mesh records the new commit
// as what its source now has, and only what the merge actually changed is rebuilt. A module that
// only **packages source from** it has not moved — its own source is somewhere else, at the
// commit it already records — so it is rebuilt and its record left alone. Writing this commit as
// its source would make it permanently behind a repository its manifest does not come from.
var from, packaging []inventory.Entry
already := 0
for _, e := range entries {
switch {
case sourceIs(e.Source, m):
if e.Source.BuiltFrom == m.Commit {
already++
continue
}
// **A merge older than the last look at the source is history, not a move.** The forge
// announces what it finds merged, and an old merge surfacing late would otherwise move
// the recorded head backwards and rebuild everything built from that repository, once
// per old merge (2026-09-28).
if isHistory(m.MergedAt, e.Source.Seen) {
continue
}
from = append(from, e)
case readsFrom(read[e.Manifest.Module], m):
packaging = append(packaging, e)
}
}
if len(from) == 0 && len(packaging) == 0 {
// "Already built from it" and "nothing reads it" are different facts, and reading the first
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
@@ -417,57 +438,6 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
return nil
}
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
var actingOnMerges sync.Mutex
// mergeCandidates is what one merge could move, judged against what the catalogue holds.
//
// Two kinds of module are affected by one merge, and they are affected differently.
//
// A module **built from** this repository and branch has moved: the mesh records the new commit as
// what its source now has, and only what the merge actually changed is rebuilt. A module that only
// **packages source from** it has not moved — its own source is somewhere else, at the commit it
// already records — so it is rebuilt and its record left alone. Writing this commit as its source
// would make it permanently behind a repository its manifest does not come from. `already` counts
// the modules built from this repository that are already built from this very commit.
func mergeCandidates(m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository) (from, packaging []inventory.Entry, already int) {
for _, e := range entries {
switch {
case sourceIs(e.Source, m):
if e.Source.BuiltFrom == m.Commit {
already++
continue
}
// **A merge older than the last look at the source is history, not a move.** The forge
// announces what it finds merged, and an old merge surfacing late would otherwise move
// the recorded head backwards and rebuild everything built from that repository, once
// per old merge (2026-09-28).
if isHistory(m.MergedAt, e.Source.Seen) {
continue
}
from = append(from, e)
case readsFrom(read[e.Manifest.Module], m):
packaging = append(packaging, e)
}
}
return from, packaging, already
}
// wouldMove is the modules built from the merged repository that acting on this merge would mark as
// moved and rebuild — SourceMoved's judgement, made without acting (novox/hq issue 266). Empty for a
// merge already acted on: acting marks each of them as looked at, so the merge then reads as history.
//
// **Only the modules built from it, never the ones that merely package source from it.** Acting
// records nothing about those, so a merge acted on would go on reading as unacted for them, and be
// acted on again on every look. A merge that moves both is caught by the first kind, and acting on it
// rebuilds the second as well.
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository) []inventory.Entry {
from, _, _ := mergeCandidates(m, entries, read)
return whatTheMergeTouched(from, entries, m)
}
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
// An empty recorded ref is the repository's default branch, which is what a merge into the base
internal/broker/nats.go
+3 -12
View File
@@ -18,7 +18,6 @@ import (
"regexp"
"sort"
"strings"
"time"
)
// A Kind is what a principal is, which decides the shape of its authority rather than its
@@ -190,13 +189,6 @@ type Permissions struct {
AllowResponses bool
}
// ResponseTTL is how long the bus lets a principal answer a request it received. Its one answer has
// to come inside this, and a seat's holder answers within link.AnswerWithin — inside it by design.
// **A broker reloading its user list forgets every answer it was about to permit**, whatever this
// says (novox/hq issue 265): a call that is still running when the list reloads has its answer
// refused, which is why a holder answers before it does what can reload it.
const ResponseTTL = time.Minute
// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
// a permission that cannot be derived from a declaration is a permission nobody can explain.
func PermissionsFor(p Principal) (Permissions, error) {
@@ -261,11 +253,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
sub = append(sub, announcing(ControllerSeat)...)
// The events it reacts to, and its ack subject on the stream they arrive from
// The two events it reacts to, and its ack subject on the stream they arrive from
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
// controller a subscriber to every event in the mesh, and its permission list would stop
// saying what it is for. The one wildcard is the emitter of a provider's standing (ADR
// 0224) — still two named events, from whichever module provides. The ack grant below is scoped per stream because the controller's
// saying what it is for. The ack grant below is scoped per stream because the controller's
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
// ever, refused by the list it already has.
@@ -786,7 +777,7 @@ func ComposeAccounts(principals []Principal) (string, error) {
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish))
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
if perms.AllowResponses {
fmt.Fprintf(&b, " allow_responses: { max: 1, ttl: \"%dm\" }\n", int(ResponseTTL/time.Minute))
b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n")
}
b.WriteString(" } }\n")
}
internal/broker/seattools_test.go
+8 -8
View File
@@ -5,16 +5,16 @@ import "testing"
// A node-scoped seat's tool carries the node (novox/hq ADR 0132, design 33 §4): two nodes holding one
// node-scoped seat derive two addresses, and a user of the seat may publish any node's.
func TestTwoNodesHoldingOneNodeSeatDeriveTwoToolAddresses(t *testing.T) {
seat := Seat{Name: "node-hostname", Scope: "node", Serves: []string{"entries"}}
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
has(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.one")
has(t, two.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries")
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
seat := Seat{Name: "node-dns-resolver", Scope: "node", Serves: []string{"lookup"}}
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
has(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.one")
has(t, two.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup")
hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "three", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
has(t, user.Publish, "mesh.seat.node-hostname.tool.entries.*")
has(t, user.Publish, "mesh.seat.node-dns-resolver.tool.lookup.*")
}
// A mesh-scoped seat's tool stays flat: nothing about it changes.
internal/broker/streams.go
+1 -16
View File
@@ -226,23 +226,8 @@ var ControllerFollows = []string{
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
// with the retired seat row.
seatEventSubject("mesh-build-machine", "built"),
// **Every provider's standing** (novox/hq ADR 0224): a consumer it has failed for minutes, and
// that consumer recovered. The one pattern on this list, and a narrow one — two named events,
// from whichever module provides — because the rule is about every provider, and a list of
// providers here would be a list somebody forgets to extend. On 2026-10-05 the identity provider
// failed every consumer for a day and only its journal said so (issue 179). Appended, because
// the index is a name.
moduleEventSubject("*", ProvisionerFailing),
moduleEventSubject("*", ProvisionerRecovered),
}
// The provider standing events, by their local names. Written here as well as in the catalogue
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
const (
ProvisionerFailing = "provisioner.failing"
ProvisionerRecovered = "provisioner.recovered"
)
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
func moduleEventSubject(module, event string) string {
@@ -286,7 +271,7 @@ func MeshConsumers() []Consumer {
// client and come back to be acted on again.
MaxAckPending: 1,
FromNow: true,
Why: "the events the mesh's own controller reacts to, one at a time; after " +
Why: "the two events the mesh's own controller reacts to, one at a time; after " +
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
"become actionable",
},
internal/broker/testdata/composed.conf
Vendored
+1 -1
View File
@@ -25,7 +25,7 @@ accounts {
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
internal/catalogue/catalogue_check_test.go
-38
View File
@@ -119,41 +119,3 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
}
}
// TestEveryCatalogueIdentityFitsWhatItRequires is ADR 0225's check over the real catalogue: every
// module's identity, on the longest machine name, fits the bound of every provision it wants. An
// overflow fails here, in the pull request that introduces it, rather than on the provider's machine
// the first time a real machine's name meets the module's (issue 263).
func TestEveryCatalogueIdentityFitsWhatItRequires(t *testing.T) {
root := catalogueRoot(t)
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil || len(found) == 0 {
t.Fatalf("no manifests under %s: %v", root, err)
}
shelf := Shelf{}
for _, p := range found {
raw, err := os.ReadFile(p)
if err != nil {
t.Fatalf("%s: %v", p, err)
}
m, err := ParseManifest(raw)
if err != nil {
t.Fatalf("%s: %v", p, err)
}
shelf[m.Module] = m
}
for _, p := range IdentityProblems(shelf, DefaultLongestMachine) {
t.Error(p)
}
// The night it was found: the resolver provision bounds nothing, and the object store still 20.
if dns, ok := shelf["dnsmasq"]; ok {
if b := dns.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
t.Errorf("the resolver provision bounds its consumers' identities: %+v", b)
}
}
if store, ok := shelf["minio"]; ok {
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 {
t.Errorf("the object store's access key is not bounded at 20: %+v", b)
}
}
}
internal/catalogue/computed_test.go
+1 -2
View File
@@ -159,8 +159,7 @@ func TestTwoWaysToBeOnAPrivateNetworkRefuseAndNameBoth(t *testing.T) {
}
}
// reloading answers the runtime's trust as the networking module once did (novox/hq ADR 0102; it no
// longer does, ADR 0222 — and beside the runtime's module it is refused, generated_collision_test): a file
// reloading answers the runtime's trust as the networking module does (novox/hq ADR 0102): a file
// written into, and the runtime reloaded on it.
type reloading struct{}
internal/catalogue/consumer_into_serves.go
+1 -23
View File
@@ -114,9 +114,7 @@ func consumerInto(value, as string) (string, error) {
// asDNSLabel writes a minted identity as a DNS label.
//
// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
// already inside the bound of the provision serving them: a provider that serves one as a DNS label
// bounds it at 63 or less (CheckServes, novox/hq ADR 0225), and one that serves it at all without
// saying is held to twenty (IdentityBoundOf). So
// already short enough for the tightest backend they reach (CheckIdentity, twenty characters). So
// this is the separator and nothing else — no lower-casing of what is already lower case, no
// truncation to a limit the identity is already inside, no padding of a name that is already long
// enough. Each of those would be the mesh guessing at a rule it has not been given.
@@ -143,31 +141,11 @@ func CheckServes(m Manifest) []string {
problems = append(problems, fmt.Sprintf(
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
}
// A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own
// bound has to keep the identity inside one (ADR 0225).
if strings.Contains(text, "${consumer:as:dns}") {
if b := m.IdentityBoundOf(provision); !b.Bounded() || b.Max > dnsLabelLimit {
problems = append(problems, fmt.Sprintf(
"%s serves %s's consumers their identity as a DNS label in %q, and bounds "+
"that identity at %s: a label keeps %d — state an `identity` of at most %d",
m.Module, provision, key, boundWords(b), dnsLabelLimit, dnsLabelLimit))
}
}
}
}
return problems
}
// dnsLabelLimit is the longest DNS label (RFC 1035 §2.3.4).
const dnsLabelLimit = 63
func boundWords(b IdentityBound) string {
if !b.Bounded() {
return "nothing"
}
return fmt.Sprintf("%d", b.Max)
}
func sortedServes(serves map[string]map[string]any) []string {
out := make([]string, 0, len(serves))
for k := range serves {
internal/catalogue/declaration.go
+12 -68
View File
@@ -159,21 +159,12 @@ type Rendering struct {
// 0199): the mesh's resolver forwards each one there.
Zones []ZoneAt
// Holders is, for each replicated mesh seat, every machine holding it, by internal name and
// private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver
// file lists: every holder of the mesh's resolver, this machine first if it is one.
Holders map[string]map[string]string
Settings SettingsBy
Generators map[string]Generator
// Grants are the credentials this node must create, for the provisions it offers. Passed in
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
// resolution answers questions about one machine.
Grants []Grant
// Withheld is every consumer left out of Grants because its identity overflows the provision's
// bound (novox/hq ADR 0225). Composed into nothing; carried so the machine's declaration can say
// whom it does not serve, and why, beside what it does.
Withheld []Overflow
// Ports is where this machine puts what each module needs reachable, by module and by the
// port the software itself uses (novox/hq ADR 0038).
@@ -210,11 +201,6 @@ type Rendering struct {
// stored (novox/hq 04-ISSUES/102).
ArtifactStore string
// SeatReach is where this machine reaches the holder of each mesh-scoped seat it may be asked
// about (host:port), by seat: what ${seat:<seat>:reach} answers with (novox/hq ADR 0222). Absent
// when no holder is reachable yet; see seat_into.go for which seats are answered.
SeatReach map[string]string
// Built is every `<module>/<artifact>` the mesh has built. What tells a reference recorded
// with an address — before references were kept without one — from an image a module runs
// straight from a public registry.
@@ -440,11 +426,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
return nil, err
}
generated, err := r.generatedHere(with)
if err != nil {
return nil, err
}
var out []map[string]any
for _, m := range r.Modules {
if with.Adopted && m.Filtering != nil {
@@ -711,14 +692,23 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
first = append(first, file)
}
if m.Computed != "" {
mine, part := generated[m.Module]
generator, known := with.Generators[m.Computed]
if !known {
return nil, fmt.Errorf(
"%s says its resources are computed by %q, and this control plane has no %q",
m.Module, m.Computed, m.Computed)
}
generated, part, err := generator.Resources(r.Node)
if err != nil {
return nil, err
}
if !part {
// Assigned, and not yet part of what this generates. Nothing to put on the
// machine, which is different from an error: a node given the network module
// before it has an address is in exactly that state, briefly.
continue
}
resources = mine
resources = generated
}
// Now, and not before: a module whose resources are computed replaces them wholesale, and
@@ -989,7 +979,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
// this module's name, so they are applied, reported and removed exactly as anything else
// it declares.
given, err := FactsFrom(m, r, with)
given, err := FactsWithZonesInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix, with.Zones)
if err != nil {
return nil, err
}
@@ -2358,49 +2348,3 @@ func accountsFirst(resources []map[string]any) (accounts, rest []map[string]any)
}
return accounts, rest
}
// generatedHere is what each computed module's generator answers for this machine, by module —
// absent for a module whose machine is not yet part of what it generates — held to the rule every
// module is held to: no two modules on a machine declare one path, unit, name or package (novox/hq
// issue 190, step 5; ADR 0222).
//
// Resolution checks the catalogue's manifests, and a computed module's manifest has none of the
// resources it will declare — they exist only once its generator has answered for this machine,
// here — so a generated resource writing into another module's file was never seen. That is how
// the private network came to declare the container runtime's daemon file and service beside the
// runtime's own module. Asked once, so what is checked is exactly what is declared.
func (r Resolution) generatedHere(with Rendering) (map[string][]map[string]any, error) {
generated := map[string][]map[string]any{}
placed := make([]Manifest, 0, len(r.Modules))
for _, m := range r.Modules {
if m.Computed == "" {
placed = append(placed, m)
continue
}
generator, known := with.Generators[m.Computed]
if !known {
return nil, fmt.Errorf(
"%s says its resources are computed by %q, and this control plane has no %q",
m.Module, m.Computed, m.Computed)
}
resources, part, err := generator.Resources(r.Node)
if err != nil {
return nil, err
}
computed := m
computed.Resources = nil
if part {
generated[m.Module] = resources
computed.Resources = resources
}
placed = append(placed, computed)
}
if len(generated) == 0 {
return generated, nil // nothing resolution has not already judged
}
if problems := checkResources(placed); len(problems) > 0 {
return nil, fmt.Errorf("what the mesh computes for this machine collides with a module's own: %s",
strings.Join(problems, "; "))
}
return generated, nil
}
internal/catalogue/events.go
-32
View File
@@ -3,7 +3,6 @@ package catalogue
import (
"fmt"
"regexp"
"slices"
"strings"
)
@@ -160,34 +159,3 @@ func consumePattern(pattern string) error {
}
return nil
}
// The events a provider says about its consumers (novox/hq ADR 0224): a consumer it has failed
// without one success for minutes, and that consumer succeeding again or being withdrawn. The
// controller follows them from every module and `status` names a consumer failing until it recovers.
const (
ProvisionerFailing = "provisioner.failing"
ProvisionerRecovered = "provisioner.recovered"
)
// ProvisionerEvents are both, in the order they are said.
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered}
// EmitsAll is every event a module may publish: what it declares and, for a module that receives
// contributions — a provider, running a provisioner over them — the provider's standing events.
//
// **Derived, not declared**, because they are the mesh's rule about every provider rather than
// anything one module chose to say: a provider whose manifest forgot them would fail its consumers
// as silently as on 2026-10-05, with its announcement refused by the bus (novox/hq issue 179). Every
// grant of a module's publishing reads this, never the declared list alone.
func (m Manifest) EmitsAll() []string {
out := append([]string(nil), m.Emits...)
if len(m.Receives) == 0 {
return out
}
for _, e := range ProvisionerEvents {
if !slices.Contains(out, e) {
out = append(out, e)
}
}
return out
}
internal/catalogue/generated_collision_test.go
-84
View File
@@ -1,84 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// novox/hq issue 190, step 5 (ADR 0222): what the mesh computes for a module is held to the rule
// every module is — no two modules on a machine declare one path, unit, name or package. The
// private network once declared the container runtime's file and service beside the runtime's own
// module, and nothing refused it, because the collision check only ever saw catalogue manifests.
func runtimesOwn() Manifest {
return Manifest{Module: "docker", Resources: []map[string]any{
{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json", "into": "json",
"content": `{"live-restore": true}`},
{"id": "runtime", "type": "service", "unit": "docker.service", "state": "running",
"reload-on": []any{"daemon"}},
}}
}
func TestAGeneratedResourceCollidingWithAModulesIsRefused(t *testing.T) {
r := Resolution{Node: "workstation", Modules: []Manifest{
{Module: "mesh-network", Computed: "mesh-network", Provides: Offers("private-network")},
runtimesOwn(),
}}
_, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": reloading{}}})
if err == nil {
t.Fatal("a generated resource declaring the runtime's file beside the runtime's module was accepted")
}
for _, want := range []string{"mesh-network", "docker", "/etc/docker/daemon.json", "docker.service"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not name %s: %v", want, err)
}
}
}
func TestAGeneratedResourceBesideAModulesOwnIsComposed(t *testing.T) {
r := Resolution{Node: "workstation", Modules: []Manifest{
{Module: "mesh-network", Computed: "mesh-network", Provides: Offers("private-network")},
runtimesOwn(),
}}
gen := &fake{on: map[string]bool{"workstation": true}}
out, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": gen}})
if err != nil {
t.Fatalf("disjoint resources were refused: %v", err)
}
if fileNamed(out, "docker.daemon") == nil {
t.Fatalf("the runtime's own file is missing: %v", out)
}
// And a machine not on the network yet generates nothing, which collides with nothing.
if _, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": &fake{}}}); err != nil {
t.Fatalf("a machine off the network was refused: %v", err)
}
}
// The catalogue's container runtime module states the mesh's registry itself (ADR 0222).
func TestTheRuntimesModuleTrustsTheMeshsRegistry(t *testing.T) {
docker := catalogueManifest(t, "docker")
r := Resolution{Node: "workstation", Modules: []Manifest{docker}}
out, err := r.Declaration(Rendering{
SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"},
})
if err != nil {
t.Fatal(err)
}
daemon := fileNamed(out, "docker.daemon")
if daemon == nil || daemon["into"] != "json" {
t.Fatalf("the runtime's file is not written into: %v", daemon)
}
content, _ := daemon["content"].(string)
if !strings.Contains(content, `"insecure-registries": ["anchor.internal:5100"]`) ||
!strings.Contains(content, `"live-restore": true`) {
t.Fatalf("the runtime's file says %q", content)
}
out, err = r.Declaration(Rendering{})
if err != nil {
t.Fatal(err)
}
if content, _ := fileNamed(out, "docker.daemon")["content"].(string); strings.Contains(content, "insecure-registries") {
t.Fatalf("with no store on the network, the runtime is told %q", content)
}
}
internal/catalogue/hostname_seat_test.go
-98
View File
@@ -1,98 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// novox/hq ADR 0223 part 3: a machine's names are one seat's. The `hosts` module holding
// `node-hosts-file` became `hostname` holding `node-hostname`, which writes /etc/hostname beside the
// machine's own lines in /etc/hosts. The seat was renamed (ADR 0122), so what claims the old name — a
// manifest registered before the rename — still holds the one seat.
func withHostnameAlias(t *testing.T) {
t.Helper()
was := aliases
t.Cleanup(func() { aliases = was })
UseAliases(map[string]string{"node-hosts-file": "node-hostname"})
}
func TestTheHostsFilesFormerNameResolvesToTheHostnameSeat(t *testing.T) {
if _, known := SeatNamed("node-hostname"); !known {
t.Fatal("node-hostname is not in the mesh's set")
}
withHostnameAlias(t)
seat, known := SeatNamed("node-hosts-file")
if !known || seat.Name != "node-hostname" || seat.Scope != ScopeNode {
t.Fatalf("the former name did not resolve: %+v %v", seat, known)
}
var verbs []string
for _, v := range seat.Serves {
verbs = append(verbs, v.Name)
}
if strings.Join(verbs, " ") != "entries add remove" {
t.Errorf("the renamed seat serves %v; its verbs are unchanged", verbs)
}
}
// One seat under either name: the module registered before the rename and the one after cannot both
// hold it on one machine.
func TestTheOldAndTheNewClaimantAreOneSeatOnAMachine(t *testing.T) {
withHostnameAlias(t)
cat := shelf(
mod("hosts", nil, nil, nil, Claim{Name: "node-hosts-file"}),
mod("hostname", nil, nil, nil, Claim{Name: "node-hostname"}),
)
_, err := Resolve(cat, []string{"hosts", "hostname"}, workstation(), World{})
if err == nil || !strings.Contains(err.Error(), "node-hostname") {
t.Errorf("hosts and hostname both held the machine's names on one machine: %v", err)
}
if _, err := Resolve(cat, []string{"hosts"}, workstation(), World{}); err != nil {
t.Errorf("a machine still assigned hosts under the old name does not resolve: %v", err)
}
}
// The catalogue's module: /etc/hostname is the operator's `hostname` setting. Without it the module is
// left out, naming the key — the mesh never renames a machine on its own — and a mesh-wide setting
// naming ${machine:name} gives every machine its mesh name.
func TestTheMachinesNameIsItsSetting(t *testing.T) {
cat := map[string]Manifest{"hostname": catalogueManifest(t, "hostname")}
got, err := Resolve(cat, []string{"hostname"}, Node{Name: "ace", At: "ace.internal"}, World{})
if err != nil {
t.Fatal(err)
}
machines := map[string]string{"ace.internal": "10.42.0.2"}
nameOf := func(settings SettingsBy) (string, string) {
t.Helper()
composed, err := got.Compose(Rendering{Names: machines, Machines: machines, Suffix: "internal",
Settings: settings})
if err != nil {
t.Fatal(err)
}
if why := composed.LeftOut["hostname"]; why != "" {
return "", why
}
for _, r := range composed.Resources {
if r["path"] == "/etc/hostname" {
return r["content"].(string), ""
}
}
t.Fatal("no /etc/hostname composed")
return "", ""
}
if _, why := nameOf(nil); !strings.Contains(why, "hostname") {
t.Errorf("with no setting the machine's name was written, or left out for another reason: %q", why)
}
if name, why := nameOf(SettingsBy{"hostname": {{From: "ace", Values: map[string]any{"hostname": "Ace"}}}}); name != "Ace\n" {
t.Errorf("the operator's name for the machine gave %q (%s)", name, why)
}
mesh := Layer{From: "the mesh", Values: map[string]any{"hostname": "${machine:name}"}}
if name, why := nameOf(SettingsBy{"hostname": {mesh}}); name != "ace\n" {
t.Errorf("a mesh-wide ${machine:name} gave %q (%s)", name, why)
}
node := Layer{From: "ace", Values: map[string]any{"hostname": "Ace"}}
if name, why := nameOf(SettingsBy{"hostname": {mesh, node}}); name != "Ace\n" {
t.Errorf("a machine's own name over the mesh-wide one gave %q (%s)", name, why)
}
}
internal/catalogue/hosts_region_test.go
+106
View File
@@ -0,0 +1,106 @@
package catalogue_test
import (
"reflect"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/overlay"
)
// novox/hq issue 128, held where the host will see it: the shipped networking module's names,
// through the whole composition, and not only through FactsInto.
//
// Composition prefixes a fact's id with the module that asked for it and passes everything else
// through; a step that dropped `into` on the way would send the region as a whole file, and the
// host would write the machine's hosts file over again with every unit test above still green.
// onTheNetwork stands in for the overlay's generator: the node is part of the private network,
// and what the generator writes is not what is under test here.
type onTheNetwork struct{}
func (onTheNetwork) Resources(string) ([]map[string]any, bool, error) {
return []map[string]any{{"id": "overlay-config", "type": "file",
"path": "/etc/wireguard/mesh0.conf", "mode": "0600", "content": "[Interface]\n"}}, true, nil
}
func TestTheHostsRegionArrivesAsTheHostWillReadIt(t *testing.T) {
shelf := provided(t)
// A resolver restarting on the names another module put on the machine, and one resource it
// only runs at start — neither of which composition has any business changing.
resolver, err := catalogue.ParseManifest([]byte(`{
"module": "resolver", "version": "1", "requires": ["mesh-addressing"],
"resources": [
{"id": "seed", "type": "file", "path": "/etc/resolver/seed", "mode": "0644",
"content": "seed\n", "at": "start"},
{"id": "daemon", "type": "service", "unit": "resolver.service", "state": "running",
"restart-on": ["seed", "mesh-wireguard.fact-node-names"]}
]}`))
if err != nil {
t.Fatal(err)
}
shelf[resolver.Module] = resolver
got, err := catalogue.Resolve(shelf, []string{overlay.Domain, "resolver"},
catalogue.Node{Name: "homer", At: "homer.internal"}, catalogue.World{})
if err != nil {
t.Fatal(err)
}
names := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
out, err := got.Declaration(catalogue.Rendering{
Names: names, Machines: names, Suffix: "internal",
Generators: map[string]catalogue.Generator{overlay.Name: onTheNetwork{}},
})
if err != nil {
t.Fatal(err)
}
ids := map[string]map[string]any{}
for _, r := range out {
ids[r["id"].(string)] = r
}
hosts := ids[overlay.Name+".fact-node-names"]
if hosts == nil {
t.Fatalf("no names reached the machine; the declaration has %v", keys(ids))
}
if hosts["path"] != "/etc/hosts" || hosts["into"] != "block" {
t.Fatalf("the hosts file is not written into as a region: %v", hosts)
}
content := hosts["content"].(string)
if !strings.Contains(content, "10.42.0.1\thomer.internal\thomer\t# this machine\n") {
t.Errorf("the region does not name the machine:\n%s", content)
}
for _, floor := range []string{"Generated by the mesh", "localhost", "127.0.1.1"} {
if strings.Contains(content, floor) {
t.Errorf("the region carries %q, which is the machine's:\n%s", floor, content)
}
}
// The resolver's reference to it still names a resource the host will be sent.
daemon := ids["resolver.daemon"]
if daemon == nil {
t.Fatalf("the resolver's service was not composed: %v", keys(ids))
}
for _, named := range daemon["restart-on"].([]any) {
if ids[named.(string)] == nil {
t.Errorf("the resolver restarts on %v, which is nothing the host is sent", named)
}
}
if !reflect.DeepEqual(daemon["restart-on"], []any{"resolver.seed", overlay.Name + ".fact-node-names"}) {
t.Errorf("restart-on is %v", daemon["restart-on"])
}
// And a resource's own `at` passes through as the manifest wrote it.
if seed := ids["resolver.seed"]; seed == nil || seed["at"] != "start" {
t.Errorf("a resource's at did not survive composition: %v", seed)
}
}
func keys(m map[string]map[string]any) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
return out
}
internal/catalogue/identity.go
+9 -148
View File
@@ -3,7 +3,6 @@ package catalogue
import (
"fmt"
"regexp"
"sort"
"strings"
)
@@ -57,36 +56,13 @@ func ConsumerIdentity(node, module string) string {
return IdentityPrefix + clean(node) + "_" + clean(module)
}
// IdentityBound is the longest consumer identity one provision's backend keeps, and what keeps it
// (novox/hq ADR 0049, refined by ADR 0225). Max zero means no bound: the provision keeps no name
// derived from its consumer, or keeps one in something with no limit the mesh need respect.
type IdentityBound struct {
// Max is the longest identity that backend keeps, in characters; zero for none.
Max int `json:"max,omitempty"`
// In is what keeps it, in words a refusal can quote: "an S3 access key", "a PostgreSQL role".
In string `json:"in,omitempty"`
}
// identityLimit is the shortest identifier limit among the systems these names reach: an S3 access
// key's 20 (novox/hq 04-ISSUES/010). PostgreSQL keeps 63 and MinIO 20, so 20 is the one that binds —
// the comment used to name PostgreSQL and was wrong. A name over it is refused, with the remedy a
// short slug (ADR 0049), not silently cut to fit.
const identityLimit = 20
// Bounded is whether this bound refuses anything.
func (b IdentityBound) Bounded() bool { return b.Max > 0 }
// DefaultIdentityLimit is the bound on a provision whose provider receives its consumers and does
// not say how long a name it keeps: an S3 access key's 20 (novox/hq 04-ISSUES/010, 034), the
// tightest backend the mesh has met. It was the bound on every provision until ADR 0225; it stays
// the bound on any that has not said otherwise, because a provider that is told each consumer's
// identity may create a name from it in a backend nobody has measured.
const DefaultIdentityLimit = 20
// DefaultIdentityBound is DefaultIdentityLimit, said as a bound.
var DefaultIdentityBound = IdentityBound{Max: DefaultIdentityLimit,
In: "a backend that has not said its limit (the tightest known, an S3 access key's)"}
// identityLimit is the bound CheckIdentity applies, for a caller that does not know which provision
// the identity is for.
const identityLimit = DefaultIdentityLimit
// CheckIdentity refuses an identity that would not fit the tightest backend the mesh knows. A caller
// that knows the provision uses CheckIdentityWithin and that provision's own bound (ADR 0225).
// CheckIdentity refuses an identity that would not fit the tightest backend a consumer reaches.
//
// **Truncation is not an error in most of these systems** — a name past the limit is cut to fit and
// the statement succeeds, so two consumers agreeing for the first N bytes would become one login
@@ -94,127 +70,12 @@ const identityLimit = DefaultIdentityLimit
// name and is the only thing that can choose another. The remedy is a first-class one: give the
// module a short `slug` (ADR 0049), or shorten the machine's name.
func CheckIdentity(node, module string) error {
return CheckIdentityWithin(node, module, IdentityBound{Max: identityLimit, In: "a backend (an S3 access key)"})
}
// CheckIdentityWithin refuses an identity that would not fit one provision's bound, and accepts any
// identity for a provision with none (novox/hq ADR 0225).
func CheckIdentityWithin(node, module string, bound IdentityBound) error {
if !bound.Bounded() {
return nil
}
got := ConsumerIdentity(node, module)
if len(got) <= bound.Max {
if len(got) <= identityLimit {
return nil
}
return fmt.Errorf(
"%s on %s is identified as %q, %d characters where %s keeps %d — "+
"%s on %s is identified as %q, %d characters where a backend (an S3 access key) keeps %d — "+
"give the module a shorter `slug` or shorten the machine's name",
module, node, got, len(got), bound.In, bound.Max)
}
// Overflow is one consumer whose identity does not fit the provision it requires: left out of its
// provider's grants and reported, never a reason to refuse the provider's machine (ADR 0225).
type Overflow struct {
// Provision is what was required, Provider the machine answering it.
Provision string `json:"provision"`
Provider string `json:"provider"`
// Consumer is the machine, Module the module on it that required it.
Consumer string `json:"consumer"`
Module string `json:"module"`
// Identity is the name the mesh derived, and Bound what it overflows.
Identity string `json:"identity"`
Bound IdentityBound `json:"bound"`
}
func (o Overflow) String() string {
return fmt.Sprintf("%s on %s requires %s from %s and is identified as %q, %d characters where %s "+
"keeps %d — left out of %s's grants until the module's `slug` is shorter",
o.Module, o.Consumer, o.Provision, o.Provider, o.Identity, len(o.Identity), o.Bound.In,
o.Bound.Max, o.Provider)
}
// Overflowing is every requirement of this machine's modules whose identity overflows the bound of
// the provision answering it. The same judgement the provider's composition makes before it grants
// (grantsFor), made from the consumer's side so `status` can say it about every machine.
func (r Resolution) Overflowing() []Overflow {
slugs := map[string]string{}
for _, m := range r.Modules {
slugs[m.Module] = m.Slug
}
var out []Overflow
seen := map[string]bool{}
for _, n := range r.Needs {
if n.ByRecord || !n.Identity.Bounded() {
continue
}
source := IdentitySource(slugs[n.For], n.For)
if CheckIdentityWithin(r.Node, source, n.Identity) == nil {
continue
}
key := n.Name + "\x00" + n.From + "\x00" + n.For
if seen[key] {
continue // one line per requirement, however many local names it has
}
seen[key] = true
out = append(out, Overflow{Provision: n.Name, Provider: n.From, Consumer: r.Node, Module: n.For,
Identity: ConsumerIdentity(r.Node, source), Bound: n.Identity})
}
sort.Slice(out, func(i, j int) bool {
if out[i].Module != out[j].Module {
return out[i].Module < out[j].Module
}
return out[i].Provision < out[j].Provision
})
return out
}
// DefaultLongestMachine is the machine name the catalogue check judges identities on when it is not
// told one: the longest name of the mesh this catalogue is written for, so a catalogue that passes
// passes on every machine that mesh has. `module check --longest-machine-name` says another mesh's;
// a mesh that names a longer machine raises this in the same change (ADR 0225).
const DefaultLongestMachine = 6
// IdentityProblems is every module whose identity would overflow a provision it wants, on a machine
// whose name is `longestMachine` characters — judged before merge, over the catalogue alone, so the
// pull request that introduces an overflow is the one refused (novox/hq ADR 0225, issue 263). A
// provision no module in the shelf offers is not judged: its bound is not known here.
func IdentityProblems(shelf Shelf, longestMachine int) []string {
offeredBy := map[string][]string{}
for _, name := range shelfOrder(shelf) {
for _, o := range shelf[name].Offers() {
offeredBy[o] = append(offeredBy[o], name)
}
}
machine := strings.Repeat("n", longestMachine)
var problems []string
for _, name := range shelfOrder(shelf) {
m := shelf[name]
source := IdentitySource(m.Slug, m.Module)
for _, want := range m.Wants() {
// The tightest bound among the modules offering it: whichever one answers on a given
// machine, the identity has to fit it.
tightest, by := IdentityBound{}, ""
for _, provider := range offeredBy[want] {
if provider == name {
continue // a module answering its own requirement is not its own consumer
}
b := shelf[provider].IdentityBoundOf(want)
if b.Bounded() && (!tightest.Bounded() || b.Max < tightest.Max) {
tightest, by = b, provider
}
}
if CheckIdentityWithin(machine, source, tightest) == nil {
continue
}
got := ConsumerIdentity(machine, source)
problems = append(problems, fmt.Sprintf(
"%s wants %s, and %s keeps its consumers' identities in %s of at most %d characters: "+
"on a machine with a %d-character name it is identified as %q, %d — give %s a "+
"`slug` of at most %d characters",
name, want, by, tightest.In, tightest.Max, longestMachine, got, len(got), name,
tightest.Max-len(IdentityPrefix)-longestMachine-1))
}
}
return problems
module, node, got, len(got), identityLimit)
}
internal/catalogue/identity_bound_test.go
-190
View File
@@ -1,190 +0,0 @@
package catalogue
import (
"encoding/json"
"strings"
"testing"
)
// Each test names the decision it defends: novox/hq ADR 0225, which refines ADR 0049 after issue 263.
// The provision a module requires sets the bound on its identity, not the tightest backend anywhere.
func TestABoundIsTheProvisionsOwn(t *testing.T) {
store := Manifest{Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}
database := Manifest{Module: "db", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh,
Identity: &OfferIdentity{Max: 63, In: "a PostgreSQL role"}}},
Receives: map[string]string{"postgres-database": "/var/lib/mesh/db/mesh.json"}}
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 || b.In != "an S3 access key" {
t.Errorf("an object store's stated bound was not taken: %+v", b)
}
if b := database.IdentityBoundOf("postgres-database"); b.Max != 63 {
t.Errorf("a database's stated bound was not taken: %+v", b)
}
// mesh_workstation_keycloak is 25: refused by the object store, accepted by the database.
if CheckIdentityWithin("workstation", "keycloak", store.IdentityBoundOf("s3-bucket")) == nil {
t.Error("a 25-character identity fit a 20-character access key")
}
if err := CheckIdentityWithin("workstation", "keycloak", database.IdentityBoundOf("postgres-database")); err != nil {
t.Errorf("a database consumer paid the object store's limit: %v", err)
}
}
// What an offer leaves unsaid follows from whether its provider can keep a name at all.
func TestAnUnstatedBoundFollowsWhatTheProviderIsTold(t *testing.T) {
// Told nothing about its consumers — no receives, nothing served from their identity: no bound.
// The resolver provision is exactly this, and its consumers paid an object store's limit (263).
resolver := Manifest{Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}}
if b := resolver.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
t.Errorf("a provision that is told nothing of its consumers bounds them: %+v", b)
}
// Told each consumer and silent about its backend: the old global bound, not none.
told := Manifest{Module: "told", Provides: []Offer{{Name: "thing", Scope: ScopeMesh}},
Receives: map[string]string{"thing": "/var/lib/mesh/told/mesh.json"}}
if b := told.IdentityBoundOf("thing"); b.Max != DefaultIdentityLimit {
t.Errorf("a provider told its consumers and silent about its backend is not held to %d: %+v",
DefaultIdentityLimit, b)
}
// Serving a value built from the identity is being told it, too.
serving := Manifest{Module: "serving", Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}},
Serves: map[string]map[string]any{"bucket": {"name": "b-${consumer:as:dns}"}}}
if b := serving.IdentityBoundOf("bucket"); b.Max != DefaultIdentityLimit {
t.Errorf("a provider deriving a name from its consumers is not bounded: %+v", b)
}
// And `false` says it outright, even for a provider that receives.
routes := Manifest{Module: "routes", Provides: []Offer{{Name: "route", Scope: ScopeMesh,
Identity: &OfferIdentity{None: true}}},
Receives: map[string]string{"route": "/var/lib/mesh/routes/mesh.json"}}
if b := routes.IdentityBoundOf("route"); b.Bounded() {
t.Errorf("`identity: false` still bounds: %+v", b)
}
}
// The field reads as written and writes back the same, and refuses what says nothing.
func TestAnOffersIdentityIsParsedStrictly(t *testing.T) {
for _, raw := range []string{
`{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}`,
`{"name":"wildcard-resolution","scope":"mesh","identity":false}`,
`{"name":"redis-cache","scope":"mesh","identity":{"in":"a Redis ACL user"}}`,
} {
var o Offer
if err := json.Unmarshal([]byte(raw), &o); err != nil {
t.Fatalf("%s: %v", raw, err)
}
back, err := json.Marshal(o)
if err != nil || string(back) != raw {
t.Errorf("did not round-trip:\n%s\n%s (%v)", raw, back, err)
}
}
for _, raw := range []string{
`{"name":"x","identity":true}`,
`{"name":"x","identity":{"max":20,"in":"y","most":3}}`,
} {
var o Offer
if err := json.Unmarshal([]byte(raw), &o); err == nil {
t.Errorf("accepted %s", raw)
}
}
bad := Manifest{Module: "bad", Version: "1", Provides: []Offer{
{Name: "unsaid", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 20}},
{Name: "tiny", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 4, In: "nothing usable"}},
}}
raw, err := json.Marshal(bad)
if err != nil {
t.Fatal(err)
}
_, err = ParseManifest(raw)
if err == nil || !strings.Contains(err.Error(), "without saying what keeps them") ||
!strings.Contains(err.Error(), "the shortest the mesh makes") {
t.Fatalf("a bound with no `in`, or too short for any identity, was accepted: %v", err)
}
}
// Refused before merge: the catalogue check judges each module's identity, on the longest machine
// name, against the bound of every provision it wants — and names the module and the slug to set.
func TestTheCatalogueCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
shelf := Shelf{
"objects": {Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}},
"photoalbum": {Module: "photoalbum", Requires: []string{"s3-bucket"}},
"files": {Module: "files", Requires: []string{"s3-bucket"}},
}
problems := IdentityProblems(shelf, 6)
if len(problems) != 1 || !strings.Contains(problems[0], "photoalbum wants s3-bucket") ||
!strings.Contains(problems[0], `"mesh_nnnnnn_photoalbum", 22`) ||
!strings.Contains(problems[0], "`slug` of at most 8 characters") {
t.Fatalf("one overflow, named with its remedy, was expected: %q", problems)
}
// A longer machine name refuses more: the check is about the mesh's machines, not one.
if got := IdentityProblems(shelf, 10); len(got) != 2 {
t.Fatalf("on a 10-character name both overflow (mesh_nnnnnnnnnn_files is 21): %q", got)
}
// And a slug is the remedy it names.
album := shelf["photoalbum"]
album.Slug = "album"
shelf["photoalbum"] = album
if got := IdentityProblems(shelf, 6); len(got) != 0 {
t.Fatalf("a slug that fits is still refused: %q", got)
}
}
// Tonight's case (issue 263): networkmanager, no slug, requiring the mesh's resolver provision on a
// machine with a six-character name. Under ADR 0049's one bound it was refused, and its provider's
// whole machine with it; the resolver keeps no name, so it is not refused at all.
func TestARequirementOnAKeylessProvisionComposesWithALongName(t *testing.T) {
shelf := Shelf{
"resolver": {Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}},
"networkmanager": {Module: "networkmanager", Requires: []string{"wildcard-resolution"}},
}
if CheckIdentity("laptop", "networkmanager") == nil {
t.Fatal("the regression is not reproduced: mesh_laptop_networkmanager fits the old global bound")
}
if got := IdentityProblems(shelf, 6); len(got) != 0 {
t.Fatalf("a requirement on a keyless provision was refused for its length: %q", got)
}
r := Resolution{Node: "laptop", Modules: []Manifest{shelf["networkmanager"]},
Needs: []Needed{{Name: "wildcard-resolution", From: "anchor", For: "networkmanager",
Identity: shelf["resolver"].IdentityBoundOf("wildcard-resolution")}}}
if got := r.Overflowing(); len(got) != 0 {
t.Fatalf("a keyless requirement is reported as overflowing: %+v", got)
}
}
// The consumer's side of the same judgement the provider's composition makes: what `status` says.
func TestAnOverflowingRequirementIsNamedFromTheConsumersSide(t *testing.T) {
bound := IdentityBound{Max: 20, In: "an S3 access key"}
r := Resolution{Node: "laptop",
Modules: []Manifest{{Module: "photoalbum"}, {Module: "files"}, {Module: "gallery", Slug: "gal"}},
Needs: []Needed{
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Identity: bound},
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Local: "second", Identity: bound},
{Name: "s3-bucket", From: "anchor", For: "files", Identity: bound},
{Name: "s3-bucket", From: "anchor", For: "gallery", Identity: bound},
{Name: "licence", From: "records", For: "photoalbum", ByRecord: true, Identity: bound},
}}
got := r.Overflowing()
if len(got) != 1 || got[0].Module != "photoalbum" || got[0].Identity != "mesh_laptop_photoalbum" ||
got[0].Provider != "anchor" {
t.Fatalf("one overflow, once, was expected: %+v", got)
}
if said := got[0].String(); !strings.Contains(said, "an S3 access key keeps 20") ||
!strings.Contains(said, "slug") {
t.Fatalf("the overflow does not say what keeps it or the remedy: %s", said)
}
}
// An identity served as a DNS label is never truncated, so the offer's bound must keep it in one.
func TestAnIdentityServedAsADNSLabelIsBoundedToOne(t *testing.T) {
serves := map[string]map[string]any{"bucket": {"name": "${consumer:as:dns}"}}
wide := Manifest{Module: "wide", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh,
Identity: &OfferIdentity{Max: 255, In: "a client id"}}}}
if got := CheckServes(wide); len(got) != 1 || !strings.Contains(got[0], "a label keeps 63") {
t.Fatalf("a 255-character bound on a DNS label passed: %q", got)
}
unsaid := Manifest{Module: "unsaid", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}}}
if got := CheckServes(unsaid); len(got) != 0 {
t.Fatalf("the default bound (20) on a DNS label was refused: %q", got)
}
}
internal/catalogue/manifest.go
+9 -107
View File
@@ -8,7 +8,6 @@ package catalogue
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"regexp"
"sort"
@@ -154,84 +153,6 @@ type Offer struct {
// a seat's holder gated by the machine's graphical session, and pulling one in for whatever asked
// is the misassignment research 026 found. Unmet, the requirement is refused naming who could.
Reach string `json:"reach,omitempty"`
// Identity is the longest consumer identity this provision's backend keeps (novox/hq ADR 0225):
// `{"max": 63, "in": "a PostgreSQL role"}`, or `false` for a provision that keeps no name derived
// from its consumer. Unsaid, the mesh assumes the tightest backend it knows when the provider is
// told its consumers, and no bound when it is not — see IdentityBoundOf.
Identity *OfferIdentity `json:"identity,omitempty"`
}
// OfferIdentity is what an offer says about the names its backend keeps for its consumers.
type OfferIdentity struct {
// None is set by `"identity": false`: the provision keeps no name derived from its consumer.
None bool
// Max is the longest identity kept, in characters; zero with In set means no limit worth stating.
Max int
// In is what keeps it, for a refusal to quote.
In string
}
// UnmarshalJSON accepts `false` or `{"max": N, "in": "..."}`.
func (i *OfferIdentity) UnmarshalJSON(raw []byte) error {
var flag bool
if err := json.Unmarshal(raw, &flag); err == nil {
if flag {
return errors.New("an offer's identity is false (it keeps no name) or {max, in}; true says nothing")
}
*i = OfferIdentity{None: true}
return nil
}
var full struct {
Max int `json:"max,omitempty"`
In string `json:"in"`
}
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return fmt.Errorf("an offer's identity is false or {max, in}: %w", err)
}
*i = OfferIdentity{Max: full.Max, In: full.In}
return nil
}
// MarshalJSON writes it back in the form it was written.
func (i OfferIdentity) MarshalJSON() ([]byte, error) {
if i.None {
return []byte("false"), nil
}
return json.Marshal(struct {
Max int `json:"max,omitempty"`
In string `json:"in"`
}{i.Max, i.In})
}
// IdentityBoundOf is the bound this module's offer of a provision puts on its consumers' identities
// (novox/hq ADR 0225).
//
// **What an offer says, it gets.** Where it says nothing, the bound follows from whether the
// provider can keep a name at all: a provider that receives the provision, or serves its consumers
// a value built from their identity, is told who each consumer is and may create a name from it in
// a backend nobody measured — so it keeps the old global bound, DefaultIdentityBound. One that does
// neither is told nothing about its consumers and keeps nothing of them: no bound. The resolver
// provision is that case, and its consumers paid an object store's limit until this (issue 263).
func (m Manifest) IdentityBoundOf(provision string) IdentityBound {
for _, o := range m.Provides {
if o.Name != provision || o.Identity == nil {
continue
}
if o.Identity.None {
return IdentityBound{}
}
return IdentityBound{Max: o.Identity.Max, In: o.Identity.In}
}
if _, receives := m.Receives[provision]; receives {
return DefaultIdentityBound
}
if served, err := json.Marshal(m.Serves[provision]); err == nil &&
bytes.Contains(served, []byte("${consumer:as")) {
return DefaultIdentityBound
}
return IdentityBound{}
}
// MachineReach is whether a provision is usable only on its provider's own machine.
@@ -285,21 +206,20 @@ func (o *Offer) UnmarshalJSON(raw []byte) error {
Scope string `json:"scope,omitempty"`
Credential *OfferCredential `json:"credential,omitempty"`
Reach string `json:"reach,omitempty"`
Identity *OfferIdentity `json:"identity,omitempty"`
}
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach, identity}: %w", err)
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach}: %w", err)
}
o.Name, o.Scope, o.Credential, o.Reach, o.Identity = full.Name, full.Scope, full.Credential, full.Reach, full.Identity
o.Name, o.Scope, o.Credential, o.Reach = full.Name, full.Scope, full.Credential, full.Reach
return nil
}
// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
// went through the mesh comes out looking like the one that went in.
func (o Offer) MarshalJSON() ([]byte, error) {
if o.Scope == "" && o.Credential == nil && o.Reach == "" && o.Identity == nil {
if o.Scope == "" && o.Credential == nil && o.Reach == "" {
return json.Marshal(o.Name)
}
return json.Marshal(struct {
@@ -307,8 +227,7 @@ func (o Offer) MarshalJSON() ([]byte, error) {
Scope string `json:"scope,omitempty"`
Credential *OfferCredential `json:"credential,omitempty"`
Reach string `json:"reach,omitempty"`
Identity *OfferIdentity `json:"identity,omitempty"`
}{o.Name, o.Scope, o.Credential, o.Reach, o.Identity})
}{o.Name, o.Scope, o.Credential, o.Reach})
}
// Manifest is everything a module says about itself.
@@ -318,10 +237,9 @@ type Manifest struct {
// Slug is a short identifier the mesh uses in place of the module name when it derives a
// consumer's login (novox/hq ADR 0049). Optional: a module with a short name needs none. It
// exists because `mesh_<node>_<module>` must fit the bound of every provision the module
// requires — an S3 access key's 20 characters is the tightest — and a long module name would
// overflow it (ADR 0225: the bound is the provision's own, and a keyless one has none). A person
// choosing `kc` for keycloak keeps the identity legible where a hash would not.
// exists because `mesh_<node>_<module>` must fit the tightest backend a consumer reaches — an S3
// access key is 20 characters — and a long module name would overflow it. A person choosing
// `kc` for keycloak keeps the identity legible where a hash would not.
Slug string `json:"slug,omitempty"`
// Provides are the names other modules may require. A module always provides its own name;
@@ -1347,10 +1265,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
}
// A slug is a short identifier the mesh derives a login from (novox/hq ADR 0049). The same
// charset as a name; its length is judged against the bound of each provision it wants on the
// longest machine name by the catalogue check (IdentityProblems, ADR 0225), and against the
// machine it is on when its provider grants it — a slug that is fine on one machine's short name
// can overflow another's.
// charset as a name; its length is checked against a backend's limit at assignment, where the
// node it joins is known — a slug that is fine on one machine's short name can overflow another's.
if m.Slug != "" && !name.MatchString(m.Slug) {
problems = append(problems, fmt.Sprintf(
"%q is not a usable slug: lower-case letters, digits, dashes and dots", m.Slug))
@@ -1387,20 +1303,6 @@ func ParseManifest(raw []byte) (Manifest, error) {
if !name.MatchString(p) {
problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
}
// A bound says what keeps the name, so the refusal it causes can say it (ADR 0225); and it
// leaves room for the shortest identity the mesh makes, or it would refuse every consumer.
if id := offer.Identity; id != nil && !id.None {
if strings.TrimSpace(id.In) == "" {
problems = append(problems, fmt.Sprintf(
"%s bounds the identities of %s's consumers without saying what keeps them: `in`",
m.Module, p))
}
if shortest := len(IdentityPrefix) + 3; id.Max < 0 || id.Max > 0 && id.Max < shortest {
problems = append(problems, fmt.Sprintf(
"%s bounds %s's consumers' identities at %d characters, and the shortest the mesh "+
"makes is %d", m.Module, p, id.Max, shortest))
}
}
if offer.Credential != nil {
own, declared := m.OwnSecrets[offer.Credential.Own]
switch {
internal/catalogue/provided_test.go
+5 -7
View File
@@ -50,14 +50,12 @@ func TestTheShippedNetworkingModulesResolveOnTheirOwn(t *testing.T) {
}
}
// **The network writes no names** (novox/hq ADR 0199): /etc/hosts is the hosts seat holder's
// file, and the mesh's resolver answers the machines' names. No fact of the network's module
// may name that file.
// **The names come WITH the network now, not from a third module.** Being on the private
// network is what gives a machine a name, so the provider asks for the node-names fact and
// there is nothing else to bring in. A module that ran nothing used to be here.
for _, m := range got.Modules {
for name, f := range m.Facts {
if m.Module == overlay.Name && f.Path == "/etc/hosts" {
t.Fatalf("the network's provider still writes /etc/hosts, as its %q fact", name)
}
if m.Module == overlay.Name && m.Facts["node-names"].Path == "" {
t.Fatalf("the network's provider does not ask for the names: %+v", m.Facts)
}
}
}
internal/catalogue/resolve.go
+13 -116
View File
@@ -194,11 +194,6 @@ type Needed struct {
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
// licence's manager; empty for every consumer.
Manager bool
// Identity is the longest consumer identity the answering provision keeps (novox/hq ADR 0225),
// from the provider's own offer: what the mesh judges this consumer's identity against, on the
// consumer's side for `status` and on the provider's before it grants. No bound for a provision
// answered by a record, which keeps no name of anybody's.
Identity IdentityBound
}
// Refusal is why a set of assignments cannot become a declaration.
@@ -346,7 +341,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
// trust boundary the moment both ends are containers**, and treating it as one gave the
// commonest arrangement of all — a service and its database on one node — the weakest
// handling, silently.
if satisfied[want] && !isModule(catalogue, want) && !answeredElsewhere(want, node, world, brokered) {
if satisfied[want] && !isModule(catalogue, want) {
here := func(name string) bool { return chosen[name] || assignedHere[name] }
local := providersHere(catalogue, here, want)
// Which of them it matters to choose between. A plain capability — a shell, a display
@@ -408,7 +403,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
needs = append(needs, Needed{
Name: want, From: node.Name, At: at,
Serves: servedByOne(by, want), For: because[want],
SharedOwn: sharedByOne(by, want), Identity: by.IdentityBoundOf(want)})
SharedOwn: sharedByOne(by, want)})
} else if served := servedByOne(by, want); len(served) > 0 {
// Answered here with no credential to mint, but the provider serves facts the
// consumer cannot guess — a port, a model name — and so still needs a binding.
@@ -452,15 +447,11 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
return
}
shared := ""
// A provider whose definition is not in hand is held to the tightest bound the
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
bound := DefaultIdentityBound
if pm, known := catalogue[p.Module]; known {
shared, _ = pm.SharedCredentialOf(want)
bound = pm.IdentityBoundOf(want)
}
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At,
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound})
Serves: p.Serves, For: because[want], SharedOwn: shared})
}
switch {
case world.Unchecked:
@@ -776,27 +767,16 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
var problems []string
var held []Held
// onRecord is every recorded holder of a seat, if a handover ever named one: one for most seats,
// and as many as were added for a replicated one (novox/hq ADR 0223).
onRecord := func(claim, scope string) []Held {
var out []Held
// onRecord is the recorded holder of a seat, if a handover ever named one.
onRecord := func(claim, scope string) (Held, bool) {
for _, h := range holdings {
hs, ok := SeatNamed(h.Claim)
cs, cok := SeatNamed(claim)
if ok && cok && hs.Name == cs.Name && h.Scope == scope {
out = append(out, h)
return h, true
}
}
return out
}
// recordedHere says this node's module is one of a seat's holders on record.
recordedHere := func(claim, scope, module string) bool {
for _, rec := range onRecord(claim, scope) {
if rec.Node == node.Name && rec.Module == module {
return true
}
}
return false
return Held{}, false
}
byScope := map[string]map[string]string{} // scope → claim → module
@@ -807,35 +787,21 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
// the seat but is not the one on record is eligible, and that is all: it is not a second
// holder, so it is not refused, and it does not hold (novox/hq ADR 0131). This is what
// lets the next holder stand beside the current one until the seat is handed over.
if recs := onRecord(c.Name, scope); len(recs) > 0 {
// **A seat held once is on record once** (novox/hq ADR 0223). Only a replicated seat
// may have several holders on record; several for any other seat is a store that
// disagrees with the mesh's definition of the role, and it is refused, named, rather
// than letting two machines answer for what the mesh has one of.
if s, known := SeatNamed(c.Name); known && !s.Replicated && len(recs) > 1 {
problems = append(problems, fmt.Sprintf(
"%q is on record as held by %d assignments, and it is held once per %s — "+
"`seat %s --to <node>/<module>` records one", c.Name, len(recs), scope, c.Name))
continue
}
if !recordedHere(c.Name, scope, m.Module) {
if rec, recorded := onRecord(c.Name, scope); recorded {
if rec.Node != node.Name || rec.Module != m.Module {
continue
}
}
if byScope[scope] == nil {
byScope[scope] = map[string]string{}
}
// **One seat under either of its names** (novox/hq ADR 0122): a manifest registered before
// a rename claims the former name, and one written after it the current — two claimants of
// one seat, compared by the seat they resolve to and not by how each spelled it.
seat := canonicalSeat(c.Name)
if other, taken := byScope[scope][seat]; taken {
if other, taken := byScope[scope][c.Name]; taken {
problems = append(problems, fmt.Sprintf(
"%s and %s both claim %q, and only one thing may hold it per %s",
other, m.Module, seat, scope))
other, m.Module, c.Name, scope))
continue
}
byScope[scope][seat] = m.Module
byScope[scope][c.Name] = m.Module
held = append(held, Held{Claim: c.Name, Scope: scope, Node: node.Name,
Module: m.Module, Site: node.Site})
}
@@ -844,17 +810,11 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
// And against the rest of the mesh, for the scopes that reach past this machine.
for _, h := range held {
for _, e := range elsewhere {
if e.Node == node.Name || canonicalSeat(e.Claim) != canonicalSeat(h.Claim) || e.Scope != h.Scope {
if e.Node == node.Name || e.Claim != h.Claim || e.Scope != h.Scope {
continue
}
switch h.Scope {
case ScopeMesh:
// **A holder on record is never a second claimant** (novox/hq ADR 0223). Records are
// the mesh's settled answer: one for most seats, several only for a replicated seat,
// each added by an act. Two holders here are two records, and both hold.
if recordedHere(h.Claim, h.Scope, h.Module) {
continue
}
// Both claim and nobody is on record, or this refusal could not have happened.
// The remedy is the handover that records the holder (novox/hq ADR 0131,
// 04-ISSUES/170), so it is named here rather than left to be found.
@@ -875,15 +835,6 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
return held, problems
}
// canonicalSeat is the seat a claimed name refers to, by its current name: itself for a name the mesh
// does not know (a module's own seat), its seat's name for a former one.
func canonicalSeat(name string) string {
if s, known := SeatNamed(name); known {
return s.Name
}
return name
}
// checkResources refuses two modules writing the same thing.
//
// This costs no manifest field: the mesh already holds every resource of every module, so two
@@ -961,23 +912,6 @@ func checkResources(modules []Manifest) []string {
}
}
}
// **A file the mesh renders for a module is that module's path too** (novox/hq ADR 0223). The
// machine's resolver file is a fact the uplink's holder asks for, and a second module asking
// for it, or declaring it, would have the host write one path twice in every apply, the last
// one winning. A fact under the operator's home is placed per account and compared nowhere.
for _, name := range sortedFacts(m.Facts) {
fact := m.Facts[name]
if fact.Home || !strings.HasPrefix(fact.Path, "/") {
continue
}
key := "path " + fact.Path
if other, taken := owner[key]; taken && other != m.Module {
problems = append(problems, fmt.Sprintf(
"%s and %s both declare the path %q", other, m.Module, fact.Path))
}
owner[key] = m.Module
ownedPath[fact.Path] = m.Module
}
}
// An accessed path is the operator's, so no module may declare it as one of its own
@@ -1200,40 +1134,3 @@ func machineReachRemedy(catalogue map[string]Manifest, want, node string) string
}
return fmt.Sprintf("assign one to %s: %s", node, strings.Join(named, "; "))
}
// answeredElsewhere says that a mesh-wide provision this machine could answer itself is answered by
// another machine all the same: one this machine was pinned to, or the holder of the mesh seat that
// delivers it (novox/hq ADR 0110, ADR 0194).
//
// **A provider on the machine was taken before either was asked.** The branch for a provision
// answered here bound the consumer to the local provider and consulted a pin only between two local
// ones, and the seat's holder never; both were read only for a provider on another machine. So when
// every machine ran a provider of `wildcard-resolution` — each machine's own resolver, still assigned
// while the mesh moved to one — every machine bound its resolver configuration to itself, with the
// mesh's one resolver recorded, held and pinned (novox/hq issue 258). The seat is the mesh's choice of
// who answers, made once; a provider that merely runs here does not overrule it, and neither does it
// overrule a pin somebody set on this machine.
//
// Not in the first pass, which asks only what this machine offers and has no providers to read.
func answeredElsewhere(want string, node Node, world World, brokered map[string]bool) bool {
if world.Unchecked || !brokered[want] {
return false
}
if c, pinned := world.Pinned[want]; pinned {
return c.Node != node.Name
}
// **A machine holding the seat answers itself** (novox/hq ADR 0223). With a replicated seat
// another machine holds it too, and the first holder in the providers' order may be that one; a
// holder is still where this machine's own requirement is answered, so its resolver file lists
// itself first.
if seat, delivered := SeatDelivering(want); delivered {
for _, h := range append(append([]Held(nil), world.Holdings...), world.Held...) {
if hs, ok := SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope &&
h.Node == node.Name {
return false
}
}
}
holder, held := HolderAmong(want, world.Offered[want], world.Held)
return held && holder.Node != node.Name
}
internal/catalogue/resolver_file_is_the_uplinks_test.go
-57
View File
@@ -1,57 +0,0 @@
package catalogue
import (
"reflect"
"testing"
)
// novox/hq ADR 0223 part 2: /etc/resolv.conf belongs to the module holding node-uplink. The seat that
// wrote it, node-resolver-config, and ADR 0220's dependency of it on the uplink retire: one owner for
// the file, and it is the program that would otherwise rewrite it.
func TestTheResolverConfigSeatIsGone(t *testing.T) {
if _, known := SeatNamed("node-resolver-config"); known {
t.Error("node-resolver-config is still in the mesh's set; the uplink's holder writes the resolver file")
}
// An uplink's holder needs no seat beside it for the file: it is its own.
if got := DependsOn(mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"})); len(got) != 0 {
t.Errorf("an uplink holder with nothing declared depends on %v", got)
}
}
// The catalogue as it is: nothing claims the retired seat, and every manager the mesh knows holds the
// uplink and writes the resolver file.
func TestTheCataloguesUplinksWriteTheResolverFileAndNothingElseDoes(t *testing.T) {
cat := map[string]Manifest{}
for _, m := range theCatalogue(t) {
cat[m.Module] = m
}
if got := PossibleHolders(cat, "node-uplink"); !reflect.DeepEqual(got, uplinks) {
t.Errorf("node-uplink can be held by %v, not %v", got, uplinks)
}
for name, m := range cat {
for _, c := range m.Claims {
if c.Name == "node-resolver-config" {
t.Errorf("%s still claims node-resolver-config", name)
}
}
_, writes := m.Facts["resolvers"]
isUplink := false
for _, u := range uplinks {
isUplink = isUplink || u == name
}
for _, f := range m.Facts {
if f.Path == "/etc/resolv.conf" && !isUplink {
t.Errorf("%s writes /etc/resolv.conf and does not hold the uplink", name)
}
}
for _, r := range m.Resources {
if r["path"] == "/etc/resolv.conf" {
t.Errorf("%s declares /etc/resolv.conf as a file of its own", name)
}
}
if isUplink && !writes {
t.Errorf("%s holds the uplink and does not write the resolver file", name)
}
}
}
internal/catalogue/resolver_manifests_test.go
+40 -54
View File
@@ -15,8 +15,8 @@ import (
// same arrangement, and to the two things a resolver here must never do — read resolv.conf for
// its upstreams, or take an address systemd-resolved holds.
// resolverShelf is the resolver, an uplink module that writes what the machine asks (novox/hq ADR
// 0223), and the container runtime beside something that answers `mesh-addressing`.
// resolverShelf is the three resolver modules and the container runtime beside something that
// answers `mesh-addressing`.
// The networking module that really does is composed in the controller and cannot be imported
// here, so a stand-in offers the same word; what is under test is the manifests, not the network.
func resolverShelf(t *testing.T) map[string]Manifest {
@@ -24,7 +24,7 @@ func resolverShelf(t *testing.T) map[string]Manifest {
shelf := map[string]Manifest{
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
}
for _, name := range []string{"dnsmasq", "systemd-networkd", "docker"} {
for _, name := range []string{"dnsmasq", "resolv-conf", "resolved-split-dns", "docker"} {
shelf[name] = catalogueManifest(t, name)
}
return shelf
@@ -84,25 +84,31 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
t.Errorf("the mesh's resolver still reads or listens on %q", never)
}
}
// And the file that decides what the machine asks names every one of the mesh's resolvers, by
// address, from the seat's holders, and no public one (ADR 0223): a resolver library that asks
// every listed server at once takes the first reply, and a public "no such name" for a mesh name
// won it. Written by every uplink module, since the uplink's holder owns the file.
for _, uplink := range uplinks {
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
if !ok || fact.Path != "/etc/resolv.conf" {
t.Fatalf("%s's resolver file is not rendered from the roster: %+v", uplink, fact)
// And the file that decides what the machine asks names the mesh's resolver first, by address,
// and a public one second, asked only when the first is silent (ADR 0196).
var resolv string
for _, r := range catalogueManifest(t, "resolv-conf").Resources {
if r["path"] == "/etc/resolv.conf" {
resolv, _ = r["content"].(string)
}
if !strings.Contains(fact.Template, `{{range index .Holders "mesh-dns-resolver"}}nameserver {{.Address}}`) {
t.Errorf("%s's resolv.conf does not list every holder of the mesh's resolver:\n%s", uplink, fact.Template)
}
var nameservers []string
for _, line := range strings.Split(resolv, "\n") {
if strings.HasPrefix(line, "nameserver ") {
nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
}
for _, line := range strings.Split(fact.Template, "\n") {
if strings.HasPrefix(line, "nameserver ") && !strings.Contains(line, "{{") {
t.Errorf("%s's resolv.conf names a resolver of its own beside the mesh's: %s", uplink, line)
}
}
if !strings.Contains(fact.Template, "options timeout:1 attempts:2 edns0\n") {
t.Errorf("%s: a silent resolver is not passed over after one short wait:\n%s", uplink, fact.Template)
}
if len(nameservers) != 2 || nameservers[0] != "${bound:wildcard-resolution:address}" || nameservers[1] != "1.1.1.1" {
t.Errorf("resolv.conf names %v; the mesh's resolver by address first, a public one second", nameservers)
}
if !strings.Contains(resolv, "\noptions timeout:1 attempts:1") {
t.Errorf("the fallback is not reached after one short attempt:\n%s", resolv)
}
// The split-DNS alternative points at the same resolver, or a machine that keeps
// systemd-resolved in charge would route the mesh's suffix to nothing.
for _, r := range catalogueManifest(t, "resolved-split-dns").Resources {
if content, _ := r["content"].(string); content != "" && !strings.Contains(content, "DNS=${bound:wildcard-resolution:address}\n") {
t.Errorf("resolved-split-dns does not point at the resolver's address:\n%s", content)
}
}
}
@@ -112,10 +118,9 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
// that file, the machine pointed at the resolver by address, and the runtime given no resolver of
// its own but kept running across a restart (ADR 0196).
func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "systemd-networkd", "docker"},
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf", "docker"},
Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
"package-manager": true, "service-manager": true, "privileged": true,
"uplink-systemd-networkd": true}}, World{})
"package-manager": true, "service-manager": true, "privileged": true}}, World{})
if err != nil {
t.Fatal(err)
}
@@ -128,7 +133,6 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
// happen to be the same map, since nothing routed is part of it.
Names: twoMachines, Machines: twoMachines, Suffix: "internal",
Zones: []ZoneAt{{Zone: "incus", Address: "10.42.0.2", Port: 5353}},
Holders: map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
})
@@ -173,7 +177,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
t.Errorf("the resolver still writes the runtime's dns: %v", ids["dnsmasq.runtime-dns"])
}
for id := range ids {
if strings.HasPrefix(id, "systemd-networkd.runtime") {
if strings.HasPrefix(id, "resolv-conf.runtime") {
t.Errorf("what the machine asks still writes the runtime's file: %s", id)
}
}
@@ -208,40 +212,22 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
}
resolv := ids["systemd-networkd.fact-resolvers"]
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\noptions ") {
t.Fatalf("the machine is not pointed at the resolver by address, and only at it: %v", resolv)
resolv := ids["resolv-conf.resolv"]
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\nnameserver 1.1.1.1\n") {
t.Fatalf("the machine is not pointed at the resolver by address, with the public fallback: %v", resolv)
}
}
// Two modules deciding what a machine asks are refused on one machine, as before — now that the file
// is the uplink's (novox/hq ADR 0223), by two things: a machine runs one network manager, and no other
// module may write the resolver file beside the uplink's, as a file or as a rendered fact.
// Two modules deciding what a machine asks are refused on one machine, as before — the claim
// exists so they never take turns overwriting each other.
func TestTwoThingsDecidingWhatAMachineAsksAreRefused(t *testing.T) {
shelf := resolverShelf(t)
shelf["networkmanager"] = catalogueManifest(t, "networkmanager")
node := Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
"package-manager": true, "service-manager": true,
"uplink-systemd-networkd": true, "uplink-networkmanager": true}}
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "networkmanager"}, node, World{})
if err == nil || !strings.Contains(err.Error(), "node-uplink") {
t.Fatalf("two network managers were both assigned to one machine: %v", err)
_, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf", "resolved-split-dns"},
Node{Name: "anchor", At: "anchor.internal"}, World{})
if err == nil {
t.Fatal("resolv-conf and resolved-split-dns were both assigned to one machine")
}
// The second is made up: what is under test is that the resolver file has one owner, so any
// module asking the mesh to render it — or declaring it — will do.
for name, m := range map[string]Manifest{
"a-rendered-one": {Module: "a-rendered-one", Version: "1",
Facts: map[string]RosterFile{"mine": {Path: "/etc/resolv.conf", Template: "nameserver 10.42.0.1\n"}}},
"a-declared-one": {Module: "a-declared-one", Version: "1", Resources: []map[string]any{
{"id": "mine", "type": "file", "path": "/etc/resolv.conf", "content": "nameserver 10.42.0.1\n"}}},
} {
shelf := resolverShelf(t)
shelf[name] = m
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", name}, node, World{})
if err == nil || !strings.Contains(err.Error(), "/etc/resolv.conf") {
t.Errorf("%s wrote the resolver file beside the uplink's: %v", name, err)
}
if !strings.Contains(err.Error(), "node-resolver-config") {
t.Fatalf("the refusal does not say what was claimed: %v", err)
}
}
internal/catalogue/roster.go
+9 -44
View File
@@ -64,12 +64,6 @@ type rosterView struct {
// Zones is every zone a module in the mesh answers itself, with where its answerer is (novox/hq
// ADR 0199) — what the mesh's resolver forwards. Ordered by zone.
Zones []rosterZone
// Holders is the machines holding each replicated mesh seat, by seat (novox/hq ADR 0223) — what
// a machine's resolver file lists for `mesh-dns-resolver`. **This machine first when it is one
// of them**, then the rest by name: the nearest holder is asked first, and two renderings of
// one mesh on one machine are one file. A template reads one seat's with
// `index .Holders "<seat>"`; a seat nobody holds ranges over nothing.
Holders map[string][]rosterEntry
}
// rosterZone is one zone as a template sees it: the zone, and the address and port answering it.
@@ -102,25 +96,21 @@ func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]st
// FactsWithZonesInto is FactsInto with the mesh's zones in the view, for a template that ranges them.
func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string,
zones []ZoneAt) ([]map[string]any, error) {
return FactsFrom(m, r, Rendering{Names: every, Machines: machines, Accounts: accounts, Suffix: suffix,
Zones: zones})
}
// FactsFrom renders the roster files a module asked for from everything the mesh composed for this
// machine: its machines, zones and the holders of each replicated seat.
func FactsFrom(m Manifest, r Resolution, with Rendering) ([]map[string]any, error) {
if len(m.Facts) == 0 {
return nil, nil
}
names := sortedFacts(m.Facts)
names := make([]string, 0, len(m.Facts))
for name := range m.Facts {
names = append(names, name)
}
sort.Strings(names)
view := rosterView{
Node: r.Node,
Suffix: strings.TrimPrefix(suffixOr(with.Suffix), "."),
Names: entriesFrom(with.Names, with.Accounts, with.Suffix),
Machines: entriesFrom(with.Machines, with.Accounts, with.Suffix),
Zones: zonesFrom(with.Zones),
Holders: holdersFrom(with.Holders, with.Accounts, with.Suffix, r.Node),
Suffix: strings.TrimPrefix(suffixOr(suffix), "."),
Names: entriesFrom(every, accounts, suffix),
Machines: entriesFrom(machines, accounts, suffix),
Zones: zonesFrom(zones),
}
out := make([]map[string]any, 0, len(names))
@@ -260,28 +250,3 @@ func zonesFrom(zones []ZoneAt) []rosterZone {
sort.Slice(out, func(i, j int) bool { return out[i].Zone < out[j].Zone })
return out
}
// holdersFrom is each replicated seat's holders as a template ranges them: this machine first when
// it holds the seat, then the others by name (novox/hq ADR 0223). A machine with no address is left
// out, as everywhere in the roster — a resolver named at nothing is a lookup that hangs.
func holdersFrom(holders map[string]map[string]string, accounts map[string]string, suffix, node string) map[string][]rosterEntry {
out := make(map[string][]rosterEntry, len(holders))
for seat, at := range holders {
entries := entriesFrom(at, accounts, suffix)
sort.SliceStable(entries, func(i, j int) bool {
return entries[i].Name == node && entries[j].Name != node
})
out[seat] = entries
}
return out
}
// sortedFacts is a module's fact names in order, so what is said about them is said the same way twice.
func sortedFacts(facts map[string]RosterFile) []string {
out := make([]string, 0, len(facts))
for name := range facts {
out = append(out, name)
}
sort.Strings(out)
return out
}
internal/catalogue/seat_answers_before_local_test.go
-67
View File
@@ -1,67 +0,0 @@
package catalogue
import "testing"
// A mesh-wide provision whose seat is held on another machine is answered by that holder, even on a
// machine that runs a provider of its own (novox/hq issue 258). Every machine ran its own resolver
// while the mesh moved to one; each bound its resolver configuration to itself, with the mesh's
// resolver held elsewhere and pinned.
func resolverMesh() map[string]Manifest {
return map[string]Manifest{
"resolver": {Module: "resolver", Version: "1",
Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}},
Claims: []Claim{{Name: "mesh-dns-resolver", Scope: ScopeMesh}}},
"asker": {Module: "asker", Version: "1", Requires: []string{"wildcard-resolution"}},
}
}
func resolverWorld(held string, pin *Chosen) World {
w := World{
Offered: map[string][]Provider{"wildcard-resolution": {
{Node: "anchor", At: "anchor.internal", Module: "resolver"},
{Node: "laptop", At: "laptop.internal", Module: "resolver"},
}},
}
// Held is who holds it across the mesh; Holdings is the same holder on record, which lets this
// machine's own claimant stand beside it (ADR 0131).
w.Held = []Held{{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: held, Module: "resolver"}}
w.Holdings = w.Held
if pin != nil {
w.Pinned = map[string]Chosen{"wildcard-resolution": *pin}
}
return w
}
func answeredFrom(t *testing.T, world World) string {
t.Helper()
laptop := Node{Name: "laptop", At: "laptop.internal"}
got, err := Resolve(resolverMesh(), []string{"resolver", "asker"}, laptop, world)
if err != nil {
t.Fatal(err)
}
for _, n := range got.Needs {
if n.Name == "wildcard-resolution" {
return n.From
}
}
t.Fatalf("no binding for wildcard-resolution: %+v", got.Needs)
return ""
}
func TestTheSeatsHolderElsewhereAnswersBeforeThisMachinesOwnProvider(t *testing.T) {
if from := answeredFrom(t, resolverWorld("anchor", nil)); from != "anchor" {
t.Fatalf("bound to %s; the seat is held on anchor", from)
}
}
func TestAPinElsewhereAnswersBeforeThisMachinesOwnProvider(t *testing.T) {
if from := answeredFrom(t, resolverWorld("laptop", &Chosen{Node: "anchor", Module: "resolver"})); from != "anchor" {
t.Fatalf("bound to %s; this machine was pinned to anchor", from)
}
}
func TestTheHolderOnThisMachineStillAnswersHere(t *testing.T) {
if from := answeredFrom(t, resolverWorld("laptop", nil)); from != "laptop" {
t.Fatalf("bound to %s; the seat is held here", from)
}
}
internal/catalogue/seat_dependencies.go
+2 -4
View File
@@ -6,10 +6,8 @@ import (
"strings"
)
// A module depends on the node seats that apply its resources (novox/hq ADR 0207) and on the
// seats it contributes to (novox/hq ADR 0210). A third source — what a seat it holds needs beside it
// (novox/hq ADR 0220) — had one user, node-resolver-config needing node-uplink, and went with that
// seat when the resolver file became the uplink's own (novox/hq ADR 0223).
// A module depends on the node seats that apply its resources (novox/hq ADR 0207), and on the
// seats it contributes to (novox/hq ADR 0210).
//
// Some of what a module declares is applied through software on the machine that is itself a
// module: a service through the service manager, a package through the package manager, a container
internal/catalogue/seat_into.go
+6 -108
View File
@@ -1,7 +1,6 @@
package catalogue
import (
"encoding/json"
"fmt"
"regexp"
"sort"
@@ -43,27 +42,6 @@ import (
// ofSeat is where a module asks about a seat: ${seat:<seat>:<the port its holder's software uses>}.
var ofSeat = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):([0-9]+)\}`)
// **Where this machine reaches a mesh seat's holder** (novox/hq ADR 0222, issue 190).
//
// `${seat:<mesh-seat>:reach}` is host:port — the address this machine dials to reach whatever holds a
// seat the mesh holds once. The same reasoning as the port above, one step further: nothing is
// required, nothing is granted, no credential is minted, and the answer is an address the mesh
// already holds in the clear and composes into every reference it built. What it is for is a module
// that must *state* where a mesh service is to software it owns — the container runtime trusting
// the mesh's registry is the case — without becoming that service's consumer.
//
// Answered for the seats in reachedSeats only. Another seat is refused by name rather than answered
// with nothing: a module asking where something is that the mesh does not say would otherwise be
// given an empty value and never know the question was not understood.
//
// The answer may be empty: no machine on the private network holds the seat yet (genesis raises
// the store before the network). In a file written into as JSON, an empty member is dropped from
// its list, and a list left with none is dropped, so the runtime is never told to trust "".
var ofSeatReach = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):reach\}`)
// reachedSeats is every seat ${seat:…:reach} answers for.
var reachedSeats = map[string]bool{"mesh-artifact-store": true}
// seatInto replaces a resource's ${seat:…} placeholders with where this machine put each seat's
// holder — in a file's content, and in a value of a container's or a process's environment. The
// same places portInto fills, for the same reason: they are where a program reads a number from.
@@ -71,24 +49,13 @@ func seatInto(resource map[string]any, module string, with Rendering) error {
switch fmt.Sprint(resource["type"]) {
case "file":
content, ok := resource["content"].(string)
if !ok || (!ofSeat.MatchString(content) && !ofSeatReach.MatchString(content)) {
if !ok || !ofSeat.MatchString(content) {
return nil
}
where := fmt.Sprintf("%s has a file that", module)
filled, err := seatsFilledInto(content, where, with)
filled, err := seatsFilledInto(content, fmt.Sprintf("%s has a file that", module), with)
if err != nil {
return err
}
if ofSeatReach.MatchString(filled) {
if filled, err = reachFilledInto(filled, where, with); err != nil {
return err
}
if fmt.Sprint(resource["into"]) == "json" {
if filled, err = withoutEmptyMembers(filled, where); err != nil {
return err
}
}
}
resource["content"] = filled
case "container", "process":
@@ -107,18 +74,15 @@ func seatInto(resource map[string]any, module string, with Rendering) error {
var filled map[string]any
for _, key := range named {
written, ok := env[key].(string)
if !ok || (!ofSeat.MatchString(written) && !ofSeatReach.MatchString(written)) {
if !ok || !ofSeat.MatchString(written) {
continue
}
where := fmt.Sprintf("%s's %s %s sets %s to something that",
module, resource["type"], resource["name"], key)
value, err := seatsFilledInto(written, where, with)
value, err := seatsFilledInto(written,
fmt.Sprintf("%s's %s %s sets %s to something that",
module, resource["type"], resource["name"], key), with)
if err != nil {
return err
}
if value, err = reachFilledInto(value, where, with); err != nil {
return err
}
if filled == nil {
filled = map[string]any{}
for k, v := range env {
@@ -154,69 +118,3 @@ func seatsFilledInto(written, where string, with Rendering) (string, error) {
}
return written, nil
}
// reachFilledInto answers every ${seat:…:reach} in one written value with where this machine
// reaches the seat's holder, or with nothing when no holder is reachable yet. A seat the mesh does
// not answer this for is refused by name.
func reachFilledInto(written, where string, with Rendering) (string, error) {
for _, m := range ofSeatReach.FindAllStringSubmatch(written, -1) {
seat := m[1]
if !reachedSeats[seat] {
known := make([]string, 0, len(reachedSeats))
for s := range reachedSeats {
known = append(known, s)
}
sort.Strings(known)
return "", fmt.Errorf("%s says ${seat:%s:reach}, and the mesh says where a seat's holder is "+
"reached only for %s (novox/hq ADR 0222)", where, seat, strings.Join(known, ", "))
}
written = strings.ReplaceAll(written, m[0], with.SeatReach[seat])
}
return written, nil
}
// withoutEmptyMembers drops every empty string from the lists at the top of a JSON object written
// into a machine's file, and a list left with no members, so an unanswered ${seat:…:reach} adds
// nothing to the machine's list rather than adding "".
func withoutEmptyMembers(content, where string) (string, error) {
var object map[string]json.RawMessage
if err := json.Unmarshal([]byte(content), &object); err != nil {
return "", fmt.Errorf("%s is written into as JSON and is not a JSON object: %w", where, err)
}
changed := false
for key, raw := range object {
var members []json.RawMessage
if err := json.Unmarshal(raw, &members); err != nil {
continue // not a list
}
kept := make([]json.RawMessage, 0, len(members))
for _, member := range members {
var s string
if json.Unmarshal(member, &s) == nil && s == "" {
continue
}
kept = append(kept, member)
}
if len(kept) == len(members) {
continue
}
changed = true
if len(kept) == 0 {
delete(object, key)
continue
}
list, err := json.Marshal(kept)
if err != nil {
return "", err
}
object[key] = list
}
if !changed {
return content, nil
}
out, err := json.Marshal(object)
if err != nil {
return "", err
}
return string(out) + "\n", nil
}
internal/catalogue/seat_into_test.go
-81
View File
@@ -1,7 +1,6 @@
package catalogue
import (
"fmt"
"os"
"strings"
"testing"
@@ -212,83 +211,3 @@ func withSeatPorts(m Manifest) Manifest {
}
return out
}
// Where this machine reaches a mesh seat's holder (novox/hq ADR 0222, issue 190): the container
// runtime's module tells the runtime to trust the mesh's registry without binding the store.
func runtimeTrust() map[string]any {
return map[string]any{
"type": "file", "id": "daemon", "path": "/etc/docker/daemon.json", "into": "json",
"content": `{"live-restore": true, "insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n",
}
}
func TestASeatsReachIsWhereThisMachineReachesItsHolder(t *testing.T) {
file := runtimeTrust()
with := Rendering{SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"}}
if err := seatInto(file, "docker", with); err != nil {
t.Fatal(err)
}
want := `{"live-restore": true, "insecure-registries": ["anchor.internal:5100"]}` + "\n"
if file["content"] != want {
t.Fatalf("content = %q, want %q", file["content"], want)
}
process := map[string]any{"type": "process", "name": "p",
"env": map[string]any{"REGISTRY": "${seat:mesh-artifact-store:reach}"}}
if err := seatInto(process, "x", with); err != nil {
t.Fatal(err)
}
if got := process["env"].(map[string]any)["REGISTRY"]; got != "anchor.internal:5100" {
t.Fatalf("an environment value was filled with %q", got)
}
}
// With no holder reachable, the runtime is not told to trust "": the member is dropped, and the
// list with it when nothing else is in it.
func TestAnUnansweredReachAddsNothingToAList(t *testing.T) {
file := runtimeTrust()
if err := seatInto(file, "docker", Rendering{}); err != nil {
t.Fatal(err)
}
if got := file["content"]; got != `{"live-restore":true}`+"\n" {
t.Fatalf("with no store reachable, the runtime's keys are %q", got)
}
kept := map[string]any{"type": "file", "into": "json",
"content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}", "192.0.2.7:5000"]}`}
if err := seatInto(kept, "docker", Rendering{}); err != nil {
t.Fatal(err)
}
if got := kept["content"]; got != `{"insecure-registries":["192.0.2.7:5000"]}`+"\n" {
t.Fatalf("a list's other members were not kept: %q", got)
}
}
func TestAReachTheMeshDoesNotAnswerIsRefused(t *testing.T) {
file := map[string]any{"type": "file", "content": "${seat:mesh-store:reach}"}
err := seatInto(file, "x", Rendering{SeatReach: map[string]string{"mesh-store": "anchor.internal:5432"}})
if err == nil || !strings.Contains(err.Error(), "mesh-artifact-store") {
t.Fatalf("a reach the mesh does not answer was not refused by name: %v", err)
}
}
// Through the whole composition, as the container runtime's module declares it.
func TestTheRuntimesTrustIsComposedFromTheSeatsReach(t *testing.T) {
r := Resolution{Node: "anchor", Modules: []Manifest{{
Module: "docker", Resources: []map[string]any{runtimeTrust()},
}}}
out, err := r.Declaration(Rendering{
SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"},
})
if err != nil {
t.Fatal(err)
}
file := fileNamed(out, "docker.daemon")
if file == nil {
t.Fatalf("no file in %v", out)
}
if got := file["content"]; !strings.Contains(fmt.Sprint(got), `["anchor.internal:5100"]`) {
t.Fatalf("the runtime's file says %q", got)
}
}
internal/catalogue/seats.go
+27 -45
View File
@@ -21,16 +21,8 @@ import (
type Seat struct {
// Name is what a manifest claims.
Name string
// Scope is where there may be only one holder — unless the seat is Replicated.
// Scope is where there may be only one holder.
Scope string
// Replicated says a mesh seat may be held on several machines at once, each holder answering the
// same thing (novox/hq ADR 0223): the mesh's resolver, held on the anchor and the home server so a
// machine's resolver file lists two that give one answer. Each holder is on record, added by an
// act (`seat <name> --add <node>/<module>`), never by being assigned: two claimants with nothing on
// record are refused exactly as for any mesh seat. One per machine still — two modules on one
// node claiming it are refused. Compiled, never stored, like Receives: it is the mesh's definition of
// the role, and the store's rows carry no column for it.
Replicated bool
// Delivers is the provision the seat's holder answers for, or empty. A seat that delivers a
// provision may only be held by a module providing it at the seat's scope, and its holder is
// what a requirement for that provision resolves to when several modules provide it.
@@ -141,20 +133,18 @@ var defaultSeats = append([]Seat{
// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
{Name: "mesh-build-machine", Scope: ScopeMesh,
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
// than one machine, each answering the same names from the same roster, and every machine's
// resolver file lists every holder — its own first — and no public resolver, so whichever answers
// first gives the one answer. Delivers what a machine's resolver configuration requires, so that
// requirement resolves to a holder wherever they are placed.
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Replicated: true,
Decision: "novox/hq ADR 0194, ADR 0223"},
// **A machine's names are one module's** (novox/hq ADR 0199, ADR 0223): its holder writes
// /etc/hostname and the machine's own lines in /etc/hosts, and keeps every other line of the hosts
// file as the operator's, changed through these three verbs on that machine alone. The controller
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
// alias on a mesh that knew it.
{Name: "node-hostname", Scope: ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
// **The mesh's one resolver** (novox/hq ADR 0194, 0196): every node's internal domain, held in one
// place, and every node and container asks it first. Delivers what a machine's resolver
// configuration requires, so that requirement resolves to the holder wherever it is placed.
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Decision: "novox/hq ADR 0194"},
// **Retired by ADR 0194, kept while a manifest still claims it** — the same reason as
// mesh-build-machine above: a machine still holds it until the mesh's resolver replaces it, and
// removing the row first would make that machine unresolvable. Deleted once nothing claims it.
{Name: "node-dns-resolver", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
// **A machine's /etc/hosts is one module's** (novox/hq ADR 0199): its holder writes the machine's
// own lines and keeps every other line as the operator's, changed through these three verbs on that
// machine alone. The controller holds none of it.
{Name: "node-hosts-file", Scope: ScopeNode, Decision: "novox/hq ADR 0199",
Serves: []Verb{
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
"the operator's, or the block of the module or tool that writes it.",
@@ -255,12 +245,11 @@ var defaultSeats = append([]Seat{
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
// model change, not a rename, so it stays until that is built.
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
// The program that manages the machine's own network. It delivers nothing: its holder keeps the
// manager and the mesh from contradicting each other — the private network's interface left
// alone — and writes the machine's resolver file itself, because the manager is what would
// otherwise rewrite it (novox/hq ADR 0223, which retired node-resolver-config into this seat).
// It never declares a link, an address or a wireless network, because the link is the only
// channel a fix could arrive on. A seat
{Name: "node-resolver-config", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
// The program that manages the machine's own network. It delivers nothing: its holder only
// keeps the manager and the mesh from contradicting each other — the resolver file left to the
// mesh, the private network's interface left alone — and never declares a link, an address or
// a wireless network, because the link is the only channel a fix could arrive on. A seat
// rather than a condition in the resolver's module, so a machine running two managers is
// refused at assignment instead of found by the resolver being rewritten (novox/hq ADR 0117).
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117"},
@@ -315,11 +304,9 @@ func UseSeats(s []Seat) {
row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
}
}
// What a seat receives and whether it is replicated are never stored (novox/hq ADR 0212, ADR
// 0223), so they are always the compiled ones.
// What a seat receives is never stored (novox/hq ADR 0212), so it is always the compiled one.
if d, known := byName[row.Name]; known {
row.Receives = d.Receives
row.Replicated = d.Replicated
}
merged = append(merged, row)
}
@@ -499,24 +486,19 @@ func seatNames() string {
// two modules on one node could both provide a provision, and only the one holding the seat
// answers for it. Nothing when no seat delivers the provision, when nobody holds
// it, or when the holder is not among the providers offered.
//
// **The first holder in the providers' own order** (novox/hq ADR 0223). A replicated seat has
// several, and the answer must not depend on the order the mesh happened to resolve its machines
// in: the providers come sorted by machine, so every consumer is bound to the same one. A seat with
// one holder gets the same answer as before.
func HolderAmong(provision string, providers []Provider, held []Held) (Provider, bool) {
seat, delivered := SeatDelivering(provision)
if !delivered {
return Provider{}, false
}
for _, p := range providers {
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a
// seat's former name still matches it after a rename (novox/hq ADR 0122).
hs, ok := SeatNamed(h.Claim)
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
continue
}
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a seat's
// former name still matches it after a rename (novox/hq ADR 0122).
hs, ok := SeatNamed(h.Claim)
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
continue
}
for _, p := range providers {
if p.Node == h.Node && p.Module == h.Module {
return p, true
}
internal/catalogue/seats_test.go
+5 -9
View File
@@ -46,18 +46,14 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
delivered[s.Delivers] = s.Name
}
}
// Thirty-six since node-resolver-config retired into node-uplink (novox/hq ADR 0223); thirty-seven
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
// node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215);
// thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199, now
// node-hostname); thirty-four
// Thirty-eight with node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215); thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199); thirty-four
// with node-hotkeys (ADR 0212); thirty-three with node-power (ADR 0211); thirty-two since the
// graphical session's eleven (ADR 0208); twenty-one with node-package-manager and
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
// mesh-build-machine row goes, when no registered manifest claims it.
if len(Seats()) != 36 {
t.Errorf("the mesh defines %d seats rather than 36; the set is closed, so a change here is "+
// ADR 0204); seventeen with node-build-agent (ADR 0190). Two fewer once the retired
// mesh-build-machine and node-dns-resolver rows go, when no registered manifest claims either.
if len(Seats()) != 38 {
t.Errorf("the mesh defines %d seats rather than 38; the set is closed, so a change here is "+
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
}
}
internal/catalogue/two_resolvers_test.go
-270
View File
@@ -1,270 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// The mesh has two resolvers (novox/hq ADR 0223): `mesh-dns-resolver` is replicated, held on the
// anchor and on the home server, each answering the same names; every machine's resolver file lists
// every holder — its own first when it is one — and no public resolver. ADR 0196 listed the mesh's
// resolver then a public one, and musl asks both at once and takes the first reply: from the home
// server the public "no such name" for the anchor's mesh name won, every time, in every Alpine build.
// The file is written by the module holding the machine's uplink (ADR 0223 part 2).
// resolverMachines is the anchor and the home server holding the resolver, and a laptop holding nothing.
var resolverMachines = map[string]string{
"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "home.internal": "10.42.0.3"}
// bothResolvers is the two holders on record, as `seat mesh-dns-resolver --add` leaves them.
var bothResolvers = []Held{
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "anchor", Module: "dnsmasq"},
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "home", Module: "dnsmasq"},
}
// uplinks is every module in the catalogue holding node-uplink, and so writing the machine's resolver
// file (novox/hq ADR 0223 part 2): the program that would otherwise rewrite it is the one that writes it.
var uplinks = []string{"dhcpcd", "networkmanager", "systemd-networkd"}
// managing is a machine as each uplink module needs it: able to install, run a service and run the
// manager that module is for.
func managing(node string) Node {
caps := map[string]bool{"package-manager": true, "service-manager": true}
for _, u := range uplinks {
caps["uplink-"+u] = true
}
return Node{Name: node, At: node + ".internal", Capabilities: caps}
}
// twoResolverShelf is the resolver, the uplink modules that write what a machine asks, and a stand-in
// answering `mesh-addressing`.
func twoResolverShelf(t *testing.T) map[string]Manifest {
t.Helper()
out := map[string]Manifest{
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
"dnsmasq": catalogueManifest(t, "dnsmasq"),
}
for _, u := range uplinks {
out[u] = catalogueManifest(t, u)
}
return out
}
// worldWithout is the rest of the mesh as a plan for one machine sees it: every other holder's claim
// and offer, and both holders on record.
func worldWithout(node string) World {
w := World{Holdings: bothResolvers, Offered: map[string][]Provider{}}
for _, h := range bothResolvers {
if h.Node == node {
continue
}
w.Held = append(w.Held, h)
w.Offered["wildcard-resolution"] = append(w.Offered["wildcard-resolution"],
Provider{Node: h.Node, At: h.Node + ".internal", Module: h.Module})
}
return w
}
// resolvConfOn resolves and composes one machine and answers with the nameservers its resolver file
// lists, in order, and the file. The file is the uplink's — `uplink` is one of the assigned modules —
// and nothing else on the machine declares that path.
func resolvConfOn(t *testing.T, node, uplink string, assigned []string) ([]string, string) {
t.Helper()
got, err := Resolve(twoResolverShelf(t), assigned, managing(node), worldWithout(node))
if err != nil {
t.Fatalf("%s with %s does not resolve with two resolvers on record: %v", node, uplink, err)
}
out, err := got.Declaration(Rendering{
Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
Holders: map[string]map[string]string{"mesh-dns-resolver": {
"anchor.internal": "10.42.0.1", "home.internal": "10.42.0.3"}},
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
})
if err != nil {
t.Fatalf("%s with %s does not compose: %v", node, uplink, err)
}
var file map[string]any
for _, r := range out {
if r["path"] != "/etc/resolv.conf" {
continue
}
if file != nil {
t.Fatalf("%s declares /etc/resolv.conf twice: %v and %v", node, file["id"], r["id"])
}
file = r
}
if file == nil || file["id"] != uplink+".fact-resolvers" {
t.Fatalf("%s's resolver file is not %s's: %v", node, uplink, file)
}
content, _ := file["content"].(string)
var servers []string
for _, line := range strings.Split(content, "\n") {
if strings.HasPrefix(line, "nameserver ") {
servers = append(servers, strings.TrimPrefix(line, "nameserver "))
}
}
return servers, content
}
// Per uplink module: each writes a resolver file listing both holders, the machine's own first on a
// holder, and only the holders on a machine that is none.
func TestEveryUplinkListsBothResolversItsOwnFirst(t *testing.T) {
for _, uplink := range uplinks {
for node, want := range map[string][]string{
"anchor": {"10.42.0.1", "10.42.0.3"},
"home": {"10.42.0.3", "10.42.0.1"},
"laptop": {"10.42.0.1", "10.42.0.3"},
} {
assigned := []string{uplink}
if node != "laptop" {
assigned = append(assigned, "dnsmasq")
}
servers, content := resolvConfOn(t, node, uplink, assigned)
if strings.Join(servers, " ") != strings.Join(want, " ") {
t.Errorf("%s on %s lists %v; want %v\n%s", uplink, node, servers, want, content)
}
for _, public := range []string{"1.1.1.1", "8.8.8.8", "9.9.9.9"} {
if strings.Contains(content, public) {
t.Errorf("%s on %s lists a public resolver beside the mesh's (ADR 0223):\n%s", uplink, node, content)
}
}
if !strings.HasSuffix(content, "\noptions timeout:1 attempts:2 edns0\n") {
t.Errorf("%s on %s does not end with two short attempts:\n%s", uplink, node, content)
}
}
}
}
// One file, whichever manager the machine runs: the three modules carry the same template, so a
// machine changing its manager changes nothing in what it asks, and a fix made to one is made to all.
func TestEveryUplinkWritesTheSameResolverFile(t *testing.T) {
var first, firstOf string
for _, uplink := range uplinks {
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
if !ok || fact.Path != "/etc/resolv.conf" || fact.Shared || fact.Home {
t.Fatalf("%s does not write the machine's resolver file whole: %+v", uplink, fact)
}
if first == "" {
first, firstOf = fact.Template, uplink
continue
}
if fact.Template != first {
t.Errorf("%s's resolver file differs from %s's; the three are kept identical", uplink, firstOf)
}
}
}
// The requirement stays with what writes the file (ADR 0223 part 1): a machine is refused when nothing
// in the mesh resolves, rather than given a file listing nothing.
func TestEveryUplinkRequiresTheMeshsResolver(t *testing.T) {
for _, uplink := range uplinks {
found := false
for _, r := range catalogueManifest(t, uplink).Requires {
found = found || r == "wildcard-resolution"
}
if !found {
t.Errorf("%s writes the resolver file and does not require wildcard-resolution", uplink)
}
}
_, err := Resolve(twoResolverShelf(t), []string{"networkmanager"}, managing("laptop"), World{})
if err == nil || !strings.Contains(err.Error(), "wildcard-resolution") {
t.Errorf("an uplink was composed on a mesh with no resolver: %v", err)
}
}
// A holder answers its own requirement, even though the other holder sorts first (issue 258 kept).
func TestAHolderAnswersItsOwnRequirement(t *testing.T) {
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq", "networkmanager"},
managing("home"), worldWithout("home"))
if err != nil {
t.Fatal(err)
}
for _, n := range got.Needs {
if n.Name == "wildcard-resolution" && n.From != "home" {
t.Errorf("the home server's uplink is bound to %s; it holds the seat itself", n.From)
}
}
}
// A seat held once is still held once: a second claimant on another machine is refused while
// nothing is on record, and a store recording two holders of it is refused, naming the seat.
func TestASingleHolderMeshSeatStillRefusesASecondHolder(t *testing.T) {
store := shelf(mod("postgres", nil, nil, nil, Claim{Name: "mesh-store", Scope: ScopeMesh}))
other := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: "anchor", Module: "postgres"}
if _, err := Resolve(store, []string{"postgres"}, workstation(), World{Held: []Held{other}}); err == nil ||
!strings.Contains(err.Error(), "one per mesh") {
t.Errorf("a second claimant of a seat held once was not refused: %v", err)
}
here := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: workstation().Name, Module: "postgres"}
_, err := Resolve(store, []string{"postgres"}, workstation(),
World{Held: []Held{other}, Holdings: []Held{other, here}})
if err == nil || !strings.Contains(err.Error(), "on record as held by 2") {
t.Errorf("two holders on record for a seat held once were not refused: %v", err)
}
}
// Replicated is not "whoever is assigned": two claimants with nothing on record are refused, as for
// any mesh seat, and each holder is added by an act.
func TestTwoUnrecordedClaimantsOfTheReplicatedSeatAreRefused(t *testing.T) {
w := worldWithout("home")
w.Holdings = nil
_, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "home", At: "home.internal"}, w)
if err == nil || !strings.Contains(err.Error(), "seat mesh-dns-resolver --to") {
t.Errorf("a second resolver with nothing on record was not refused, naming the handover: %v", err)
}
}
// Only the mesh's resolver is replicated: a seat being replicated is a decision, recorded.
func TestOnlyTheResolverIsReplicated(t *testing.T) {
for _, s := range Seats() {
if s.Replicated != (s.Name == "mesh-dns-resolver") {
t.Errorf("%s replicated = %v; only mesh-dns-resolver is (ADR 0223)", s.Name, s.Replicated)
}
}
UseSeats([]Seat{{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution"}})
defer UseSeats(DefaultSeats())
if s, _ := SeatNamed("mesh-dns-resolver"); !s.Replicated {
t.Error("loading the set from the store, which has no column for it, lost the resolver's replication")
}
}
// Every consumer is bound to the same holder whatever order the mesh resolved its machines in.
func TestTheFirstHolderIsTheFirstProvider(t *testing.T) {
providers := []Provider{{Node: "anchor", Module: "dnsmasq"}, {Node: "home", Module: "dnsmasq"}}
for _, held := range [][]Held{bothResolvers, {bothResolvers[1], bothResolvers[0]}} {
if p, ok := HolderAmong("wildcard-resolution", providers, held); !ok || p.Node != "anchor" {
t.Errorf("held in order %v answered %v", held, p)
}
}
}
// One host record per machine, beside its wildcard (novox/hq issue 262): a name with a host record
// says it exists and has no IPv6 address, where the wildcard alone said there is no such name, and
// musl reads that as final.
func TestTheResolverHasOneHostRecordPerMachine(t *testing.T) {
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor", At: "anchor.internal"},
World{})
if err != nil {
t.Fatal(err)
}
out, err := got.Declaration(Rendering{Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}}})
if err != nil {
t.Fatal(err)
}
content, _ := byID(out)["dnsmasq.fact-node-zones"]["content"].(string)
records := map[string]int{}
for _, line := range strings.Split(content, "\n") {
if strings.HasPrefix(line, "host-record=") {
records[strings.TrimPrefix(line, "host-record=")]++
}
}
for name, at := range resolverMachines {
if records[name+","+at] != 1 {
t.Errorf("%s has %d host records at %s, and has one:\n%s", name, records[name+","+at], at, content)
}
}
if len(records) != len(resolverMachines) {
t.Errorf("%d host records for %d machines:\n%s", len(records), len(resolverMachines), content)
}
}
internal/catalogue/verbs.go
+13 -48
View File
@@ -73,13 +73,6 @@ var ControllerVerbs = []Verb{
{Name: "tools", Description: "Every seat's tools, from the mesh's own records: what each role " +
"answers, whether or not its holder is up. The mesh's own verbs are the mesh-controller seat's.",
Input: schema(nil, nil)},
{Name: "calls", Description: "The calls this controller answered lately and what came of each — " +
"one still running, one that finished after its caller was told it was running, one whose answer " +
"the bus refused — and, given a call's id, its whole answer (novox/hq issue 265). A call that has " +
"not finished within ten seconds answers that it is running, with its id; this is where it ends.",
Input: schema(map[string]string{
"call": "a call's id, as a running answer or `calls` gives it: that call and its whole answer",
}, nil)},
{Name: "status", Description: "What is wrong, what is quiet, what is out of date, and which " +
"machines are behind what the mesh would send them.",
Input: schema(nil, nil)},
@@ -97,7 +90,6 @@ var ControllerVerbs = []Verb{
Input: schema(map[string]string{
"module": "one module's name; every module when absent",
"log": "a build's id (as `builds` lists it): print what the build machine said, line by line",
"limit": "how many builds to list (default 20); not with log",
}, nil)},
{Name: "plans", Description: "What the last merges produced and where each stands (novox/hq ADR 0162): " +
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id.",
@@ -109,14 +101,9 @@ var ControllerVerbs = []Verb{
"repository": "owner/repository: the plan a merge there would produce, saving nothing (what-if); with paths or modules",
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
"modules": "with repository: or the modules it would change, comma-separated",
"limit": "how many plans to list (default 10); only when listing",
}, nil)},
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
"or, with files, the files it would be given.",
Input: schema(map[string]string{
"node": "the machine's name",
"files": "\"true\": the files this machine would be given, instead of the declaration as JSON",
}, []string{"node"}, "files")},
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it.",
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})},
{Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " +
"or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207).",
Input: schema(map[string]string{"node": "the machine's name",
@@ -134,15 +121,9 @@ var ControllerVerbs = []Verb{
}, []string{"node", "provision", "from", "module"})},
{Name: "unpin", Description: "Take that choice back, putting the question to the mesh again.",
Input: schema(map[string]string{"node": "the machine's name", "provision": "the provision"}, []string{"node", "provision"})},
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
"(a push can reload the bus, which then refuses any answer still to come).",
Input: schema(map[string]string{
"node": "the machine's name; without it, every machine that is behind",
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
}, nil, "behind")},
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine and module, " +
{Name: "push", Description: "Send a machine everything it should be — or every machine that is behind, when no machine is named.",
Input: schema(map[string]string{"node": "the machine's name; every machine behind when absent"}, nil)},
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine, " +
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
@@ -150,7 +131,7 @@ var ControllerVerbs = []Verb{
"provision": "a pair credential: the provision whose credential to replace",
"consumer": "with provision: only the holder on this machine (optional)",
"node": "an own secret: the machine",
"module": "an own secret: the module; with provision: only this consuming module's credential (optional)",
"module": "an own secret: the module",
"secret": "an own secret: its name in the module's definition",
}, nil)},
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
@@ -167,8 +148,8 @@ var ControllerVerbs = []Verb{
"module": "the module's name",
"values": "the settings as a JSON object, for set",
"node": "one machine; the whole mesh when absent",
"clear": "\"true\" to remove the layer instead of setting it; not with values",
}, []string{"module"}, "clear")},
"clear": "\"true\" to remove the layer instead of setting it",
}, []string{"module"})},
{Name: "command", Description: "Run one command line of the controller's own, as you would type it at its " +
"shell — `node account g14 jochen`, `node show ace`, `module list` — and answer what it printed. The " +
"generic verb beside the named ones (novox/hq ADR 0154): everything the binary can do, without a verb " +
@@ -186,7 +167,7 @@ var ControllerVerbs = []Verb{
Input: schema(map[string]string{"id": "the ask's build id, as `queue` lists it"}, []string{"id"})},
{Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " +
"recorded failed, cancelled by hand. Never touches one in flight.",
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil, "dead")},
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil)},
{Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " +
"build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.",
Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})},
@@ -197,7 +178,7 @@ var ControllerVerbs = []Verb{
"id": "the build's id",
"register": "\"true\" to register what it builds",
"older": "\"true\", with register: even though a newer build of the module is registered",
}, []string{"id"}, "register", "older")},
}, []string{"id"})},
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
"announces it failed, killed by hand — settled, never handed to another machine.",
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
@@ -216,27 +197,11 @@ var ControllerVerbs = []Verb{
}
// schema is a JSON schema for an object of string properties, which is every argument the verbs
// above take. Kept small on purpose: a schema an agent cannot read is a tool it cannot call. The
// switches are the properties that are "true" or "false" and nothing else, said in the schema as an
// enum so a caller sees it and the verb can refuse any other word rather than read it as false.
//
// **The schema is the whole of what a verb takes** (novox/hq issue 244): an argument it does not
// name is refused when the verb is called, never passed over.
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
isSwitch := map[string]bool{}
for _, s := range switches {
if _, declared := properties[s]; !declared {
panic("a switch that is not a property: " + s)
}
isSwitch[s] = true
}
// above take. Kept small on purpose: a schema an agent cannot read is a tool it cannot call.
func schema(properties map[string]string, required []string) map[string]any {
props := map[string]any{}
for name, description := range properties {
p := map[string]any{"type": "string", "description": description}
if isSwitch[name] {
p["enum"] = []string{"true", "false"}
}
props[name] = p
props[name] = map[string]any{"type": "string", "description": description}
}
out := map[string]any{"type": "object", "properties": props}
if len(required) > 0 {
internal/inventory/busrecords.go
+1 -1
View File
@@ -124,7 +124,7 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
d := broker.Declared{
Module: m.Module,
Emits: m.EmitsAll(),
Emits: m.Emits,
Consumes: fromModules,
Watches: watches,
// The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the
internal/inventory/catalogue.go
-31
View File
@@ -1158,37 +1158,6 @@ func (i *Inventory) SetUpgradeOf(ctx context.Context, module string, u Upgrade)
return nil
}
// CurrentBuild is the build a module is at and whether its policy rolls a new one out (novox/hq
// issue 259).
type CurrentBuild struct {
// Commit is the commit the module's manifest was read at — its `built_from` — and empty for a
// module held without a source.
Commit string
// RollOut is the module's upgrade policy, as Upgrade.RollOut.
RollOut bool
}
// CurrentBuilds is every module's current build and upgrade policy, in one read: what a send records
// it carried, and what a push compares a machine's last send against.
func (i *Inventory) CurrentBuilds(ctx context.Context) (map[string]CurrentBuild, error) {
rows, err := i.store.Pool().Query(ctx,
`select name, coalesce(built_from, ''), upgrade = 'roll-out' from module`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]CurrentBuild{}
for rows.Next() {
var name string
var b CurrentBuild
if err := rows.Scan(&name, &b.Commit, &b.RollOut); err != nil {
return nil, err
}
out[name] = b
}
return out, rows.Err()
}
// Running is every machine assigned a module, in a stable order.
//
// **Assigned, not reported.** A machine that is assigned the module and has not applied it yet is
internal/inventory/doing_test.go
+2 -2
View File
@@ -189,7 +189,7 @@ func TestAMachineIsWaitingWhenWhatItWasSentIsNotWhatItShouldBe(t *testing.T) {
}
// Sent what it should be: not waiting.
if err := inv.RecordSent(ctx, anchor.ID, "aaa", nil); err != nil {
if err := inv.RecordSent(ctx, anchor.ID, "aaa"); err != nil {
t.Fatal(err)
}
waiting, err = inv.Waiting(ctx, map[string]string{"anchor": "aaa", "laptop": "bbb"})
@@ -234,7 +234,7 @@ func TestAMachineWithNothingComputedForItIsNotWaiting(t *testing.T) {
// It has been sent something before, which is what makes this the case the guard is for: a
// machine with a digest and nothing computed for it would compare against the empty string
// and look out of date, when the truth is that nobody worked out what it should be.
if err := inv.RecordSent(ctx, node.ID, "what-it-got-last-time", nil); err != nil {
if err := inv.RecordSent(ctx, node.ID, "what-it-got-last-time"); err != nil {
t.Fatal(err)
}
waiting, err := inv.Waiting(ctx, map[string]string{})
internal/inventory/holdings_test.go
-54
View File
@@ -97,57 +97,3 @@ func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T)
t.Fatalf("a re-told membership did not replace the first: %v", got)
}
}
// A replicated seat has several holders on record (novox/hq ADR 0223): each is added beside the
// others, adding one twice changes nothing, a handover still leaves exactly one, and unassigning one
// takes only its own row.
func TestAReplicatedSeatHasSeveralHoldersOnRecord(t *testing.T) {
resolver := catalogue.Manifest{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}
inv, ctx := aMeshWith(t, resolver)
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
for _, n := range []string{"anchor", "home"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, n, "resolver"); err != nil {
t.Fatal(err)
}
}
if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "anchor", "resolver"); err != nil {
t.Fatal(err)
}
for range 2 {
if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
t.Fatal(err)
}
}
held, err := inv.Holdings(ctx)
if err != nil {
t.Fatal(err)
}
if len(held) != 2 || held[0].Node != "anchor" || held[1].Node != "home" {
t.Fatalf("the two holders are not both on record, once each: %+v", held)
}
if err := inv.Unassign(ctx, "home", "resolver"); err != nil {
t.Fatal(err)
}
if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "anchor" {
t.Fatalf("unassigning one holder took more or less than its own row: %+v", held)
}
if _, err := inv.Assign(ctx, "home", "resolver"); err != nil {
t.Fatal(err)
}
if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
t.Fatal(err)
}
if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
t.Fatal(err)
}
if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "home" {
t.Fatalf("a handover left other holders on record: %+v", held)
}
}
internal/inventory/migrations/0060-the-per-node-resolver-seat-is-gone.sql
-17
View File
@@ -1,17 +0,0 @@
-- The per-node resolver's seat is gone (novox/hq ADR 0220, retiring what ADR 0194 decided).
--
-- ADR 0194 retired `node-dns-resolver` when the mesh moved to one resolver, and kept its row while a
-- machine still held it: removing a seat that is claimed makes the claiming machine unresolvable. On
-- the mesh this was written for, nothing has held it since every node's resolver moved to the mesh's
-- one, and no module in the catalogue claims it, so the row goes.
--
-- **The compiled defaults no longer carry it, and that alone would not remove it.** Seeding adds a
-- seat a release ships and never takes one away (ADR 0122: the table is the live set, and an
-- operator's row is left as it is), so a seat the mesh stops defining stays in the table until
-- something deletes it — this.
--
-- A holding on record goes with it by cascade; there is none. No alias is kept: nothing was renamed,
-- and a manifest still claiming the old name should be refused at registration, naming it, rather
-- than resolve to anything.
delete from seat_alias where seat = 'node-dns-resolver' or alias = 'node-dns-resolver';
delete from seat where name = 'node-dns-resolver';
internal/inventory/migrations/0061-a-send-records-the-builds-it-carried.sql
-14
View File
@@ -1,14 +0,0 @@
-- A send records the build of each module it carried (novox/hq issue 259, ADR 0221).
--
-- A named push ends by sending every other machine whose declaration differs from what it was last
-- sent (ADR 0083). Read from the declaration's digest alone, a module whose upgrade policy records
-- rather than rolls out, or whose plan sends one machine first (ADR 0218), made every machine running
-- it differ, so `push <one machine>` sent all of them the build the policy was holding back. Which
-- build of each module a machine was last sent is what tells a held upgrade from a consequence of the
-- push, and it is not in a digest.
--
-- Module name to the commit its build was made from — the module's `built_from` when the declaration
-- was composed, empty for a module the mesh holds without a source. NULL for a machine last sent
-- before this was kept, or sent a declaration by hand: what it carried is not known, and the push
-- treats such a machine as held until it is pushed by name.
alter table node add column sent_builds jsonb;
internal/inventory/migrations/0062-a-replicated-seat-has-several-holders-on-record.sql
-14
View File
@@ -1,14 +0,0 @@
-- A replicated seat has several holders on record (novox/hq ADR 0223).
--
-- 0039 recorded one holder per seat, keyed by the seat: a handover replaced the row, so the seat was
-- never without a holder in between. The mesh's resolver is now held on more than one machine, each
-- answering the same names, and each of those holders is recorded — added by `seat <name> --add`,
-- never by being assigned. So a holding is keyed by the seat and the assignment holding it.
--
-- Nothing else changes. A handover (`seat <name> --to`) still leaves exactly one row, replacing every
-- holder in one transaction; whether a seat may have more than one is the seat's compiled definition,
-- judged by the controller before a row is added, and a store holding two for any other seat is
-- refused at resolution, naming the seat. Every existing row is one per seat, so it satisfies the new
-- key as it stands.
alter table seat_holding drop constraint seat_holding_pkey;
alter table seat_holding add primary key (seat, node, module);
internal/inventory/migrations/0063-the-resolver-file-is-the-uplinks.sql
-17
View File
@@ -1,17 +0,0 @@
-- What a machine asks for names is the uplink's holder's to write (novox/hq ADR 0223, retiring what
-- ADR 0121 and ADR 0220 decided for node-resolver-config).
--
-- `/etc/resolv.conf` is written by the module holding `node-uplink` — the program that would otherwise
-- rewrite it — so the seat whose holder wrote it, and its need of the uplink beside it, go. Its only
-- claimant, `resolv-conf`, declared nothing for one release while every machine handed the file to its
-- uplink module in one apply, and was then unassigned everywhere and forgotten before this runs.
--
-- **The compiled defaults no longer carry it, and that alone would not remove it**: seeding adds a
-- seat a release ships and never takes one away (ADR 0122), as 0060 found for the per-node resolver.
-- A holding on record goes with it by cascade; a node seat has none. No alias is kept: nothing was
-- renamed, and a manifest still claiming the old name should be refused at registration, naming it.
--
-- Numbered after 0063 (node-hosts-file renamed to node-hostname), which is expected to merge first;
-- if this one lands first, the two are renumbered so the order they merge in is the order they run.
delete from seat_alias where seat = 'node-resolver-config' or alias = 'node-resolver-config';
delete from seat where name = 'node-resolver-config';
internal/inventory/migrations/0064-a-machines-names-are-one-seats.sql
-23
View File
@@ -1,23 +0,0 @@
-- A machine's names are one seat's (novox/hq ADR 0223 part 3): `node-hosts-file` is renamed
-- `node-hostname`, whose holder writes /etc/hostname beside the machine's own lines in /etc/hosts.
--
-- A rename is a database update (ADR 0122): the row keeps its verbs, the former name becomes an alias
-- that resolves to it, so a manifest registered under the old name — the `hosts` module still assigned
-- while machines move to `hostname` — goes on holding the one seat, and two claimants of it on one
-- machine are still refused.
--
-- **Both rows may exist when this runs**, as 0048 found for the artifact store: a controller whose
-- compiled defaults carry the new name may seed it before this migration. Then the old row's holding
-- moves to it and the old row goes; otherwise the old row is renamed. Either way the old name becomes
-- an alias.
update seat_holding set seat = 'node-hostname'
where seat = 'node-hosts-file'
and exists (select 1 from seat where name = 'node-hostname');
delete from seat
where name = 'node-hosts-file'
and exists (select 1 from seat where name = 'node-hostname');
update seat set name = 'node-hostname', decided = 'novox/hq ADR 0199, ADR 0223'
where name = 'node-hosts-file';
insert into seat_alias (alias, seat) values ('node-hosts-file', 'node-hostname')
on conflict (alias) do update set seat = excluded.seat;
update seat_alias set seat = 'node-hostname' where seat = 'node-hosts-file';
internal/inventory/migrations/0065-a-provider-says-which-consumer-it-keeps-failing.sql
-20
View File
@@ -1,20 +0,0 @@
-- A provider says which consumer it keeps failing (novox/hq ADR 0224).
--
-- On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a day and
-- only its journal said so (issue 179). A provider now announces a consumer it has failed for minutes
-- without one success, and the consumer recovering; the controller keeps the newest failing word per
-- provider module, the machine it runs on and the consumer, and removes it on recovery. `status` and
-- `node show` read this table: a row is a problem until it is gone.
create table provider_standing (
module text not null,
provider_node text not null,
consumer text not null,
consumer_node text not null default '',
provision text not null default '',
class text not null default '',
error text not null default '',
since timestamptz not null,
attempts integer not null default 0,
said_at timestamptz not null default now(),
primary key (module, provider_node, consumer)
);
internal/inventory/nodes.go
+5 -40
View File
@@ -851,9 +851,9 @@ func (i *Inventory) DoingOf(ctx context.Context, name string) (Doing, bool, erro
var d Doing
var failed []byte
err = i.store.Pool().QueryRow(ctx,
`select outcome, refused, failed, applied, at, failing_since, failures, coalesce(declared, '')
`select outcome, refused, failed, applied, at, failing_since, failures
from node_report where node = $1`, node.ID).
Scan(&d.Outcome, &d.Refused, &failed, &d.Applied, &d.At, &d.Since, &d.Times, &d.Declared)
Scan(&d.Outcome, &d.Refused, &failed, &d.Applied, &d.At, &d.Since, &d.Times)
if errors.Is(err, pgx.ErrNoRows) {
return Doing{}, false, nil
}
@@ -909,53 +909,18 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
return out, rows.Err()
}
// RecordSent keeps a digest of the declaration a machine was last sent, and the build of each module
// it carried.
// RecordSent keeps a digest of the declaration a machine was last sent.
//
// **A digest rather than the declaration.** The mesh can compute what a machine should be at any
// moment; keeping a copy would be a second account of it, able to disagree with the first. What
// cannot be recomputed is what was *actually sent*, and that is the whole difference between a
// machine that is out of date and one that has never been told.
//
// **And which build of each module** (novox/hq issue 259, ADR 0221): module name to the commit its
// build was made from. A digest cannot say whether a machine differs because a module moved to a build
// its upgrade policy holds back, or because of something a push made — a grant — and only the second
// is a push's to send to a machine it did not name. Nil records that it is not known, as for a
// declaration sent by hand.
func (i *Inventory) RecordSent(ctx context.Context, node, digest string, builds map[string]string) error {
var carried *string
if builds != nil {
raw, err := json.Marshal(builds)
if err != nil {
return err
}
text := string(raw)
carried = &text
}
func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error {
_, err := i.store.Pool().Exec(ctx,
`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb where id = $1`, node, digest, carried)
`update node set sent = $2, sent_at = now() where id = $1`, node, digest)
return err
}
// SentBuilds is the build of each module a machine was last sent, by its name: module to the commit
// its build was made from (novox/hq issue 259). Known is false when that was not kept — a machine
// last sent before it was, sent a declaration by hand, or one the mesh does not know.
func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) {
var raw []byte
err = i.store.Pool().QueryRow(ctx, `select sent_builds from node where name = $1`, name).Scan(&raw)
if errors.Is(err, pgx.ErrNoRows) {
return nil, false, nil
}
if err != nil || raw == nil {
return nil, false, err
}
builds = map[string]string{}
if err := json.Unmarshal(raw, &builds); err != nil {
return nil, false, err
}
return builds, true, nil
}
// RecordSentBusUsers keeps a digest of the bus's user list a machine was just sent, by its name
// (novox/hq issue 249): whether the machine holding the bus must go first is whether this differs
// from the list composed now.
internal/inventory/seats.go
+10 -37
View File
@@ -215,50 +215,23 @@ func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error {
return nil
}
// HoldSeat records that one assignment holds a seat, replacing whoever held it — every holder, for a
// replicated seat (novox/hq ADR 0223) — as one transaction, so the seat is never without a holder in
// between (novox/hq ADR 0131, design 28 task 5.3). The assignment must exist; the store refuses
// otherwise, and that refusal is the right one: a seat cannot be handed to something that is not
// running anywhere.
// HoldSeat records that one assignment holds a seat, replacing whoever held it — as one write, so
// the seat is never without a holder in between (novox/hq ADR 0131, design 28 task 5.3). The
// assignment must exist; the store refuses otherwise, and that refusal is the right one: a seat
// cannot be handed to something that is not running anywhere.
func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module string) error {
node, err := i.NodeByName(ctx, nodeName)
if err != nil {
return err
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
if _, err := tx.Exec(ctx,
`delete from seat_holding where seat = $1 and not (node = $2 and module = $3)`,
seat, node.ID, module); err != nil {
return fmt.Errorf("handing %s to %s on %s: %w", seat, module, nodeName, err)
}
if _, err := tx.Exec(ctx,
_, err = i.store.Pool().Exec(ctx,
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
on conflict (seat, node, module) do update set scope = excluded.scope, since = now()`,
seat, scope, node.ID, module); err != nil {
on conflict (seat) do update set scope = excluded.scope, node = excluded.node,
module = excluded.module, since = now()`,
seat, scope, node.ID, module)
if err != nil {
return fmt.Errorf("recording %s on %s as the holder of %s: %w", module, nodeName, seat, err)
}
return tx.Commit(ctx)
}
// AddSeatHolder records one more assignment holding a replicated seat, beside those already on
// record (novox/hq ADR 0223). Whether the seat may have several holders is the caller's to judge —
// the seat's definition is compiled, and the store holds no column for it. Recording a holder
// already on record changes nothing.
func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, module string) error {
node, err := i.NodeByName(ctx, nodeName)
if err != nil {
return err
}
if _, err := i.store.Pool().Exec(ctx,
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
on conflict (seat, node, module) do nothing`,
seat, scope, node.ID, module); err != nil {
return fmt.Errorf("adding %s on %s as a holder of %s: %w", module, nodeName, seat, err)
}
return nil
}
@@ -267,7 +240,7 @@ func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, mo
func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
rows, err := i.store.Pool().Query(ctx,
`select h.seat, h.scope, n.name, h.module, coalesce(n.site, '')
from seat_holding h join node n on n.id = h.node order by h.seat, n.name, h.module`)
from seat_holding h join node n on n.id = h.node order by h.seat`)
if err != nil {
return nil, err
}
internal/inventory/sent_builds_test.go
-81
View File
@@ -1,81 +0,0 @@
package inventory
import (
"reflect"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// novox/hq issue 259: a send keeps which build of each module it carried. A machine never sent
// anything, or sent with nothing recorded — before this was kept, or by hand — is not known, which
// is not the same as having been sent nothing.
func TestASendKeepsTheBuildsItCarried(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
node, err := inv.AddNode(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
t.Fatalf("a machine never sent anything has known builds %v (%v): %v", builds, known, err)
}
carried := map[string]string{"resolver": "abc123", "network": ""}
if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil {
t.Fatal(err)
}
builds, known, err := inv.SentBuilds(ctx, "anchor")
if err != nil || !known || !reflect.DeepEqual(builds, carried) {
t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err)
}
// Sent with nothing carried: known, and empty.
if err := inv.RecordSent(ctx, node.ID, "d2", map[string]string{}); err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || !known || len(builds) != 0 {
t.Fatalf("an empty send: %v %v %v", builds, known, err)
}
// Sent by hand: not known, and the digest still recorded.
if err := inv.RecordSent(ctx, node.ID, "d3", nil); err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
t.Fatalf("a send whose builds are not known read as %v %v: %v", builds, known, err)
}
if sent, err := inv.Outstanding(ctx, "anchor"); err != nil || sent != "d3" {
t.Fatalf("the digest was not recorded with it: %q %v", sent, err)
}
if _, known, err := inv.SentBuilds(ctx, "nobody"); err != nil || known {
t.Fatalf("a machine the mesh does not know: %v %v", known, err)
}
}
// A module's current build is the commit its manifest was read at, with its upgrade policy.
func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "resolver", Version: "1"},
Source{Repository: "novox/mesh-catalog", BuiltFrom: "c1"}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "by-hand", Version: "1"}, Source{}); err != nil {
t.Fatal(err)
}
if err := inv.SetUpgradeOf(ctx, "by-hand", Upgrade{RollOut: true}); err != nil {
t.Fatal(err)
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
t.Fatal(err)
}
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1"}) {
t.Errorf("resolver is at %+v", got)
}
if got := current["by-hand"]; got != (CurrentBuild{RollOut: true}) {
t.Errorf("a module with no source is at %+v", got)
}
}
internal/inventory/standing.go
-76
View File
@@ -1,76 +0,0 @@
package inventory
import (
"context"
"time"
)
// ProviderStanding is a consumer a provider says it keeps failing (novox/hq ADR 0224).
type ProviderStanding struct {
// Module is the provider's module, and ProviderNode the machine it runs on.
Module string `json:"module"`
ProviderNode string `json:"provider-node"`
// Provision is the interface it provides, e.g. `oidc-client`.
Provision string `json:"provision"`
// Consumer is the identity the mesh derived for the consumer, ConsumerNode its machine.
Consumer string `json:"consumer"`
ConsumerNode string `json:"consumer-node"`
// Class is what kind of failure: credentials-rejected, unreachable, secret-unreadable, refused.
Class string `json:"class"`
Error string `json:"error"`
// Since is when the unbroken run of failures began; Attempts how many it has been.
Since time.Time `json:"since"`
Attempts int `json:"attempts"`
// SaidAt is when the controller last heard it. A provider says it again every quarter of an hour
// while it lasts, so an old one is a provider that stopped saying anything.
SaidAt time.Time `json:"said-at"`
}
// SayAgainWithin is how long a failing standing stays current without being said again: twice the
// quarter of an hour a provider repeats it at. Older, and status says the provider has gone quiet.
const SayAgainWithin = 30 * time.Minute
// Quiet says the provider has not repeated this standing for longer than it would while it lasts.
func (s ProviderStanding) Quiet(now time.Time) bool { return now.Sub(s.SaidAt) > SayAgainWithin }
// KeepStanding records a provider's newest word: failing keeps it, recovered removes it, and says
// whether a recovery removed anything.
func (i *Inventory) KeepStanding(ctx context.Context, failing bool, s ProviderStanding) (bool, error) {
if !failing {
tag, err := i.store.Pool().Exec(ctx,
`delete from provider_standing where module = $1 and provider_node = $2 and consumer = $3`,
s.Module, s.ProviderNode, s.Consumer)
return err == nil && tag.RowsAffected() > 0, err
}
_, err := i.store.Pool().Exec(ctx, `
insert into provider_standing
(module, provider_node, consumer, consumer_node, provision, class, error, since, attempts, said_at)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
on conflict (module, provider_node, consumer) do update set
consumer_node = excluded.consumer_node, provision = excluded.provision,
class = excluded.class, error = excluded.error, since = excluded.since,
attempts = excluded.attempts, said_at = excluded.said_at`,
s.Module, s.ProviderNode, s.Consumer, s.ConsumerNode, s.Provision, s.Class, s.Error, s.Since, s.Attempts)
return false, err
}
// FailingProviders is every consumer a provider last said it keeps failing, oldest run first.
func (i *Inventory) FailingProviders(ctx context.Context) ([]ProviderStanding, error) {
rows, err := i.store.Pool().Query(ctx, `
select module, provider_node, provision, consumer, consumer_node, class, error, since, attempts, said_at
from provider_standing order by since, module, consumer`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []ProviderStanding
for rows.Next() {
var s ProviderStanding
if err := rows.Scan(&s.Module, &s.ProviderNode, &s.Provision, &s.Consumer, &s.ConsumerNode,
&s.Class, &s.Error, &s.Since, &s.Attempts, &s.SaidAt); err != nil {
return nil, err
}
out = append(out, s)
}
return out, rows.Err()
}
internal/inventory/standing_test.go
-130
View File
@@ -1,130 +0,0 @@
package inventory
import (
"slices"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A provider's standing (novox/hq ADR 0224), from the grant that lets it say so to the row status
// reads.
// The broker spells the events itself because it cannot import the catalogue; the two agree.
func TestTheBrokerAndTheCatalogueNameTheSameStandingEvents(t *testing.T) {
if broker.ProvisionerFailing != catalogue.ProvisionerFailing ||
broker.ProvisionerRecovered != catalogue.ProvisionerRecovered {
t.Fatal("the broker and the catalogue disagree about what a provider's standing is called")
}
}
// **Every provider may say it, whatever its manifest lists**: a provider whose manifest forgot the
// events would have its announcement refused by the bus, and fail its consumers as silently as on
// 2026-10-05 (issue 179). A module that receives no contributions provides nothing and is given
// nothing.
func TestEveryProviderIsGrantedItsStandingAndNothingElseIs(t *testing.T) {
provider := catalogue.Manifest{Module: "keycloak", Version: "1",
Emits: []string{"client.created"}, Receives: map[string]string{"oidc-client": "/x/mesh.json"}}
consumer := catalogue.Manifest{Module: "grafana", Version: "1", Emits: []string{"dashboard.saved"}}
d := declaredFor(provider, nil)
for _, e := range []string{"client.created", catalogue.ProvisionerFailing, catalogue.ProvisionerRecovered} {
if !slices.Contains(d.Emits, e) {
t.Fatalf("a provider is not granted %s: %v", e, d.Emits)
}
}
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindModule, Node: "anchor",
Module: "keycloak", Emits: d.Emits})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(perms.Publish, "mesh.mod.keycloak.event.provisioner.failing") {
t.Fatalf("the bus would refuse a provider's standing: %v", perms.Publish)
}
if got := declaredFor(consumer, nil).Emits; slices.Contains(got, catalogue.ProvisionerFailing) {
t.Fatalf("a module that provides nothing was granted a provider's standing: %v", got)
}
// Declared by hand as well: said once.
provider.Emits = append(provider.Emits, catalogue.ProvisionerFailing)
n := 0
for _, e := range provider.EmitsAll() {
if e == catalogue.ProvisionerFailing {
n++
}
}
if n != 1 {
t.Fatalf("%v", provider.EmitsAll())
}
}
// And the controller may hear it from every provider, and only those two events.
func TestTheControllerHearsEveryProvidersStanding(t *testing.T) {
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"} {
if !slices.Contains(perms.Subscribe, want) {
t.Fatalf("the controller may not hear %s: %v", want, perms.Subscribe)
}
}
if slices.Contains(perms.Subscribe, "mesh.mod.*.event.>") {
t.Fatal("the controller hears every event in the mesh")
}
}
func TestAFailingStandingIsKeptUntilItRecovers(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
s := ProviderStanding{Module: "keycloak", ProviderNode: "anchor", Provision: "oidc-client",
Consumer: "mesh_home_grafana", ConsumerNode: "home-server", Class: "credentials-rejected",
Error: "401 invalid_grant", Since: since, Attempts: 60}
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
t.Fatal(err)
}
// Said again: one row, the newest word.
s.Attempts = 31000
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
t.Fatal(err)
}
got, err := inv.FailingProviders(ctx)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].Attempts != 31000 || !got[0].Since.Equal(since) || got[0].ConsumerNode != "home-server" ||
got[0].Class != "credentials-rejected" || got[0].SaidAt.IsZero() {
t.Fatalf("%+v", got)
}
if got[0].Quiet(time.Now()) {
t.Fatal("a standing just said reads as quiet")
}
if !got[0].Quiet(time.Now().Add(SayAgainWithin + time.Minute)) {
t.Fatal("a standing not said again for longer than a provider repeats it does not read as quiet")
}
// The same consumer from another machine's provider is its own row.
other := s
other.ProviderNode = "laptop"
if _, err := inv.KeepStanding(ctx, true, other); err != nil {
t.Fatal(err)
}
cleared, err := inv.KeepStanding(ctx, false, s)
if err != nil || !cleared {
t.Fatalf("recovered cleared nothing: %v %v", cleared, err)
}
cleared, err = inv.KeepStanding(ctx, false, s)
if err != nil || cleared {
t.Fatalf("a recovery for nothing kept said it cleared something: %v %v", cleared, err)
}
got, err = inv.FailingProviders(ctx)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].ProviderNode != "laptop" {
t.Fatalf("%+v", got)
}
}
internal/link/calls.go
-329
View File
@@ -1,329 +0,0 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"regexp"
"strconv"
"sync"
"time"
"github.com/nats-io/nats.go"
)
// A role's tool call, kept after it is answered (novox/hq issue 265).
//
// **A call that outlasts its caller must not end in silence.** A caller waits a bounded time (the
// console thirty seconds), and the bus lets a holder answer a request exactly once and only while the
// server still remembers it was asked (`allow_responses`). Before this, a push that ran longer than
// its caller waited did everything it was asked and its caller read "did not answer in time"; and a
// push whose first act sent the bus's own machine a changed user list had its answer refused
// outright — a broker reloading its users forgets every reply it was about to permit — so the
// answer went nowhere and the only trace was the client library's line on the controller's standard
// error. So every call is answered within AnswerWithin, with its whole answer when it has one by
// then and with "still running as call <id>" when it does not; and every call is kept here, with
// what came of it, including an answer the bus refused, so `calls` can say it.
// AnswerWithin is how long a call runs before its caller is answered that it is still running. Well
// inside the shortest wait of a caller the mesh ships (the console's thirty seconds) and the bus's
// own window for an answer (broker.ResponseTTL), so the one answer a call has is never late for
// either. A variable so a test need not wait.
var AnswerWithin = 10 * time.Second
// KeptCalls is how many calls are kept, newest first; keptAnswer the largest answer kept of a call
// whose caller was sent it. One its caller never had is kept whole.
const (
KeptCalls = 100
keptAnswer = 64 << 10
)
// The states a kept call is in.
const (
CallRunning = "running"
CallAnswered = "answered"
// CallFinishedAfter is a call that finished after its caller was told it was still running: its
// answer is here and nowhere else.
CallFinishedAfter = "finished after its caller was answered"
)
// Call is one call of a role's tool, as `calls` shows it.
type Call struct {
ID string `json:"call"`
Seat string `json:"seat"`
Verb string `json:"verb"`
Args json.RawMessage `json:"arguments,omitempty"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
State string `json:"state"`
// Failed is the call's own answer being an error — an answer, not a timeout.
Failed bool `json:"failed,omitempty"`
// Answer is what the call answered, or would have: kept for a call whose caller did not get it.
Answer json.RawMessage `json:"answer,omitempty"`
// Refused is the bus refusing the answer this holder sent: the caller got nothing, and this is
// the only place that says what it would have.
Refused string `json:"answer refused by the bus,omitempty"`
reply string // the subject the answer went to, which the bus names when it refuses it
}
// CallLog keeps the latest calls a holder served.
type CallLog struct {
mu sync.Mutex
calls []*Call // oldest first
next uint64
now func() time.Time
// Follow is the tool that reads this log back, named in a running answer — set by a holder that
// serves one (the controller's `calls`); without it, the answer points at the holder's journal.
Follow string
}
// Calls is this process's log: one holder process serves its seats on one connection.
var Calls = NewCallLog()
func NewCallLog() *CallLog { return &CallLog{now: time.Now} }
func (l *CallLog) begin(seat, verb string, args json.RawMessage, reply string) *Call {
l.mu.Lock()
defer l.mu.Unlock()
l.next++
c := &Call{ID: "call-" + strconv.FormatInt(l.now().UnixNano(), 10) + "-" + strconv.FormatUint(l.next, 10),
Seat: seat, Verb: verb, Args: args, Started: l.now(), State: CallRunning, reply: reply}
l.calls = append(l.calls, c)
if len(l.calls) > KeptCalls {
l.calls = l.calls[len(l.calls)-KeptCalls:]
}
return c
}
// finish records a call's answer; answeredAlready is its caller having been told it was running.
func (l *CallLog) finish(c *Call, answer []byte, failed, answeredAlready bool) {
l.mu.Lock()
defer l.mu.Unlock()
at := l.now()
c.Finished = &at
c.Failed = failed
c.Answer = append(json.RawMessage(nil), answer...)
if !answeredAlready && len(answer) > keptAnswer {
// Its caller has it; kept only in case the bus refuses it, and a whole declaration a
// hundred times over is memory nobody asked for.
c.Answer, _ = json.Marshal(map[string]any{"not kept": fmt.Sprintf(
"an answer of %d bytes, sent to its caller in full", len(answer))})
}
if answeredAlready {
c.State = CallFinishedAfter
} else {
c.State = CallAnswered
}
}
// Recent is the kept calls, newest first, as copies.
func (l *CallLog) Recent() []Call {
l.mu.Lock()
defer l.mu.Unlock()
out := make([]Call, 0, len(l.calls))
for i := len(l.calls) - 1; i >= 0; i-- {
out = append(out, *l.calls[i])
}
return out
}
// Get is one kept call.
func (l *CallLog) Get(id string) (Call, bool) {
l.mu.Lock()
defer l.mu.Unlock()
for _, c := range l.calls {
if c.ID == id {
return *c, true
}
}
return Call{}, false
}
// kept is what a call's arguments are kept as: each argument by name, a value only when it is short
// and not one that carries settings or a secret — `calls` answers anyone who may call the seat.
func kept(args json.RawMessage) json.RawMessage {
var given map[string]any
if json.Unmarshal(args, &given) != nil {
return nil
}
out := map[string]any{}
for k, v := range given {
s, isString := v.(string)
switch {
case k == "values" || k == "secret":
out[k] = "(given, not kept)"
case isString && len(s) <= 120:
out[k] = s
case isString:
out[k] = s[:120] + "…"
default:
out[k] = v
}
}
body, _ := json.Marshal(out)
return body
}
// refusedPublish is the bus's words for a publish it refused, with the subject.
var refusedPublish = regexp.MustCompile(`Permissions Violation for Publish to "([^"]+)"`)
// Refusal records the bus refusing an answer, from the error the client library hands the
// connection's error handler. It reports whether the error was the refusal of a kept call's answer.
func (l *CallLog) Refusal(err error, logger *log.Logger) bool {
if err == nil {
return false
}
m := refusedPublish.FindStringSubmatch(err.Error())
if m == nil {
return false
}
l.mu.Lock()
var hit *Call
for i := len(l.calls) - 1; i >= 0; i-- {
if c := l.calls[i]; c.reply != "" && c.reply == m[1] {
hit = c
break
}
}
if hit == nil {
l.mu.Unlock()
return false
}
at := l.now()
hit.Refused = fmt.Sprintf("%s: %s", at.Format(time.RFC3339), err)
id, verb, took := hit.ID, hit.Seat+"."+hit.Verb, at.Sub(hit.Started).Round(time.Second)
l.mu.Unlock()
if logger != nil {
// Said in the mesh's words, beside the library's own line: which call, and where its answer is.
logger.Printf("the bus refused the answer to %s (%s, asked %s ago): its caller got no answer. "+
"What it answered is kept under %s. A broker reloading its user list while a call runs "+
"forgets that it may be answered", id, verb, took, id)
}
return true
}
// WatchRefusals chains the connection's error handler so a refused answer is recorded against its
// call. The handler the dialler set — the library's default, which prints — still runs after it.
func (l *CallLog) WatchRefusals(conn *nats.Conn, logger *log.Logger) {
if conn == nil {
return
}
watched.Lock()
defer watched.Unlock()
if watched.conns == nil {
watched.conns = map[*nats.Conn]bool{}
}
if watched.conns[conn] {
return
}
watched.conns[conn] = true
before := conn.ErrorHandler()
conn.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, err error) {
if errors.Is(err, nats.ErrPermissionViolation) {
l.Refusal(err, logger)
}
if before != nil {
before(c, s, err)
}
})
}
var watched struct {
sync.Mutex
conns map[*nats.Conn]bool
}
// answerNow is how a handler asks for its caller to be answered before it goes on (Acknowledge).
type answerNowKey struct{}
// Acknowledge answers the call's caller now that it is running, rather than after AnswerWithin. For
// a handler about to do what makes its answer unsendable: a push sends the bus's own machine first,
// and a broker reloading its user list forgets every answer it was about to permit. Without effect
// outside a served call, and after the first time.
func Acknowledge(ctx context.Context) {
if f, ok := ctx.Value(answerNowKey{}).(func()); ok {
f()
}
}
// running is the interim answer: what a caller reads when the call has not finished.
func running(c *Call, within time.Duration, acknowledged bool, follow string) []byte {
why := fmt.Sprintf("it has not finished in %s", within)
if acknowledged {
why = "it answers before it starts, because what it does can leave the bus unable to carry a later answer"
}
next := "its holder's journal says how it ended."
if follow != "" {
next = fmt.Sprintf("%s with call %q says what came of it.", follow, c.ID)
}
said := fmt.Sprintf("%s.%s is running as %s — %s. This is not a failure, and it may already have "+
"done what was asked: %s", c.Seat, c.Verb, c.ID, why, next)
body, _ := json.Marshal(map[string]any{"result": map[string]any{
"running": true, "call": c.ID, "started": c.Started, "output": said,
}})
return body
}
// serveCall runs one call and answers it once: in full when the handler returns within AnswerWithin
// and has not acknowledged, and otherwise that it is running — its answer then kept, and logged.
func (l *CallLog) serveCall(seat, verb string, args json.RawMessage, reply string, handle ToolHandler,
respond func([]byte) error, logger *log.Logger) {
ctx, cancel := context.WithTimeout(context.Background(), HandlerTimeout)
defer cancel()
if len(args) == 0 {
args = json.RawMessage(`{}`)
}
c := l.begin(seat, verb, kept(args), reply)
acknowledged := make(chan struct{})
var once sync.Once
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
type outcome struct {
body []byte
failed bool
}
done := make(chan outcome, 1)
go func() {
var body []byte
result, err := handle(ctx, args)
failed := err != nil
if err != nil {
body, _ = json.Marshal(map[string]any{"error": err.Error()})
} else if body, err = json.Marshal(map[string]any{"result": result}); err != nil {
failed = true
body, _ = json.Marshal(map[string]any{"error": "the answer could not be written as JSON: " + err.Error()})
}
done <- outcome{body, failed}
}()
say := func(body []byte) {
if err := respond(body); err != nil && logger != nil {
logger.Printf("%s.%s: could not answer %s: %v", seat, verb, c.ID, err)
}
}
timer := time.NewTimer(AnswerWithin)
defer timer.Stop()
select {
case o := <-done:
l.finish(c, o.body, o.failed, false)
say(o.body)
return
case <-acknowledged:
say(running(c, AnswerWithin, true, l.Follow))
case <-timer.C:
say(running(c, AnswerWithin, false, l.Follow))
}
o := <-done
l.finish(c, o.body, o.failed, true)
if logger != nil {
how := "and it succeeded"
if o.failed {
how = "and it answered an error"
}
logger.Printf("%s (%s.%s) finished after %s, after its caller was told it was running, %s — its answer is kept under %s", c.ID,
seat, verb, time.Since(c.Started).Round(time.Second), how, c.ID)
}
}
internal/link/calls_test.go
-174
View File
@@ -1,174 +0,0 @@
package link
import (
"bytes"
"context"
"encoding/json"
"errors"
"log"
"strings"
"sync"
"testing"
"time"
)
// answers collects what a call answered, and fails a second answer: the bus permits one.
type answers struct {
t *testing.T
mu sync.Mutex
got [][]byte
sent chan struct{}
}
func newAnswers(t *testing.T) *answers { return &answers{t: t, sent: make(chan struct{}, 4)} }
func (a *answers) respond(body []byte) error {
a.mu.Lock()
defer a.mu.Unlock()
a.got = append(a.got, body)
if len(a.got) > 1 {
a.t.Errorf("a call was answered %d times; the bus refuses every answer after the first", len(a.got))
}
a.sent <- struct{}{}
return nil
}
func (a *answers) only() map[string]any {
a.mu.Lock()
defer a.mu.Unlock()
if len(a.got) != 1 {
a.t.Fatalf("%d answers, want exactly one", len(a.got))
}
var out map[string]any
if err := json.Unmarshal(a.got[0], &out); err != nil {
a.t.Fatal(err)
}
return out
}
func shortWindow(t *testing.T, d time.Duration) {
t.Helper()
was := AnswerWithin
AnswerWithin = d
t.Cleanup(func() { AnswerWithin = was })
}
// A call that finishes in time answers in full, once, and is kept as answered.
func TestACallThatFinishesInTimeAnswersInFull(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
l.serveCall("mesh-controller", "status", nil, "_INBOX.x.1", func(context.Context, json.RawMessage) (any, error) {
return "all well", nil
}, a.respond, nil)
if got := a.only()["result"]; got != "all well" {
t.Fatalf("answered %v", got)
}
recent := l.Recent()
if len(recent) != 1 || recent[0].State != CallAnswered || string(recent[0].Args) != "{}" {
t.Fatalf("kept %+v", recent)
}
}
// **A call that outlasts AnswerWithin is answered that it is running, with its id** — before its
// caller gives up — and what it finally answered is kept under that id, not dropped (issue 265).
func TestACallThatOutlastsTheWindowSaysItIsRunningAndKeepsItsAnswer(t *testing.T) {
shortWindow(t, 20*time.Millisecond)
l, a := NewCallLog(), newAnswers(t)
var logged bytes.Buffer
release := make(chan struct{})
finished := make(chan struct{})
go func() {
l.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor"}`), "_INBOX.x.2",
func(context.Context, json.RawMessage) (any, error) {
<-release
return "anchor told", nil
}, a.respond, log.New(&logged, "", 0))
close(finished)
}()
<-a.sent
got := a.only()["result"].(map[string]any)
id, _ := got["call"].(string)
if got["running"] != true || id == "" || !strings.Contains(got["output"].(string), id) {
t.Fatalf("the running answer does not name its call: %v", got)
}
if c, _ := l.Get(id); c.State != CallRunning {
t.Fatalf("while it runs it is kept as %q", c.State)
}
close(release)
<-finished
c, ok := l.Get(id)
if !ok || c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "anchor told") {
t.Fatalf("its answer was not kept: %+v", c)
}
if !strings.Contains(logged.String(), id) {
t.Errorf("finishing late was not said: %q", logged.String())
}
}
// A handler that acknowledges is answered then, not when it ends: a push answers before it sends.
func TestAnAcknowledgedCallIsAnsweredBeforeItGoesOn(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
done := make(chan struct{})
go func() {
l.serveCall("mesh-controller", "push", nil, "_INBOX.x.3", func(ctx context.Context, _ json.RawMessage) (any, error) {
Acknowledge(ctx)
Acknowledge(ctx) // a second time is nothing
select {
case <-a.sent: // the caller was answered before the work goes on
case <-time.After(5 * time.Second):
t.Error("acknowledging did not answer the caller")
}
return "sent", nil
}, a.respond, nil)
close(done)
}()
<-done
if got := a.only()["result"].(map[string]any); got["running"] != true {
t.Fatalf("an acknowledged call answered %v", got)
}
if c := l.Recent()[0]; c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "sent") {
t.Fatalf("kept %+v", c)
}
}
// **A refused answer is recorded against its call, in the mesh's words** — not only as the client
// library's line on standard error, which is all there was on 2026-10-05.
func TestARefusedAnswerIsKeptAgainstItsCall(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
reply := "_INBOX.laptop.node-tools.jJ4DRJFnZYvkPUBKYwUG5v.LNRyztuX"
l.serveCall("mesh-controller", "push", nil, reply, func(context.Context, json.RawMessage) (any, error) {
return "told", nil
}, a.respond, nil)
var logged bytes.Buffer
refusal := errors.New(`nats: permissions violation: Permissions Violation for Publish to "` + reply + `" on connection [838]`)
if !l.Refusal(refusal, log.New(&logged, "", 0)) {
t.Fatal("the refusal of a kept call's answer was not recognised")
}
c := l.Recent()[0]
if c.Refused == "" || !strings.Contains(logged.String(), c.ID) {
t.Fatalf("the refusal is not kept or not said: %+v / %q", c, logged.String())
}
other := errors.New(`nats: permissions violation: Permissions Violation for Publish to "mesh.node.x" on connection [1]`)
if l.Refusal(other, nil) || l.Refusal(errors.New("nats: timeout"), nil) {
t.Error("an unrelated error was taken for a refused answer")
}
}
// What `calls` keeps of the arguments never carries settings or a secret.
func TestACallKeepsNoSettingsOrSecrets(t *testing.T) {
got := string(kept(json.RawMessage(`{"module":"m","values":"{\"token\":\"s3cret\"}","secret":"s3cret"}`)))
if strings.Contains(got, "s3cret") || !strings.Contains(got, `"module":"m"`) {
t.Fatalf("kept %s", got)
}
}
// Only the newest KeptCalls are kept.
func TestTheLogKeepsTheNewest(t *testing.T) {
l := NewCallLog()
for i := 0; i < KeptCalls+5; i++ {
l.begin("s", "v", nil, "")
}
recent := l.Recent()
if len(recent) != KeptCalls || !strings.HasSuffix(recent[0].ID, "-105") || !strings.HasSuffix(recent[KeptCalls-1].ID, "-6") {
t.Fatalf("kept %d, newest %s, oldest %s", len(recent), recent[0].ID, recent[len(recent)-1].ID)
}
}
internal/link/enrolment.go
-19
View File
@@ -410,25 +410,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
if err := e.Inventory.RecordCarried(ctx, report.Node, report.Carried); err != nil {
return false, err
}
// **An account of a declaration the mesh has moved past never replaces the account it keeps**
// (novox/hq issue 267). The machine-facts half of such a report is kept above, whenever it
// arrives; this half is the machine's word on what it did with what it was sent, and the release
// plan reads it as such. A reconcile that held a machine to the older declaration a moment before
// the newer one arrived reported *after* the newer apply's report, and stored last, it read as the
// machine never having applied what it was sent — the plan waited until somebody pushed by hand.
// One row per node means the last write wins, so the order of arrival must not decide it.
if report.Declared != "" {
sent, err := e.Inventory.Outstanding(ctx, report.Node)
if err != nil {
return false, err
}
if Superseded(report.Declared, sent) {
log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
return false, e.Inventory.Seen(ctx, node.ID)
}
}
// **Whether this is news** is the store's answer: it holds the previous report, and a machine
// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
internal/link/heard_test.go
+1 -65
View File
@@ -100,7 +100,7 @@ func TestABareAliveDoesNotWipeTheDeclarationThatSaysANodeIsCurrent(t *testing.T)
}
// The mesh sent this node a declaration, and the node applied it and named which by digest.
const digest = "d640d1b6a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f6071829304152"
if err := inv.RecordSent(ctx, node.ID, digest, nil); err != nil {
if err := inv.RecordSent(ctx, node.ID, digest); err != nil {
t.Fatal(err)
}
if _, err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{
@@ -260,67 +260,3 @@ func TestWhatFiltersAMachineIsKeptFromItsReport(t *testing.T) {
t.Fatalf("the next report did not replace what filters the machine: %+v", again)
}
}
// Measured on the home server (novox/hq issue 267): the mesh sent d2; the machine applied it and
// reported, and a reconcile's report about d1 — made just before d2 arrived — reached the mesh after
// it. Stored last, it read as the machine never having applied d2, and the release plan waited on a
// report it had already been given.
func TestAnAccountOfAnOlderDeclarationDoesNotReplaceTheNewer(t *testing.T) {
inv := inventory.ForTest(t)
ctx := context.Background()
node, err := inv.AddNode(ctx, "home-server")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
t.Fatal(err)
}
heard := link.Enrolment{Inventory: inv}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d2",
Applied: []string{"a", "b"}, Outward: []string{"eth0"}}); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d1",
Applied: []string{"a"}, Outward: []string{"eth1"}}); err != nil {
t.Fatal(err)
}
doing, said, err := inv.DoingOf(ctx, "home-server")
if err != nil || !said {
t.Fatalf("no account kept: %v", err)
}
if doing.Declared != "d2" || doing.Applied != 2 {
t.Fatalf("the older report replaced the account of what was sent: %+v", doing)
}
// What it says about the machine is kept whenever it arrives, as before.
if links, err := inv.OutwardLinksOf(ctx, "home-server"); err != nil || len(links) != 1 || links[0] != "eth1" {
t.Fatalf("what the older report said about the machine was not kept: %v %v", links, err)
}
}
// The account of the declaration that is outstanding replaces whatever was kept, and so does one
// from a machine nothing was ever recorded as sent to.
func TestAnAccountOfTheSentDeclarationReplacesTheKeptOne(t *testing.T) {
inv := inventory.ForTest(t)
ctx := context.Background()
node, err := inv.AddNode(ctx, "home-server")
if err != nil {
t.Fatal(err)
}
heard := link.Enrolment{Inventory: inv}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d1", Applied: []string{"a"}}); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d2", Applied: []string{"a", "b"}}); err != nil {
t.Fatal(err)
}
doing, _, err := inv.DoingOf(ctx, "home-server")
if err != nil {
t.Fatal(err)
}
if doing.Declared != "d2" || doing.Applied != 2 {
t.Fatalf("the account of what was sent was not kept: %+v", doing)
}
}
internal/link/merges.go
-79
View File
@@ -1,79 +0,0 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// AnnouncedMerge is one merge the forge announced, as the events stream holds it: what it said, and
// when the bus took it.
type AnnouncedMerge struct {
SourceMoved
// At is when the bus took the announcement — the stream's own time, not the forge's.
At time.Time
// Seq is its place in the events stream.
Seq uint64
}
// MergedSubject is where the forge's merges land: the controller's own follow of them.
var MergedSubject = broker.ControllerFollows[3]
// readQuiet is how long a read of the stream waits for one more message before it takes the stream
// as read to its end. The stream answers at once when it holds something; this is only the wait at
// the end.
const readQuiet = 2 * time.Second
// AnnouncedMerges is every merge the forge announced since a moment, oldest first, read back from
// the events stream (novox/hq issue 266).
//
// **Read on a consumer of its own, filtered on the one subject.** The controller's durable consumer
// carries several filters, and the bus server the mesh ran when this was written (2.10) skips a
// message now and then on a consumer with more than one filter: it moves its delivered pointer past
// the message without ever handing it over, so the controller never hears of it and nothing says so.
// A consumer filtered on a single subject reads the stream another way and was not seen to skip. This
// one is ordered, ephemeral and acknowledges nothing, so reading it changes nothing on the bus.
func (s *Server) AnnouncedMerges(ctx context.Context, since time.Time) ([]AnnouncedMerge, error) {
if s.js == nil {
return nil, errors.New("this control plane is not on the bus, so it cannot read what the forge announced")
}
sub, err := s.js.Context().SubscribeSync(MergedSubject, nats.OrderedConsumer(), nats.StartTime(since))
if err != nil {
return nil, fmt.Errorf("reading the forge's merges from the events stream: %w", err)
}
defer func() { _ = sub.Unsubscribe() }()
var out []AnnouncedMerge
for {
wait, cancel := context.WithTimeout(ctx, readQuiet)
msg, err := sub.NextMsgWithContext(wait)
cancel()
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
// Nothing more within the quiet wait: the stream has been read to its end.
break
}
meta, err := msg.Metadata()
if err != nil {
continue
}
var moved SourceMoved
if json.Unmarshal(msg.Data, &moved) == nil && moved.Commit != "" && moved.Repo != "" {
out = append(out, AnnouncedMerge{SourceMoved: moved, At: meta.Timestamp, Seq: meta.Sequence.Stream})
}
if meta.NumPending == 0 {
break
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Seq < out[j].Seq })
return out, nil
}
internal/link/receive.go
-8
View File
@@ -34,9 +34,6 @@ const (
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
KindSourceMoved = "source-moved"
KindCatchUp = "catch-up"
// KindProvisioner is a provider saying a consumer has failed for minutes, or recovered
// (novox/hq ADR 0224).
KindProvisioner = "provisioner"
)
// Control is one thing a node or a module said, as the controller must act on it.
@@ -57,11 +54,6 @@ type Control interface {
// Body is the message itself — the payload alone, never the envelope.
Body() []byte
// Subject is where it was published. For a module's event it names the emitter, which the bus
// enforces (only a module may publish into its own namespace), so who said it is read from here
// and never from the body.
Subject() string
// Redelivered says the bus has handed this message over before. An enrolment cares and
// nothing else does: one already spent is not finished a second time.
Redelivered() bool
internal/link/receive_fake_test.go
-2
View File
@@ -61,7 +61,6 @@ func (c *fakeInbound) retries(ctx context.Context, s *Server) {
type fakeControl struct {
kind string
subject string
body []byte
tag uint64
to *settled
@@ -72,7 +71,6 @@ type fakeControl struct {
func (m *fakeControl) Kind() string { return m.kind }
func (m *fakeControl) Body() []byte { return m.body }
func (m *fakeControl) Subject() string { return m.subject }
func (m *fakeControl) Redelivered() bool { return m.redelivered }
func (m *fakeControl) About(string) {}
func (m *fakeControl) Answer(context.Context, []byte) error { return nil }
internal/link/receive_nats.go
+3 -25
View File
@@ -53,7 +53,7 @@ func Nats(js *broker.JetStream) Inbound {
// whatever was asked for — and not at all when nothing was.
func (n *natsInbound) Also(kind string) error {
switch kind {
case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner:
case KindModuleMoved, KindCatchUp, KindSourceMoved:
n.follows[kind] = true
return nil
default:
@@ -241,30 +241,9 @@ func kindOfSubject(subject string) (string, bool) {
// runs (ADR 0190): the old builder still answers on the retired seat until it is unassigned.
return KindBuilt, true
}
if _, ok := ProvisionerEmitter(subject); ok {
return KindProvisioner, true
}
return "", false
}
// ProvisionerEmitter is the module a provider's standing event came from, read from its subject
// (`mesh.mod.<module>.event.provisioner.<failing|recovered>`); false for any other subject. The
// controller's own follow pattern, with `*` for the module, decodes too.
func ProvisionerEmitter(subject string) (string, bool) {
rest, ok := strings.CutPrefix(subject, "mesh.mod.")
if !ok {
return "", false
}
module, event, ok := strings.Cut(rest, ".event.")
if !ok || module == "" || strings.Contains(module, ".") {
return "", false
}
if event != broker.ProvisionerFailing && event != broker.ProvisionerRecovered {
return "", false
}
return module, true
}
// natsControl is one message from the bus being built, as the controller reads it.
type natsControl struct {
kind string
@@ -277,9 +256,8 @@ type natsControl struct {
delivered uint64
}
func (m *natsControl) Kind() string { return m.kind }
func (m *natsControl) Body() []byte { return m.msg.Data }
func (m *natsControl) Subject() string { return m.msg.Subject }
func (m *natsControl) Kind() string { return m.kind }
func (m *natsControl) Body() []byte { return m.msg.Data }
// Redelivered is what the server counted, not what the controller remembers. Which is the answer to
// a question the AMQP side could only guess at across a restart: an enrolment redelivered because
internal/link/seattools.go
+21 -7
View File
@@ -25,8 +25,8 @@ type ToolHandler func(ctx context.Context, args json.RawMessage) (any, error)
// SeatToolSubject is where a mesh-scoped seat's tool is asked (design 33 §4).
func SeatToolSubject(seat, verb string) string { return "mesh.seat." + seat + ".tool." + verb }
// HandlerTimeout bounds one call. Its caller is answered within AnswerWithin either way; this is how
// long the call itself may run before it is stopped.
// HandlerTimeout bounds one answer. A verb that runs a command — a push, a build with no wait —
// answers in seconds; anything that has not in this long is said to have not answered.
const HandlerTimeout = 5 * time.Minute
// RebindAfter is how long a refused subscription waits before it is tried again.
@@ -62,17 +62,31 @@ func (b OverNATS) serveTools(seat string, subjectOf func(string) string, handler
_ = s.Unsubscribe()
}
}
// A refused answer is recorded against its call, not only printed by the library.
Calls.WatchRefusals(b.Conn, logger)
for verb, handle := range handlers {
verb, handle := verb, handle
subject := subjectOf(verb)
bind := func() (*nats.Subscription, error) {
return b.Conn.QueueSubscribe(subject, "seat."+seat, func(msg *nats.Msg) {
// Its own goroutine per call: a slow `push` must not hold up a `status` asked beside it,
// and the library would otherwise run handlers one after another. Answered once, within
// AnswerWithin, and kept (novox/hq issue 265).
go Calls.serveCall(seat, verb, json.RawMessage(msg.Data), msg.Reply, handle, msg.Respond, logger)
// and the library would otherwise run handlers one after another.
go func() {
ctx, cancel := context.WithTimeout(context.Background(), HandlerTimeout)
defer cancel()
args := json.RawMessage(msg.Data)
if len(args) == 0 {
args = json.RawMessage(`{}`)
}
var reply []byte
result, err := handle(ctx, args)
if err != nil {
reply, _ = json.Marshal(map[string]any{"error": err.Error()})
} else if reply, err = json.Marshal(map[string]any{"result": result}); err != nil {
reply, _ = json.Marshal(map[string]any{"error": "the answer could not be written as JSON: " + err.Error()})
}
if err := msg.Respond(reply); err != nil && logger != nil {
logger.Printf("%s: could not answer: %v", subject, err)
}
}()
})
}
sub, err := bind()
internal/link/serve.go
-7
View File
@@ -79,8 +79,6 @@ type Server struct {
recorder Recorder
upgrader Upgrader
replayer Replayer
// standings keeps what providers say about their consumers (novox/hq ADR 0224).
standings Standings
log *log.Logger
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
@@ -155,9 +153,6 @@ func (s *Server) Serve(ctx context.Context) error {
if s.replayer != nil {
s.log.Printf("answering %s", KindCatchUp)
}
if s.standings != nil {
s.log.Printf("keeping every provider's %s", KindProvisioner)
}
return s.inbound.Receive(ctx, s.act)
}
@@ -178,8 +173,6 @@ func (s *Server) act(ctx context.Context, m Control) {
s.sourceMoved(ctx, m)
case KindCatchUp:
s.catchingUp(ctx, m)
case KindProvisioner:
s.provisioner(ctx, m)
default:
// Dropped: a message nothing understands will not be understood on the next attempt
// either, and asking for it again would spin.
internal/link/standing.go
-114
View File
@@ -1,114 +0,0 @@
package link
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
)
// A provider's standing (novox/hq ADR 0224).
//
// **A provider that keeps failing a consumer is a problem the controller reports**, not a line in a
// journal. On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a
// day — its admin no longer took the mesh's secret once its database was moved — and every surface
// the mesh has called the mesh well (novox/hq issue 179). A provider now says, as an event, a
// consumer it has failed for minutes without one success, and the consumer recovering; the
// controller keeps the newest word per provider, machine and consumer, and `status` names each one
// still failing.
// Standing is one provider's word about one consumer.
type Standing struct {
// Module is the emitter, read from the subject the bus let it publish on — never from the body.
Module string `json:"-"`
// Failing is which of the two it said: failing, or recovered.
Failing bool `json:"-"`
Provider string `json:"provider"`
ProviderNode string `json:"provider-node"`
Consumer string `json:"consumer"`
Node string `json:"node"`
Class string `json:"class,omitempty"`
Error string `json:"error,omitempty"`
Since time.Time `json:"since"`
Attempts int `json:"attempts"`
// Why is said with a recovery that is not a success: `withdrawn`, a consumer no longer asked for.
Why string `json:"why,omitempty"`
}
// Standings keeps what providers say about their consumers.
type Standings interface {
// Stood records a provider's newest word about a consumer: a failing one kept, a recovered one
// cleared — and says whether a recovery cleared anything, since a provider announces its first
// success for every consumer after it starts. An error the store is away for is held and asked
// again, like a report.
Stood(ctx context.Context, s Standing) (cleared bool, err error)
}
// Watches says where providers' standings are kept, and asks for them to be delivered.
func (s *Server) Watches(st Standings) error {
if err := s.inbound.Also(KindProvisioner); err != nil {
return err
}
s.standings = st
return nil
}
// ReadStanding is one standing event as the controller understands it, from its subject and body.
func ReadStanding(subject string, body []byte) (Standing, error) {
module, ok := ProvisionerEmitter(subject)
if !ok {
return Standing{}, fmt.Errorf("%s is not a provider's standing", subject)
}
var st Standing
if err := json.Unmarshal(body, &st); err != nil {
return Standing{}, fmt.Errorf("%s's standing could not be read: %w", module, err)
}
if st.Consumer == "" {
return Standing{}, fmt.Errorf("%s's standing named no consumer", module)
}
st.Module = module
st.Failing = strings.HasSuffix(subject, "."+broker.ProvisionerFailing)
return st, nil
}
// provisioner acts on one standing event.
//
// **A recovery must not be lost.** A failing standing is said again every quarter of an hour while
// it lasts, so one dropped is replaced; a recovery is said once, and dropping it would leave status
// naming a consumer that is fine. So a store that is away holds the message, as a report is held.
func (s *Server) provisioner(ctx context.Context, m Control) {
if s.standings == nil {
// Delivered because the consumer's filter names it, with nothing here keeping it: taken,
// because handing it back would not give it anywhere to go.
_ = m.Took()
return
}
st, err := ReadStanding(m.Subject(), m.Body())
if err != nil {
s.log.Printf("%v; ignored", err)
_ = m.Took()
return
}
cleared, err := s.standings.Stood(ctx, st)
what := fmt.Sprintf("%s's standing for %s", st.Module, st.Consumer)
switch s.decide(ctx, m, what, "", "", err) {
case Hold:
return
case Stale, GiveUp:
_ = m.Took()
return
}
if err != nil {
s.log.Printf("%s could not be kept: %v", what, err)
} else if st.Failing {
s.log.Printf("%s on %s is FAILING %s on %s (%s, %d attempts since %s): %s", st.Module,
st.ProviderNode, st.Consumer, st.Node, st.Class, st.Attempts, st.Since.Format(time.RFC3339), st.Error)
} else if cleared {
s.log.Printf("%s on %s recovered %s", st.Module, st.ProviderNode, st.Consumer)
}
_ = m.Took()
}
internal/link/standing_test.go
-203
View File
@@ -1,203 +0,0 @@
package link
import (
"context"
"sync"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
)
// A provider's standing (novox/hq ADR 0224): who said it is read from the subject the bus let it
// publish on, a failing one is kept and a recovery cleared, and a recovery is never lost to a store
// that is away — said once, it would leave status naming a consumer that is fine.
type keptStandings struct {
kept []Standing
err error
}
func (k *keptStandings) Stood(_ context.Context, s Standing) (bool, error) {
if k.err != nil {
return false, k.err
}
k.kept = append(k.kept, s)
return !s.Failing, nil
}
func standingSays(t *testing.T, in *fakeInbound, to *settled, subject string, body map[string]any) Control {
t.Helper()
m := in.sends(t, to, KindProvisioner, body).(*fakeControl)
m.subject = subject
return m
}
func TestTheControllerFollowsEveryProvidersStandingAndNothingElse(t *testing.T) {
for subject, want := range map[string]string{
"mesh.mod.keycloak.event.provisioner.failing": "keycloak",
"mesh.mod.postgres.event.provisioner.recovered": "postgres",
"mesh.mod.*.event.provisioner.failing": "*",
} {
got, ok := ProvisionerEmitter(subject)
if !ok || got != want {
t.Errorf("%s: %q %v", subject, got, ok)
}
if kind, _ := kindOfSubject(subject); kind != KindProvisioner {
t.Errorf("%s decodes to %q", subject, kind)
}
}
for _, subject := range []string{
"mesh.mod.keycloak.event.provisioner.other",
"mesh.mod.keycloak.event.client.created",
"mesh.mod.a.b.event.provisioner.failing",
"mesh.seat.keycloak.event.provisioner.failing",
} {
if _, ok := ProvisionerEmitter(subject); ok {
t.Errorf("%s read as a provider's standing", subject)
}
}
var follows int
for _, s := range broker.ControllerFollows {
if kind, _ := kindOfSubject(s); kind == KindProvisioner {
follows++
}
}
if follows != 2 {
t.Fatalf("the controller follows %d standing subjects, want failing and recovered", follows)
}
}
func TestAFailingStandingIsKeptNamingTheEmitterFromTheSubject(t *testing.T) {
s, in := serving()
kept := &keptStandings{}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
to := &settled{}
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing", map[string]any{
"provider": "oidc-client", "provider-node": "anchor", "consumer": "mesh_home_grafana",
"node": "home-server", "class": "credentials-rejected", "error": "401 invalid_grant",
"since": since, "attempts": 31000,
// A body naming another module is not believed: the subject is the bus's word.
"module": "postgres",
}))
if !to.acked || len(kept.kept) != 1 {
t.Fatalf("settled %+v, kept %+v", to, kept.kept)
}
got := kept.kept[0]
if got.Module != "keycloak" || !got.Failing || got.Consumer != "mesh_home_grafana" || got.Node != "home-server" ||
got.ProviderNode != "anchor" || got.Class != "credentials-rejected" || got.Attempts != 31000 || !got.Since.Equal(since) {
t.Fatalf("%+v", got)
}
to = &settled{}
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.recovered", map[string]any{
"provider": "oidc-client", "provider-node": "anchor", "consumer": "mesh_home_grafana",
}))
if !to.acked || len(kept.kept) != 2 || kept.kept[1].Failing {
t.Fatalf("settled %+v, kept %+v", to, kept.kept)
}
}
func TestARecoveryIsHeldWhileTheStoreIsAway(t *testing.T) {
s, in := serving()
kept := &keptStandings{err: restarting}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
to := &settled{}
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.recovered",
map[string]any{"consumer": "mesh_home_grafana"}))
if !to.unsettled() || len(in.held) != 1 {
t.Fatalf("a recovery was settled while the store was away: %+v", to)
}
kept.err = nil
in.retries(t.Context(), s)
if !to.acked || len(kept.kept) != 1 {
t.Fatalf("the held recovery was not kept when the store came back: %+v %+v", to, kept.kept)
}
}
func TestAStandingThatNamesNoConsumerIsTakenAndForgotten(t *testing.T) {
s, in := serving()
kept := &keptStandings{}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
to := &settled{}
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing", map[string]any{}))
if !to.acked || len(kept.kept) != 0 {
t.Fatalf("%+v %+v", to, kept.kept)
}
}
func TestAStandingWithNothingKeepingItIsTaken(t *testing.T) {
s, in := serving()
to := &settled{}
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing",
map[string]any{"consumer": "x"}))
if !to.acked {
t.Fatal("a standing nothing keeps was left for the bus to hand over again")
}
}
// Over a real bus: a provider's standing published under its own module's namespace reaches the
// controller through the events consumer's filter — the one wildcard filter on it — names the emitter
// from the subject, and is acknowledged.
func TestNatsAProvidersStandingReachesTheController(t *testing.T) {
js := aBus(t)
kept := &lockedStandings{}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(&toldAbout{}); err != nil {
t.Fatal(err)
}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
eventually(t, "the controller's event consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil
})
for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} {
if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event,
[]byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil {
t.Fatal(err)
}
}
// Somebody else's event under the same prefix is not the controller's to hear.
if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil {
t.Fatal(err)
}
eventually(t, "both standings being kept, in order, naming the emitter", func() bool {
got := kept.all()
return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing
})
eventually(t, "both being acknowledged and nothing else delivered", func() bool {
info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2
})
}
type lockedStandings struct {
mu sync.Mutex
kept []Standing
}
func (l *lockedStandings) Stood(_ context.Context, s Standing) (bool, error) {
l.mu.Lock()
defer l.mu.Unlock()
l.kept = append(l.kept, s)
return !s.Failing, nil
}
func (l *lockedStandings) all() []Standing {
l.mu.Lock()
defer l.mu.Unlock()
return append([]Standing(nil), l.kept...)
}
internal/overlay/generator.go
+68 -15
View File
@@ -31,18 +31,19 @@ const Requirement = "private-network"
// Name is the module that answers it with WireGuard.
//
// **It writes no names.** A machine's mesh names are answered by the mesh's one resolver (novox/hq
// ADR 0194), and /etc/hosts is the file of one module, the holder of `node-hostname` (ADR 0199,
// ADR 0223): the controller writes into no file another seat's holder owns. If the mesh ever needs a line there, it
// asks that holder to register it. This module asked for a `node-names` fact written into /etc/hosts
// until 2026-10-05; the host gives that region back at the first push without it.
// **The names went with it.** A mesh-names module used to sit beside this — it wrote /etc/hosts
// and ran nothing, which is not a module. Being on the private network is what gives a machine a
// name, so this module asks for the `node-names` fact and the mesh writes the file. The
// name-resolution provision went the same way: names are facts the mesh computes, not something a
// module that runs nowhere can provide.
const Name = "mesh-wireguard"
// Addressing is the mesh handing out addresses on the private network itself.
//
// The mesh's resolver answers names from it, which is why it requires this rather than a private
// network in general. A different VPN that hands out its own addresses would come with its own
// names — the mesh has nothing to answer about a machine whose address it did not choose.
// Names are computed from it, which is why they require this rather than a private network in
// general. A different VPN that hands out its own addresses would come with its own names — the
// mesh has nothing to write about a machine whose address it did not choose. Saying so here is
// what keeps a machine from being given a hosts file full of addresses that mean nothing.
const Addressing = "mesh-addressing"
// TheNetwork is what a machine can only have one of.
@@ -78,13 +79,17 @@ type Generator struct {
// keyPath is where each node keeps the private half it generated. Named rather than carried:
// the mesh has never seen it and never will.
keyPath string
// No registry. Being on the network is still what grants a machine the right to pull from the
// mesh's artifact store in the clear (novox/hq ADR 0082), but the runtime's file is the runtime's
// module's: the private network writes nothing into it, and that module states the registry
// itself, told where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR
// 0222, issue 190).
// registry is the mesh's artifact store as the network reaches it (host:port), or empty when
// the mesh has none. Being on the network is what grants a machine the right to pull from it
// (novox/hq ADR 0082), so the module that puts a machine on the network is what writes the
// runtime's trust — the same reasoning that has it write /etc/hosts.
registry string
}
// TrustRegistry names the artifact store this network's machines pull from in the clear —
// the overlay is the transport security (ADR 0082).
func (g *Generator) TrustRegistry(hostPort string) { g.registry = hostPort }
// From builds a generator over the machines that are part of the network.
//
// The nodes given are the ones assigned the module — not every node the mesh knows. A machine
@@ -123,7 +128,31 @@ func (g *Generator) Resources(node string) ([]map[string]any, bool, error) {
if err := json.Unmarshal(raw, &parsed); err != nil {
return nil, false, err
}
return parsed.Resources, true, nil
resources := parsed.Resources
if g.registry != "" {
trust, err := json.Marshal(map[string]any{"insecure-registries": []string{g.registry}})
if err != nil {
return nil, false, err
}
resources = append(resources,
map[string]any{
// Written into, not over (novox/hq ADR 0102): the runtime's daemon file is the
// machine's — its data directory, its logging, whatever a predecessor set — and
// this states one fact in it. The host sets this key and keeps every other.
// ("merge" is the operator's settings merged into this content; "into" is the
// content written into the machine's file.) The registry speaks plain HTTP
// because every path to it is already inside the overlay's encryption (ADR 0082).
"id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json",
"content": string(trust) + "\n", "mode": "0644", "merge": "json", "into": "json",
},
map[string]any{
// Reloaded, not restarted: the runtime re-reads its trusted registries on a reload,
// and a restart stops every container on the machine (measured; ADR 0102).
"id": "registry-trust-reload", "type": "service", "unit": "docker.service",
"state": "running", "reload-on": []string{"registry-trust"},
})
}
return resources, true, nil
}
// Nodes are the machines this generator was built over, so a caller can say who is on the network.
@@ -132,6 +161,20 @@ func (g *Generator) Nodes() []Node { return g.nodes }
// Graph is the peer list per node, for showing.
func (g *Generator) Graph() Graph { return g.graph }
// hostsTemplate is the mesh's region of `/etc/hosts` — every machine's mesh name at its private
// address, written into a marked region and merged (RosterFile.Shared → `into: block`), so the rest
// of the file (localhost, the machine's own name, other tools' blocks) is kept byte for byte
// (novox/hq issue 128). It is a roster template like any module's: the mesh owns the data, this owns
// the format, and the control plane holds no formatter.
//
// - No floor: no header, no localhost, no `127.0.1.1` — those are the machine's, above the region.
// - A machine's own line is marked, and its mesh name resolves to its mesh address, not loopback.
// - `.Names` is the machines (novox/hq ADR 0191): a route's internal name is under its node's
// internal domain and the resolver answers it by wildcard, and a public name is public DNS's.
// Machines with no address yet are already left out of the set.
const hostsTemplate = "# The mesh's names. This region is replaced whenever a machine joins or leaves.\n" +
"{{range .Names}}{{.Address}}\t{{.FQDN}}{{if ne .Name .FQDN}}\t{{.Name}}{{end}}{{if eq .Name $.Node}}\t# this machine{{end}}\n{{end}}"
// Manifest is the module the mesh provides for itself.
//
// It ships with the control plane rather than coming from a repository, because the thing that
@@ -143,7 +186,17 @@ func Manifest() map[string]any {
"version": "1",
"computed": Name,
"provides": []string{Requirement, Addressing},
"claims": []map[string]any{{"name": TheNetwork, "scope": "node"}},
// Being on the private network is what gives a machine a name, so the module that puts it
// there is what writes them. Asked for rather than generated by a module of its own: the
// mesh knows which machines exist and where; writing that into a hosts file is not a thing
// that needs a module to run nowhere. The format is a template like any other roster fact —
// the mesh's own module owns the `/etc/hosts` layout the way dnsmasq owns its zones, and the
// control plane holds no formatter (see catalogue.RosterFile). `shared`: the mesh owns only
// its region of the file and keeps the rest (novox/hq issue 128).
"facts": map[string]any{
"node-names": map[string]any{"path": "/etc/hosts", "template": hostsTemplate, "shared": true},
},
"claims": []map[string]any{{"name": TheNetwork, "scope": "node"}},
}
}
internal/overlay/generator_test.go
+47 -13
View File
@@ -1,13 +1,15 @@
package overlay
import (
"fmt"
"strings"
"testing"
)
// novox/hq ADR 0222, issue 190: the runtime's file and service are the runtime's module's. The
// private network writes nothing into either — being on it still grants the right to pull from the
// mesh's store in the clear (ADR 0082), and the runtime's module states that trust itself.
func TestTheNetworkWritesNothingOfTheRuntimes(t *testing.T) {
func TestTheNetworkCarriesRegistryTrust(t *testing.T) {
// novox/hq ADR 0082: being on the network is what grants a machine the right to pull from the
// mesh's artifact store in the clear, so the network module writes the runtime's trust — and
// writes nothing when the mesh has no store to trust.
nodes := []Node{
{Name: "anchor", Site: "lab", Hub: true, Endpoint: "192.0.2.10:51820", Key: "k1", Address: "10.42.0.1"},
{Name: "node2", Site: "lab", Key: "k2", Address: "10.42.0.2"},
@@ -16,15 +18,47 @@ func TestTheNetworkWritesNothingOfTheRuntimes(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "node2"} {
resources, part, err := g.Resources(node)
if err != nil || !part {
t.Fatalf("%s: resources: %v part=%v", node, err, part)
}
for _, r := range resources {
if r["path"] == "/etc/docker/daemon.json" || r["unit"] == "docker.service" {
t.Errorf("%s: the private network declares the runtime's %v", node, r)
}
plain, _, err := g.Resources("node2")
if err != nil {
t.Fatal(err)
}
for _, r := range plain {
if r["id"] == "registry-trust" {
t.Fatal("trust was written with no artifact store to trust")
}
}
g.TrustRegistry("anchor.internal:5000")
trusted, part, err := g.Resources("node2")
if err != nil || !part {
t.Fatalf("resources: %v part=%v", err, part)
}
var file, service map[string]any
for _, r := range trusted {
switch r["id"] {
case "registry-trust":
file = r
case "registry-trust-reload":
service = r
}
}
if file == nil || service == nil {
t.Fatalf("the trust file or its reload is missing: %v", trusted)
}
if file["path"] != "/etc/docker/daemon.json" || file["merge"] != "json" || file["into"] != "json" {
t.Fatalf("the trust is not written into daemon.json (ADR 0102): %v", file)
}
if content, _ := file["content"].(string); !strings.Contains(content, `"anchor.internal:5000"`) {
t.Fatalf("the trust does not name the store: %v", file["content"])
}
if service["unit"] != "docker.service" {
t.Fatalf("the reload is not the runtime's: %v", service)
}
// Reloaded, never restarted: a restart stops every container on the machine (ADR 0102).
if _, restarts := service["restart-on"]; restarts {
t.Fatalf("the runtime is restarted for its trust: %v", service)
}
if fmt.Sprint(service["reload-on"]) != "[registry-trust]" {
t.Fatalf("the runtime is not reloaded for its trust: %v", service)
}
}
internal/overlay/names.go
+17 -2
View File
@@ -20,6 +20,20 @@ const SuffixVar = "MESH_INTERNAL_SUFFIX"
// rather than reaching a stranger's machine.
const DefaultSuffix = "internal"
// HostsPath is where the names go.
//
// This is not the `/etc/hosts` floor the design removes. That floor existed because a node had to
// reach the mesh's database before its own DNS worked — a fallback for a circularity, and the
// circularity is gone. This is the mechanism itself: the complete set of names in this mesh
// (novox/hq ADR 0011). Written as a marked region *into* the file rather than as the file: the
// rest of it — `localhost`, the machine's own name, other tools' blocks — is the machine's, and
// writing it whole replaced all of that (novox/hq issue 128).
//
// A file rather than a resolver daemon, deliberately, for now: it works on every Linux, needs no
// package, and has no failure mode of its own. A daemon becomes necessary when names are wanted
// that are not one-per-node — service names, wildcards — and that is not yet true.
const HostsPath = "/etc/hosts"
// Suffix is what internal names end in.
func Suffix() string {
if v := strings.TrimSpace(os.Getenv(SuffixVar)); v != "" {
@@ -34,6 +48,7 @@ var nodeName = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`)
// InternalName is a node's name inside the mesh.
func InternalName(node string) string { return node + "." + Suffix() }
// **What remains of a larger file.** The rest wrote /etc/hosts, which the controller no longer
// writes at all (novox/hq ADR 0199): the mesh's resolver answers these names. The naming stays here, because several things compose a node's internal name and one of them
// **What remains of a larger file.** The rest wrote /etc/hosts — that is the `node-names` fact now
// (catalogue.FactsInto), computed where the graph lives instead of by a module that ran nothing.
// The naming stays here, because several things compose a node's internal name and one of them
// writing the suffix differently would be a name nothing answers to.