novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Compare commits

merge into: novox/mesh-controller:main
Branches Tags
novox/mesh-controller:main novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module novox/mesh-controller:fix/the-runtime-file-is-the-runtimes novox/mesh-controller:feat/no-change-takes-effect-unseen novox/mesh-controller:feat/build-edges-are-recorded novox/mesh-controller:fix/several-contributions-collide-in-grants-v2
..
pull from: novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module
Branches Tags
novox/mesh-controller:main novox/mesh-controller:fix/the-registry-trust-test-reads-the-runtimes-module novox/mesh-controller:fix/the-runtime-file-is-the-runtimes novox/mesh-controller:feat/no-change-takes-effect-unseen novox/mesh-controller:feat/build-edges-are-recorded novox/mesh-controller:fix/several-contributions-collide-in-grants-v2
1 Commits
main .. fix/the-registry-trust-test-reads-the-runtimes-module
Author SHA1 Message Date
jochen 92442d377c The registry-trust test reads the runtime's module, which now writes the trust (ADR 0222) 2026-10-05 22:46:57 +02:00
60 changed files with 413 additions and 3707 deletions
Expand all files Collapse all files
cmd/mesh-controller/build.go
-3
View File
@@ -676,9 +676,6 @@ type answers struct {
// refused, until the switch — and while there is any, the mesh is not all well: the order the
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
unheld []catalogue.Unheld
// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
// journal was the only place that said so for a day (04-ISSUES/179).
failing []inventory.ProviderStanding
}
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
cmd/mesh-controller/calls_verb_test.go
-81
View File
@@ -1,81 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
// RequestTimeout) — the shortest wait of a caller the mesh ships.
const consoleWaits = 30 * time.Second
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
if link.AnswerWithin >= consoleWaits/2 {
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
}
if link.AnswerWithin >= broker.ResponseTTL {
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
}
}
// A push — named or through command — answers before it runs: it sends the machine holding the bus
// first, and the broker reloading its user list forgets the answer it was about to permit.
func TestAPushAnswersBeforeItSends(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want bool
}{
{"push", map[string]any{"node": "anchor"}, true},
{"push", map[string]any{}, true},
{"command", map[string]any{"command": "push anchor"}, true},
{"command", map[string]any{"command": "push --behind"}, true},
{"command", map[string]any{"command": "builds"}, false},
{"status", map[string]any{}, false},
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil {
t.Fatalf("%s %v: %v", c.verb, c.args, err)
}
if got := answersFirst(argv); got != c.want {
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
}
}
}
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
handlers, behind, err := seatToolHandlers()
if err != nil || len(behind) != 0 {
t.Fatalf("%v %v", behind, err)
}
calls, ok := handlers["calls"]
if !ok {
t.Fatal("calls is not served")
}
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
!strings.Contains(err.Error(), `"node"`) {
t.Errorf("calls took an argument it does not declare: %v", err)
}
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
!strings.Contains(err.Error(), "not across a restart") {
t.Errorf("an unknown call was not said plainly: %v", err)
}
got, err := calls(context.Background(), json.RawMessage(`{}`))
if err != nil {
t.Fatal(err)
}
if _, listed := got.(map[string]any)["calls"]; !listed {
t.Errorf("calls answered %v", got)
}
}
cmd/mesh-controller/holdings.go
-71
View File
@@ -6,8 +6,6 @@ import (
"sort"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
@@ -92,72 +90,3 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
}
return said, nil
}
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
//
// **The holders on record, and only the sole claimant when there are none** — the same answer the
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
// listed here. A holder off the private network is left out: a resolver named at an address nothing
// answers is a lookup that waits out its timeout on every name.
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
var replicated []catalogue.Seat
for _, s := range catalogue.Seats() {
if s.Replicated && s.Scope == catalogue.ScopeMesh {
replicated = append(replicated, s)
}
}
if len(replicated) == 0 {
return nil, nil
}
recorded, err := inv.Holdings(ctx)
if err != nil {
return nil, err
}
places, err := onTheNetwork(ctx, inv, shelf)
if err != nil {
return nil, err
}
address := map[string]string{}
for _, p := range places {
address[p.Name] = p.Address
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return nil, err
}
out := map[string]map[string]string{}
for _, seat := range replicated {
var nodes []string
for _, h := range recorded {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
nodes = append(nodes, h.Node)
}
}
if len(nodes) == 0 {
var derived []string
for _, e := range entries {
for _, c := range e.Manifest.Claims {
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
derived = append(derived, e.On...)
}
}
}
if len(derived) == 1 {
nodes = derived
}
}
at := map[string]string{}
for _, n := range nodes {
if address[n] != "" {
at[overlay.InternalName(n)] = address[n]
}
}
out[seat.Name] = at
}
return out, nil
}
cmd/mesh-controller/main.go
-1
View File
@@ -194,7 +194,6 @@ func usage() {
seats [--json] every seat this mesh defines, what it delivers, and who holds it
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
board [--listen ADDR] the same three questions, as a page that holds nothing
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
assign <node> <module>... put modules on a node, judged together (ADR 0207)
cmd/mesh-controller/missed_merges.go
-131
View File
@@ -1,131 +0,0 @@
package main
import (
"context"
"fmt"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
//
// The forge's poll announces every merge on the events stream, and the controller acts on what its
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
// server moved the consumer past it — a fault of consumers with several filters in the server the
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
// built from that repository stayed behind and were built by hand.
//
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
// would still move something was never acted on — it is said, and acted on now.
const (
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
mergeGrace = 10 * time.Minute
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
mergeLookBack = 24 * time.Hour
// mergeCatchUpEvery is how often the stream is read back.
mergeCatchUpEvery = 5 * time.Minute
)
// merges is what reads back the forge's announcements; the link server, or a test's list.
type merges interface {
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
}
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
f := following{open}
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
entries, err := open.inventory.Catalogued(ctx)
if err != nil {
return nil, nil, err
}
read, err := open.inventory.ReadRepositories(ctx)
return entries, read, err
}
failing := ""
tick := time.NewTicker(mergeCatchUpEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
}
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
fmt.Printf(format+"\n", args...)
})
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
why := ""
if err != nil {
why = err.Error()
}
if why != failing {
if why != "" {
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
} else {
fmt.Println("merges the bus may not have handed over are looked for again")
}
failing = why
}
}
}
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
// move something is said and acted on, oldest first.
//
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
// because acting on an earlier missed merge of the same repository may have moved what a later one
// would have.
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
if err != nil {
return err
}
entries, read, err := catalogued(ctx)
if err != nil {
return err
}
for _, a := range all {
if now.Sub(a.At) < mergeGrace {
continue
}
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
continue
}
if entries, read, err = catalogued(ctx); err != nil {
return err
}
moves := wouldMove(a.SourceMoved, entries, read)
if len(moves) == 0 {
continue
}
var names []string
for _, e := range moves {
names = append(names, e.Manifest.Module)
}
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
isAre(len(names)))
if err := act(ctx, a.SourceMoved); err != nil {
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
a.Owner, a.Repo, a.Commit, err)
continue
}
if entries, read, err = catalogued(ctx); err != nil {
return err
}
}
return nil
}
cmd/mesh-controller/missed_merges_test.go
-180
View File
@@ -1,180 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// announcedList is the events stream's merges as a test gives them.
type announcedList []link.AnnouncedMerge
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
var out []link.AnnouncedMerge
for _, m := range a {
if !m.At.Before(since) {
out = append(out, m)
}
}
return out, nil
}
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
type aCatalogue struct {
entries []inventory.Entry
acted []string
fail error
}
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
return append([]inventory.Entry(nil), c.entries...), nil, nil
}
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
return func(_ context.Context, m link.SourceMoved) error {
if c.fail != nil {
return c.fail
}
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
for _, moved := range wouldMove(m, c.entries, nil) {
for i := range c.entries {
if c.entries[i].Manifest.Module == moved.Manifest.Module {
c.entries[i].Source.Head = m.Commit
c.entries[i].Source.Seen = now()
}
}
}
return nil
}
}
func at(s string) time.Time {
t, err := time.Parse(time.RFC3339, s)
if err != nil {
panic(err)
}
return t
}
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
return link.AnnouncedMerge{
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
MergedAt: mergedAt, Paths: paths,
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
At: at(onTheBus),
}
}
func built(module, repo, path, commit, seen string) inventory.Entry {
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
return e
}
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
// events stream; the bus never handed it to the controller, which acted on the merges around it and
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
// move something — so it is said and acted on, once, and only after the controller's own consumer
// has had its time with it.
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
cat := &aCatalogue{entries: []inventory.Entry{
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
// Acted on when it was announced: looked at after it was merged.
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
}}
stream := announcedList{
// Nothing the mesh holds is built from the records repository.
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
// Acted on: its module was looked at since.
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
// Never handed over.
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
"node-tools/internal/console/console.go"),
}
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
clock := at("2026-10-05T22:50:00Z")
now := func() time.Time { return clock }
pass := func() {
t.Helper()
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
}
pass()
if len(cat.acted) != 0 {
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
}
clock = at("2026-10-05T22:58:00Z")
pass()
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
}
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
t.Fatalf("the missed merge was not said as one: %q", said)
}
clock = at("2026-10-05T23:03:00Z")
pass()
if len(cat.acted) != 1 || len(said) != 1 {
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
}
}
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
// that could not be acted on is said and tried again on the next pass.
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
cat := &aCatalogue{
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
fail: errors.New("the store is restarting"),
}
stream := announcedList{
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
"modules/plex/module.json", "modules/plex/x.ts"),
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
}
var said []string
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
clock := at("2026-10-05T22:00:00Z")
now := func() time.Time { return clock }
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
t.Fatalf("a failed catch-up was not said: %q", said)
}
cat.fail = nil
clock = at("2026-10-05T22:05:00Z")
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
t.Fatal(err)
}
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
}
}
// A merge older than the look-back is left to the operator: a controller that did not run this
// missed it, and acting on it days later would be a surprise rebuild.
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
clock := at("2026-10-05T22:00:00Z")
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
func(string, ...any) {}); err != nil {
t.Fatal(err)
}
if len(cat.acted) != 0 {
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
}
}
cmd/mesh-controller/modules.go
+1 -1
View File
@@ -661,7 +661,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
// durable subscription nobody reads.
if consumer, needed := broker.ConsumerFor(broker.Principal{
Kind: broker.KindModule, Node: node, Module: m.Module,
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
}); needed {
if busAddress == "" {
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
cmd/mesh-controller/nodes.go
-13
View File
@@ -505,19 +505,6 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
fmt.Printf(" public domain %s\n", domain)
}
// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
// capabilities, because it is something not working now and they are a description.
failing, err := failingProviders(ctx, inv)
if err != nil {
return err
}
if here := failingOn(failing, name); len(here) > 0 {
fmt.Printf("\n %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
for _, line := range failingLines(here, time.Now()) {
fmt.Printf(" %s\n", line)
}
}
held, err := inv.Profile(ctx, name)
if err != nil {
return err
cmd/mesh-controller/plan.go
+1 -8
View File
@@ -707,13 +707,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
return catalogue.Rendering{}, inventory.Node{}, err
}
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
// lists every holder of the mesh's resolver.
replicas, err := replicatedHolders(ctx, inv, shelf)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
// before it had an address on the private network. A machine joins through the tunnel now, and
@@ -790,7 +783,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines, Zones: zones, Holders: replicas,
Machines: machines, Zones: zones,
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
cmd/mesh-controller/push.go
+2 -28
View File
@@ -127,19 +127,11 @@ func serve(ctx context.Context) error {
if err := server.Follows(following{open}); err != nil {
return err
}
// And the merges the bus announced and never handed over, read back on a timer and acted on late
// rather than never (novox/hq issue 266).
go catchingUpOnMerges(ctx, open, server)
// And a catalogue that has just started, asking for what it missed. The same type answers
// both: what a build meant and what the builds were are two questions about one record.
if err := server.Answers(following{open}); err != nil {
return err
}
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
// 0224): a provider's journal must not be the only place that says so.
if err := server.Watches(standings{inv}); err != nil {
return err
}
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
// from the store's row, so what the seat declares is what is answered.
@@ -158,8 +150,6 @@ func serve(ctx context.Context) error {
if !isNATS {
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
}
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
if err != nil {
return err
@@ -327,7 +317,7 @@ func pushCommand(ctx context.Context, args []string) error {
if len(needsOne) == 0 {
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
// must never look the same.
fmt.Println("no machine named: a push of the whole mesh, and every machine is doing what it was told — nothing sent")
fmt.Println("every machine is doing what it was told")
return nil
}
}
@@ -368,18 +358,6 @@ func pushCommand(ctx context.Context, args []string) error {
}
asked = append(asked, n.Name)
}
// **A push that named no machine says so, first.** Through the console a machine the caller
// meant to name could be lost on the way (novox/hq issue 244): the call arrived empty, ran as
// `push --behind`, and every machine behind was pushed by someone who thought they had pushed one.
// Its whole-mesh reach is the first line of the answer, with the machines it is about to send.
if len(args) == 0 {
which := "every machine"
if *behind {
which = "every machine that is behind"
}
fmt.Printf("no machine named: this is a push of the WHOLE mesh — %s (%d): %s\n",
which, len(asked), strings.Join(asked, ", "))
}
// **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's
// user list, and a module's new grants are refused by the bus until that list says them. Among
@@ -447,11 +425,7 @@ func pushCommand(ctx context.Context, args []string) error {
return err
}
release()
told := make([]string, 0, len(sending))
for _, r := range sending {
told = append(told, r.node)
}
fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", "))
fmt.Printf("\n%d node(s) told\n", len(sending))
reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
cmd/mesh-controller/readable.go
-6
View File
@@ -78,11 +78,6 @@ type meshStatus struct {
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
// dependency is met. Reported, not refused, until the switch.
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
// Failing is every consumer a provider says it keeps failing, with the class of error, since
// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
// document without this called the mesh well while the identity provider refused every consumer
// for a day (04-ISSUES/179).
Failing []inventory.ProviderStanding `json:"failing,omitempty"`
}
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
@@ -215,7 +210,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
}
}
out.Unheld = asked.unheld
out.Failing = asked.failing
for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]})
cmd/mesh-controller/rollout.go
+1 -1
View File
@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
// would bury the ones that matter under a list nobody can act on.
func speaksOnTheBus(m catalogue.Manifest) bool {
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
}
cmd/mesh-controller/seats.go
+9 -35
View File
@@ -29,13 +29,11 @@ type seatHolder struct {
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
type seatRow struct {
Seat string `json:"seat"`
Scope string `json:"scope"`
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
Replicated bool `json:"replicated,omitempty"`
Holders []seatHolder `json:"holders"`
Seat string `json:"seat"`
Scope string `json:"scope"`
Delivers string `json:"delivers,omitempty"`
Decision string `json:"decision"`
Holders []seatHolder `json:"holders"`
}
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -49,7 +47,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
rows := make([]seatRow, 0, len(seats))
for _, s := range seats {
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
Replicated: s.Replicated, Holders: []seatHolder{}}
Holders: []seatHolder{}}
seen := map[seatHolder]bool{}
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -106,13 +104,9 @@ func seatCommand(ctx context.Context, args []string) error {
return nil
}
if len(args) == 3 && args[1] == "--to" {
return handOver(ctx, args[0], args[2], false)
return handOver(ctx, args[0], args[2])
}
if len(args) == 3 && args[1] == "--add" {
return handOver(ctx, args[0], args[2], true)
}
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
"seat <name> --add <node>/<module>")
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
}
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
@@ -125,12 +119,7 @@ func seatCommand(ctx context.Context, args []string) error {
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
// and refusing to record a handover to a module the node has not started would make the handover
// impossible to do before the switch instead of as the switch.
//
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
// records the named assignment as a further holder and leaves every holder on record as it is. A
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
// the one named, as it always did.
func handOver(ctx context.Context, seatName, to string, adding bool) error {
func handOver(ctx context.Context, seatName, to string) error {
nodeName, module, ok := strings.Cut(to, "/")
if !ok || nodeName == "" || module == "" {
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
@@ -146,10 +135,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
if !known {
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
}
if adding && !seat.Replicated {
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
}
assigned, err := inv.Assigned(ctx, nodeName)
if err != nil {
return err
@@ -172,7 +157,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
}
var was string
var held []string
holdings, err := inv.Holdings(ctx)
if err != nil {
return err
@@ -180,7 +164,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
for _, h := range holdings {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
was = h.Node
held = append(held, h.Node)
}
}
@@ -204,15 +187,6 @@ func handOver(ctx context.Context, seatName, to string, adding bool) error {
} else if err := catalogue.CanHold(*m, seat); err != nil {
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
}
if adding {
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
}
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
strings.Join(held, ", "))
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
return nil
}
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
}
cmd/mesh-controller/seatverbs.go
+37 -286
View File
@@ -36,194 +36,19 @@ type verbAnswer struct {
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
//
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
// the verb declares but did not use for the command line it composed — given beside another that
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
// not take that" would have stopped it before anything was sent.
func argvFor(verb string, args map[string]any) ([]string, error) {
a, err := readArguments(verb, args)
if err != nil {
return nil, err
str := func(key string) string {
v, _ := args[key].(string)
return strings.TrimSpace(v)
}
argv, err := a.commandLine()
if len(a.misread) > 0 {
// The table and the command line disagree: the verb reads an argument no caller can see
// in its schema, so no caller could ever pass it.
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
"table and its command lines disagree", verb, quoteAll(a.misread))
}
if err != nil {
return nil, err
}
if unused := a.unused(); len(unused) > 0 {
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
}
return argv, nil
}
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
// command line was composed from.
type verbArguments struct {
verb string
given map[string]string
used map[string]bool
declared map[string]bool
misread []string // arguments the command line read that the schema does not declare: a bug here
}
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
// wrote.
func controllerVerb(name string) (catalogue.Verb, bool) {
for _, v := range catalogue.ControllerVerbs {
if v.Name == name {
return v, true
}
}
return catalogue.Verb{}, false
}
// declaredArguments are a schema's properties, and which of them are switches.
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
switches = map[string]bool{}
props, _ := v.Input["properties"].(map[string]any)
for name, p := range props {
names = append(names, name)
desc, _ := p.(map[string]any)
switch enum := desc["enum"].(type) {
case []string:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
case []any:
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
}
}
sort.Strings(names)
return names, switches
}
// readArguments refuses what the verb does not take, before anything is composed.
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
v, known := controllerVerb(verb)
if !known {
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
names, switches := declaredArguments(v)
declared := map[string]bool{}
for _, n := range names {
declared[n] = true
}
takes := "none"
if len(names) > 0 {
takes = quoteAll(names)
}
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
keys := make([]string, 0, len(args))
for k := range args {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if !declared[k] {
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
}
var value string
switch x := args[k].(type) {
case nil:
continue
case string:
value = strings.TrimSpace(x)
case bool:
if !switches[k] {
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
}
value = fmt.Sprint(x)
default:
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
}
if switches[k] {
switch value {
case "true":
case "false", "":
continue // said and off: the same as not given, and nothing passed over
default:
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
need := func(keys ...string) error {
for _, k := range keys {
if str(k) == "" {
return fmt.Errorf("%s needs %q", verb, k)
}
}
if value != "" {
a.given[k] = value
}
return nil
}
return a, nil
}
// str is one argument's value, marked as used.
func (a *verbArguments) str(key string) string {
if !a.declared[key] {
a.misread = append(a.misread, key)
}
a.used[key] = true
return a.given[key]
}
// on is a switch, marked as used.
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
// need refuses a call missing a required argument, in the verb's own words.
func (a *verbArguments) need(keys ...string) error {
for _, k := range keys {
if a.str(k) == "" {
return fmt.Errorf("%s needs %q", a.verb, k)
}
}
return nil
}
// unused are the arguments given that the command line was not composed from.
func (a *verbArguments) unused() []string {
var out []string
for k := range a.given {
if !a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
return out
}
func (a *verbArguments) usedGiven() []string {
var out []string
for k := range a.given {
if a.used[k] {
out = append(out, k)
}
}
sort.Strings(out)
if len(out) == 0 {
return []string{"nothing"}
}
return out
}
func quoteAll(xs []string) string {
q := make([]string, len(xs))
for i, x := range xs {
if x == "nothing" {
q[i] = x
continue
}
q[i] = fmt.Sprintf("%q", x)
}
return strings.Join(q, ", ")
}
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
func (a *verbArguments) commandLine() ([]string, error) {
verb, str, on, need := a.verb, a.str, a.on, a.need
switch verb {
case "command":
// The generic verb: the command line as given, split as a shell would split it, with
@@ -240,8 +65,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, errors.New("command names no command")
}
return argv, nil
case "tools":
return nil, errors.New("tools is answered from the records, not by a command")
case "status":
return []string{"status", "--json"}, nil
case "nodes":
@@ -259,14 +82,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
if id := str("log"); id != "" {
return []string{"builds", "--log", id}, nil
}
argv := []string{"builds"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
if m := str("module"); m != "" {
argv = append(argv, m)
return []string{"builds", m}, nil
}
return argv, nil
return []string{"builds"}, nil
case "plans":
if r := str("repository"); r != "" {
argv := []string{"plans", "--what-if", r}
@@ -278,19 +97,19 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return argv, nil
}
for _, act := range []string{"stop", "close", "retry"} {
if id := str(act); id != "" {
return []string{"plans", act, id}, nil
}
if id := str("stop"); id != "" {
return []string{"plans", "stop", id}, nil
}
if id := str("close"); id != "" {
return []string{"plans", "close", id}, nil
}
if id := str("retry"); id != "" {
return []string{"plans", "retry", id}, nil
}
if id := str("id"); id != "" {
return []string{"plans", id}, nil
}
argv := []string{"plans"}
if n := str("limit"); n != "" {
argv = append(argv, "-n", n)
}
return argv, nil
return []string{"plans"}, nil
// The build queue (novox/hq ADR 0219).
case "queue":
return []string{"queue"}, nil
@@ -300,7 +119,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return []string{verb, str("id")}, nil
case "clear":
if on("dead") {
if str("dead") == "true" {
return []string{"clear", "--dead"}, nil
}
return []string{"clear"}, nil
@@ -314,10 +133,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, err
}
argv := []string{"replay", str("id")}
if on("register") {
if str("register") == "true" {
argv = append(argv, "--register")
}
if on("older") {
if str("older") == "true" {
argv = append(argv, "--older")
}
return argv, nil
@@ -330,9 +149,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
if err := need("node"); err != nil {
return nil, err
}
if on("files") {
return []string{"plan", str("node"), "--files"}, nil
}
return []string{"plan", str("node"), "--json"}, nil
case "assign", "unassign":
if err := need("node", "module"); err != nil {
@@ -356,14 +172,8 @@ func (a *verbArguments) commandLine() ([]string, error) {
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
// time out on every machine that takes a minute, and say nothing about the ones that did not.
if n := str("node"); n != "" {
// behind is not read here: given with a machine, it is refused as passed over — naming
// a machine and asking for every machine behind are two requests, and guessing one
// would push a machine nobody named, or not push one somebody did.
return []string{"push", n, "--wait", "0"}, nil
}
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
// first, so a caller who meant one machine reads that it was not one.
on("behind")
return []string{"push", "--behind", "--wait", "0"}, nil
case "rotate":
if p := str("provision"); p != "" {
@@ -373,17 +183,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return argv, nil
}
_, node := a.given["node"]
_, module := a.given["module"]
_, secret := a.given["secret"]
if node || module || secret {
if err := need("node", "module", "secret"); err != nil {
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
}
if str("node") != "" && str("module") != "" && str("secret") != "" {
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
}
// Neither shape: the command says its usage, which names both, and that is the answer the
// caller needs.
// Half of either shape: the command says its usage, which names both shapes, and that is
// the answer the caller needs.
return []string{"rotate"}, nil
case "settings":
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
@@ -391,16 +195,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
if err := need("module"); err != nil {
return nil, err
}
var argv []string
if on("clear") {
argv := []string{"settings", "set", str("module")}
switch {
case str("clear") == "true":
argv = []string{"settings", "clear", str("module")}
} else {
argv = []string{"settings", "set", str("module")}
// Neither values nor clear: the command says its usage, which names both.
if v := str("values"); v != "" {
argv = append(argv, v)
}
case str("values") != "":
argv = append(argv, str("values"))
}
// Neither values nor clear: the command says its usage, which names both, and that is the
// answer the caller needs — the same as `rotate` given half of either shape.
if n := str("node"); n != "" {
argv = append(argv, "--node", n)
}
@@ -432,8 +235,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
}
return argv, nil
}
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
verb, catalogue.ControllerSeatName)
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
}
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
@@ -485,22 +287,8 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
handlers := map[string]link.ToolHandler{}
for _, v := range seat.Serves {
verb := v.Name
if inProcess[verb] {
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
args := map[string]any{}
if len(bytes.TrimSpace(raw)) > 0 {
if err := json.Unmarshal(raw, &args); err != nil {
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
}
}
// Refused like any verb's: what a verb does not take is not ignored.
a, err := readArguments(verb, args)
if err != nil {
return nil, err
}
if verb == "calls" {
return callsAnswer(link.Calls, a.given["call"])
}
if verb == "tools" {
handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
return seatTools(), nil
}
continue
@@ -539,48 +327,12 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
if err != nil {
return nil, err
}
if answersFirst(argv) {
// Before anything is sent: a push sends the bus's own machine first, and a broker
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
link.Acknowledge(ctx)
}
return runVerb(ctx, argv)
}
}
return handlers, behind, nil
}
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
// the records, `calls` from what this process served.
var inProcess = map[string]bool{"tools": true, "calls": true}
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
// through `command`. A push sends the machine holding the bus first when its user list changed, the
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
func answersFirst(argv []string) bool {
return len(argv) > 0 && argv[0] == "push"
}
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
// one call whole.
func callsAnswer(log *link.CallLog, id string) (any, error) {
if id != "" {
c, ok := log.Get(id)
if !ok {
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
}
return c, nil
}
recent := log.Recent()
for i := range recent {
recent[i].Answer = nil
}
return map[string]any{"calls": recent, "kept": link.KeptCalls,
"note": "newest first; `calls` with a call's id gives its whole answer"}, nil
}
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
func seatTools() map[string]any {
@@ -601,10 +353,9 @@ func seatTools() map[string]any {
}
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
// more, since an argument a verb does not declare is refused.
// verb runnable rather than that it happens to want the arguments the check guessed.
func sampleArguments(v catalogue.Verb) map[string]any {
sample := map[string]any{}
sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
switch required := v.Input["required"].(type) {
case []string:
for _, k := range required {
cmd/mesh-controller/seatverbs_schema_test.go
-365
View File
@@ -1,365 +0,0 @@
package main
import (
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The schemas the controller serves, verb by verb, as the console receives them: from the
// announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244).
func servedSchemas(t *testing.T) map[string]catalogue.Verb {
t.Helper()
handlers, behind, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
if len(behind) != 0 {
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
}
served := map[string]catalogue.Verb{}
for _, e := range seatAnnouncement(handlers).Endpoints {
var input map[string]any
if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil {
t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err)
}
served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input}
}
if len(served) != len(catalogue.ControllerVerbs) {
t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs))
}
return served
}
// subsetsOf is every subset of the names, the empty one included.
func subsetsOf(names []string) [][]string {
var out [][]string
for mask := 0; mask < 1<<len(names); mask++ {
var s []string
for i, n := range names {
if mask&(1<<i) != 0 {
s = append(s, n)
}
}
out = append(out, s)
}
return out
}
func argumentsFor(subset []string, switches map[string]bool) map[string]any {
args := map[string]any{}
for _, n := range subset {
if switches[n] {
args[n] = "true"
} else {
args[n] = "x-" + n
}
}
return args
}
// saidOutright are the switches that say the default aloud, so the line is the same without them —
// on purpose, and only these.
var saidOutright = map[string]string{
"push": "behind", // the whole mesh is what no machine means; behind lets a caller say they meant it
}
// **No argument a caller gives is passed over** (novox/hq issue 244). For every verb served and
// every combination of the arguments its schema declares, the verb either refuses the call, or
// composes a command line that each given argument changed: taking any one away changes the line
// or makes it refused. An argument the line is the same without is one the verb ignored — the
// shape of the push that named a machine and pushed every machine behind.
func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
for name, v := range servedSchemas(t) {
if inProcess[name] {
continue
}
names, switches := declaredArguments(v)
for _, subset := range subsetsOf(names) {
args := argumentsFor(subset, switches)
argv, err := argvFor(name, args)
if err != nil {
if strings.Contains(err.Error(), "does not declare") {
t.Errorf("%s %v: %v", name, args, err)
}
continue
}
for _, dropped := range subset {
fewer := map[string]any{}
for k, val := range args {
if k != dropped {
fewer[k] = val
}
}
without, err := argvFor(name, fewer)
if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped {
t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv)
}
}
}
}
}
// **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with
// what it requires and one argument more; and `node` in particular, for every verb whose schema
// does not take a machine.
func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
for name, v := range servedSchemas(t) {
names, _ := declaredArguments(v)
strangers := []string{"no-such-argument"}
if !contains(names, "node") {
strangers = append(strangers, "node")
}
for _, stranger := range strangers {
args := sampleArguments(v)
args[stranger] = "x"
_, err := argvFor(name, args)
if inProcess[name] {
_, err = readArguments(name, args)
}
if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err)
}
}
}
if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil {
t.Error("a switch took a word that is neither true nor false, and would have read it as false")
}
if _, err := argvFor("node", map[string]any{"node": 7}); err == nil {
t.Error("a number was taken as a machine's name, or as no machine")
}
if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" {
t.Errorf("a switch given as JSON true: %v %v", argv, err)
}
}
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
// naming one beside behind is refused rather than one of the two guessed.
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
argv, err := argvFor("push", map[string]any{"node": "g1"})
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0" {
t.Fatalf("a named push: %v %v", argv, err)
}
for _, args := range []map[string]any{{}, {"behind": "true"}} {
argv, err := argvFor("push", args)
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0" {
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
}
}
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true"}); err == nil ||
!strings.Contains(err.Error(), `"behind"`) {
t.Fatalf("a named push with behind was taken: %v", err)
}
if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) {
t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err)
}
}
// commandFlags is every flag set this package's commands parse, by the name the set is made with,
// and the flags defined on it — read from the source, so a flag added to a command is seen here
// without anyone remembering to.
func commandFlags(t *testing.T) map[string][]string {
t.Helper()
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
fset := token.NewFileSet()
out := map[string][]string{}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
file, err := parser.ParseFile(fset, f, nil, 0)
if err != nil {
t.Fatal(err)
}
for _, decl := range file.Decls {
fn, ok := decl.(*ast.FuncDecl)
if !ok || fn.Body == nil {
continue
}
sets := map[string]string{} // variable → the set's name
ast.Inspect(fn.Body, func(n ast.Node) bool {
if assign, ok := n.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 && len(assign.Rhs) == 1 {
if call, ok := assign.Rhs[0].(*ast.CallExpr); ok && isSelector(call.Fun, "flag", "NewFlagSet") {
if id, ok := assign.Lhs[0].(*ast.Ident); ok {
if name, ok := stringLit(call.Args[0]); ok {
sets[id.Name] = name
out[name] = append(out[name], []string{}...)
}
}
}
}
call, ok := n.(*ast.CallExpr)
if !ok {
return true
}
sel, ok := call.Fun.(*ast.SelectorExpr)
if !ok {
return true
}
recv, ok := sel.X.(*ast.Ident)
if !ok || sets[recv.Name] == "" {
return true
}
arg := 0
switch sel.Sel.Name {
case "Bool", "String", "Int", "Int64", "Uint", "Uint64", "Float64", "Duration", "Func", "BoolFunc", "TextVar":
case "BoolVar", "StringVar", "IntVar", "Int64Var", "UintVar", "Uint64Var", "Float64Var", "DurationVar", "Var":
arg = 1
default:
return true
}
if arg < len(call.Args) {
if flagName, ok := stringLit(call.Args[arg]); ok {
out[sets[recv.Name]] = append(out[sets[recv.Name]], flagName)
}
}
return true
})
}
}
return out
}
func isSelector(e ast.Expr, pkg, name string) bool {
sel, ok := e.(*ast.SelectorExpr)
if !ok {
return false
}
id, ok := sel.X.(*ast.Ident)
return ok && id.Name == pkg && sel.Sel.Name == name
}
func stringLit(e ast.Expr) (string, bool) {
lit, ok := e.(*ast.BasicLit)
if !ok || lit.Kind != token.STRING {
return "", false
}
s, err := strconv.Unquote(lit.Value)
return s, err == nil
}
// accountedFlags are the flags of a verb's command that are not an argument of the same name:
// carried by an argument named otherwise ("=argument"), or set by the verb itself, or withheld from
// the named verb on purpose — each with why. `command` reaches every flag of the binary regardless.
var accountedFlags = map[string]map[string]string{
"status": {"json": "set by the verb: the answer is data"},
"seats": {"json": "set by the verb: the answer is data"},
"queue": {"json": "not set: the verb answers the table a person reads"},
"plan": {"json": "set by the verb unless files is asked"},
"push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"},
"build": {
"wait": "set by the verb to 0: the id follows the build (issue 176)",
"self": "set by the verb from the repository's form: a path on the forge, or a URL",
"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
"behind": "withheld: the named verb builds one named repository; `command` reaches the rest",
"on": "withheld: the named verb builds one named repository; `command` reaches the rest",
},
"builds": {"n": "=limit"},
"plans": {"n": "=limit", "what-if": "=repository"},
}
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
// from the source, so a flag added to a command — or a verb added whose command takes flags —
// fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And
// a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody
// reads.
func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
flags := commandFlags(t)
if len(flags["push"]) == 0 || len(flags["plan"]) == 0 {
t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags)
}
reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
served := servedSchemas(t)
for name, v := range served {
if inProcess[name] || name == "command" {
continue
}
names, switches := declaredArguments(v)
for _, subset := range subsetsOf(names) {
argv, err := argvFor(name, argumentsFor(subset, switches))
if err != nil {
continue
}
// The flag set is named by the longest run of leading words that names one.
var words []string
for _, w := range argv {
if strings.HasPrefix(w, "-") {
break
}
words = append(words, w)
}
for n := len(words); n > 0; n-- {
if _, has := flags[strings.Join(words[:n], " ")]; has {
if reached[name] == nil {
reached[name] = map[string]bool{}
}
reached[name][strings.Join(words[:n], " ")] = true
break
}
}
}
}
used := map[string]map[string]bool{}
verbs := make([]string, 0, len(reached))
for verb := range reached {
verbs = append(verbs, verb)
}
sort.Strings(verbs)
for _, verb := range verbs {
declared, _ := declaredArguments(served[verb])
for set := range reached[verb] {
if used[set] == nil {
used[set] = map[string]bool{}
}
for _, flagName := range flags[set] {
why, accounted := accountedFlags[set][flagName]
switch {
case accounted && strings.HasPrefix(why, "="):
used[set][flagName] = true
if !contains(declared, strings.TrimPrefix(why, "=")) {
t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare",
verb, flagName, set, strings.TrimPrefix(why, "="))
}
case accounted:
used[set][flagName] = true
case contains(declared, flagName):
default:
t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+
"or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName)
}
}
}
}
for set, fs := range accountedFlags {
for flagName := range fs {
if !used[set][flagName] {
t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set)
}
}
}
}
// Every verb's required arguments are properties of its schema: a schema that requires what it
// does not describe is the uncallable verb of issue 244 from the other side.
func TestEveryRequiredArgumentIsDescribed(t *testing.T) {
for name, v := range servedSchemas(t) {
names, _ := declaredArguments(v)
for k := range sampleArguments(v) {
if !contains(names, k) {
t.Errorf("%s requires %q and does not describe it", name, k)
}
}
}
}
cmd/mesh-controller/seatverbs_test.go
+26 -8
View File
@@ -10,6 +10,29 @@ import (
"github.com/novox/mesh-controller/internal/catalogue"
)
// Every verb the mesh-controller seat declares is one this binary can run, with the arguments the
// schema names and no other (novox/hq ADR 0154, ADR 0035).
func TestEveryDeclaredVerbHasACommandLine(t *testing.T) {
for _, v := range catalogue.ControllerVerbs {
if v.Name == "tools" {
continue
}
args := map[string]any{}
props, _ := v.Input["properties"].(map[string]any)
for name := range props {
args[name] = "x"
}
argv, err := argvFor(v.Name, args)
if err != nil {
t.Errorf("%s: %v", v.Name, err)
continue
}
if argv[0] == "" {
t.Errorf("%s: empty command", v.Name)
}
}
}
// `builds` given a build's id reads that build's log from the bus rather than listing builds
// (novox/hq ADR 0157).
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
@@ -47,14 +70,9 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
t.Fatalf("an own secret: %v", argv)
}
if _, err := argvFor("rotate", map[string]any{"node": "ace"}); err == nil || !strings.Contains(err.Error(), `"module"`) {
t.Fatalf("half an own secret is refused, naming what it lacks: %v", err)
}
if argv, _ := argvFor("rotate", nil); strings.Join(argv, " ") != "rotate" {
t.Fatalf("neither shape falls to the command's usage: %v", argv)
}
if _, err := argvFor("rotate", map[string]any{"provision": "p", "node": "ace", "module": "m", "secret": "s"}); err == nil {
t.Fatal("both shapes at once were taken, and one of them passed over")
argv, _ = argvFor("rotate", map[string]any{"node": "ace"})
if strings.Join(argv, " ") != "rotate" {
t.Fatalf("half an own secret falls to the command's usage: %v", argv)
}
}
cmd/mesh-controller/standing.go
-120
View File
@@ -1,120 +0,0 @@
package main
import (
"context"
"fmt"
"strings"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
//
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
// provider now announces a consumer it has failed for minutes; the controller keeps it until the
// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
// standings keeps what providers say, in the inventory.
type standings struct{ inv *inventory.Inventory }
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
Consumer: st.Consumer, ConsumerNode: st.Node,
Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
})
}
// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
//
// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
// consumer clears it.
func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
all, err := inv.FailingProviders(ctx)
if err != nil {
return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
}
assigned := map[string]map[string]bool{}
var out []inventory.ProviderStanding
for _, s := range all {
on, asked := assigned[s.ProviderNode]
if !asked {
modules, err := inv.Assigned(ctx, s.ProviderNode)
if err != nil {
// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
modules = nil
}
on = map[string]bool{}
for _, m := range modules {
on[m] = true
}
assigned[s.ProviderNode] = on
}
if on[s.Module] {
out = append(out, s)
}
}
return out, nil
}
// failingLines is how status says them: one consumer per entry, the error under it, and a provider
// that stopped repeating itself said so.
func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
var out []string
for _, s := range list {
where := s.Module
if s.ProviderNode != "" {
where += " on " + s.ProviderNode
}
whom := s.Consumer
if s.ConsumerNode != "" {
whom += " (" + s.ConsumerNode + ")"
}
out = append(out, fmt.Sprintf(" %-24s fails %s: %s, for %s (%d attempts since %s)",
where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
s.Since.Local().Format("2006-01-02 15:04")))
if e := strings.TrimSpace(s.Error); e != "" {
out = append(out, fmt.Sprintf(" %-24s %s", "", firstLine(e)))
}
if s.Quiet(now) {
out = append(out, fmt.Sprintf(" %-24s not said again for %s — the provider has stopped "+
"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
}
}
return out
}
func orUnclassed(class string) string {
if class == "" {
return "failing"
}
return class
}
// printFailing is the status section, said when there is anything to say.
func printFailing(list []inventory.ProviderStanding, now time.Time) {
if len(list) == 0 {
return
}
fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
for _, line := range failingLines(list, now) {
fmt.Println(line)
}
fmt.Printf("\n the provider's journal has every attempt; it says recovered on its next success\n\n")
}
// failingOn is the standings that concern one machine: a provider running there, or a consumer.
func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
var out []inventory.ProviderStanding
for _, s := range list {
if s.ProviderNode == node || s.ConsumerNode == node {
out = append(out, s)
}
}
return out
}
cmd/mesh-controller/standing_test.go
-115
View File
@@ -1,115 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
t.Fatal(err)
}
kept := standings{open.inventory}
since := time.Now().Add(-23 * time.Hour)
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
if _, err := kept.Stood(ctx, failing); err != nil {
t.Fatal(err)
}
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if asked.well() {
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
}
said := printed(t, func() error { return printStatus(asked) })
for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
if !strings.Contains(said, want) {
t.Fatalf("status does not say %q:\n%s", want, said)
}
}
if strings.Contains(said, "all doing what they were told") {
t.Fatalf("status said all well beside a failing provider:\n%s", said)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var doc struct {
Failing []inventory.ProviderStanding `json:"failing"`
}
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
t.Fatalf("the document does not carry it: %v\n%s", err, body)
}
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
for _, node := range []string{"anchor", "laptop"} {
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
t.Fatalf("node show %s does not name it:\n%s", node, shown)
}
}
// Recovered: gone, and the mesh may be well again as far as this is concerned.
failing.Failing = false
if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
t.Fatalf("%v %v", cleared, err)
}
asked, err = theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if len(asked.failing) != 0 {
t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
}
}
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
// not a problem.
func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
t.Fatal(err)
}
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
if len(asked.failing) != 0 {
t.Fatalf("%+v", asked.failing)
}
}
func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
now := time.Now()
lines := strings.Join(failingLines([]inventory.ProviderStanding{{
Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
}}, now), "\n")
for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
if !strings.Contains(lines, want) {
t.Fatalf("%q not in:\n%s", want, lines)
}
}
if strings.Contains(lines, "more") {
t.Fatalf("more than the first line of an error:\n%s", lines)
}
}
cmd/mesh-controller/status.go
+1 -11
View File
@@ -129,10 +129,6 @@ func printStatus(asked answers) error {
fmt.Println()
}
// A provider failing a consumer, beside machines failing what they were told: both are something
// not working now (novox/hq ADR 0224).
printFailing(asked.failing, time.Now())
if len(quiet) > 0 {
var said []string
for _, n := range quiet {
@@ -420,12 +416,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
}
out.unheld = append(out.unheld, plan.Unheld...)
}
// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
// providers announced: nothing else in the mesh knows whether a provision is being made.
out.failing, err = failingProviders(ctx, inv)
if err != nil {
return answers{}, err
}
out.plans, err = inv.RecentPlans(ctx, 5)
if err != nil {
return answers{}, err
@@ -538,7 +528,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0
len(a.filtered) == 0 && len(a.unheld) == 0
}
// hostSplit is which machines report which host version, for every version more than one machine
cmd/mesh-controller/upgrades.go
+28 -58
View File
@@ -8,7 +8,6 @@ import (
"path"
"regexp"
"strings"
"sync"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
@@ -267,11 +266,6 @@ func notNow(err error) error {
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
// running the module is the upgrade's decision, taken when the catalogue announces it.
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
// what the bus did not hand over (novox/hq issue 266). Each judges against what the other wrote.
actingOnMerges.Lock()
defer actingOnMerges.Unlock()
inv := f.open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
@@ -282,7 +276,34 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
return notNow(err)
}
from, packaging, already := mergeCandidates(m, entries, read)
// Two kinds of module are affected by one merge, and they are affected differently.
//
// A module **built from** this repository and branch has moved: the mesh records the new commit
// as what its source now has, and only what the merge actually changed is rebuilt. A module that
// only **packages source from** it has not moved — its own source is somewhere else, at the
// commit it already records — so it is rebuilt and its record left alone. Writing this commit as
// its source would make it permanently behind a repository its manifest does not come from.
var from, packaging []inventory.Entry
already := 0
for _, e := range entries {
switch {
case sourceIs(e.Source, m):
if e.Source.BuiltFrom == m.Commit {
already++
continue
}
// **A merge older than the last look at the source is history, not a move.** The forge
// announces what it finds merged, and an old merge surfacing late would otherwise move
// the recorded head backwards and rebuild everything built from that repository, once
// per old merge (2026-09-28).
if isHistory(m.MergedAt, e.Source.Seen) {
continue
}
from = append(from, e)
case readsFrom(read[e.Manifest.Module], m):
packaging = append(packaging, e)
}
}
if len(from) == 0 && len(packaging) == 0 {
// "Already built from it" and "nothing reads it" are different facts, and reading the first
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
@@ -417,57 +438,6 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
return nil
}
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
var actingOnMerges sync.Mutex
// mergeCandidates is what one merge could move, judged against what the catalogue holds.
//
// Two kinds of module are affected by one merge, and they are affected differently.
//
// A module **built from** this repository and branch has moved: the mesh records the new commit as
// what its source now has, and only what the merge actually changed is rebuilt. A module that only
// **packages source from** it has not moved — its own source is somewhere else, at the commit it
// already records — so it is rebuilt and its record left alone. Writing this commit as its source
// would make it permanently behind a repository its manifest does not come from. `already` counts
// the modules built from this repository that are already built from this very commit.
func mergeCandidates(m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository) (from, packaging []inventory.Entry, already int) {
for _, e := range entries {
switch {
case sourceIs(e.Source, m):
if e.Source.BuiltFrom == m.Commit {
already++
continue
}
// **A merge older than the last look at the source is history, not a move.** The forge
// announces what it finds merged, and an old merge surfacing late would otherwise move
// the recorded head backwards and rebuild everything built from that repository, once
// per old merge (2026-09-28).
if isHistory(m.MergedAt, e.Source.Seen) {
continue
}
from = append(from, e)
case readsFrom(read[e.Manifest.Module], m):
packaging = append(packaging, e)
}
}
return from, packaging, already
}
// wouldMove is the modules built from the merged repository that acting on this merge would mark as
// moved and rebuild — SourceMoved's judgement, made without acting (novox/hq issue 266). Empty for a
// merge already acted on: acting marks each of them as looked at, so the merge then reads as history.
//
// **Only the modules built from it, never the ones that merely package source from it.** Acting
// records nothing about those, so a merge acted on would go on reading as unacted for them, and be
// acted on again on every look. A merge that moves both is caught by the first kind, and acting on it
// rebuilds the second as well.
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository) []inventory.Entry {
from, _, _ := mergeCandidates(m, entries, read)
return whatTheMergeTouched(from, entries, m)
}
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
// An empty recorded ref is the repository's default branch, which is what a merge into the base
internal/broker/nats.go
+3 -12
View File
@@ -18,7 +18,6 @@ import (
"regexp"
"sort"
"strings"
"time"
)
// A Kind is what a principal is, which decides the shape of its authority rather than its
@@ -190,13 +189,6 @@ type Permissions struct {
AllowResponses bool
}
// ResponseTTL is how long the bus lets a principal answer a request it received. Its one answer has
// to come inside this, and a seat's holder answers within link.AnswerWithin — inside it by design.
// **A broker reloading its user list forgets every answer it was about to permit**, whatever this
// says (novox/hq issue 265): a call that is still running when the list reloads has its answer
// refused, which is why a holder answers before it does what can reload it.
const ResponseTTL = time.Minute
// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
// a permission that cannot be derived from a declaration is a permission nobody can explain.
func PermissionsFor(p Principal) (Permissions, error) {
@@ -261,11 +253,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
sub = append(sub, announcing(ControllerSeat)...)
// The events it reacts to, and its ack subject on the stream they arrive from
// The two events it reacts to, and its ack subject on the stream they arrive from
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
// controller a subscriber to every event in the mesh, and its permission list would stop
// saying what it is for. The one wildcard is the emitter of a provider's standing (ADR
// 0224) — still two named events, from whichever module provides. The ack grant below is scoped per stream because the controller's
// saying what it is for. The ack grant below is scoped per stream because the controller's
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
// ever, refused by the list it already has.
@@ -786,7 +777,7 @@ func ComposeAccounts(principals []Principal) (string, error) {
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish))
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
if perms.AllowResponses {
fmt.Fprintf(&b, " allow_responses: { max: 1, ttl: \"%dm\" }\n", int(ResponseTTL/time.Minute))
b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n")
}
b.WriteString(" } }\n")
}
internal/broker/seattools_test.go
+8 -8
View File
@@ -5,16 +5,16 @@ import "testing"
// A node-scoped seat's tool carries the node (novox/hq ADR 0132, design 33 §4): two nodes holding one
// node-scoped seat derive two addresses, and a user of the seat may publish any node's.
func TestTwoNodesHoldingOneNodeSeatDeriveTwoToolAddresses(t *testing.T) {
seat := Seat{Name: "node-hostname", Scope: "node", Serves: []string{"entries"}}
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
has(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.one")
has(t, two.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries")
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
seat := Seat{Name: "node-hosts-file", Scope: "node", Serves: []string{"entries"}}
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hosts", Holds: []Seat{seat}, PasswordHash: "x"})
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hosts", Holds: []Seat{seat}, PasswordHash: "x"})
has(t, one.Subscribe, "mesh.seat.node-hosts-file.tool.entries.one")
has(t, two.Subscribe, "mesh.seat.node-hosts-file.tool.entries.two")
hasNot(t, one.Subscribe, "mesh.seat.node-hosts-file.tool.entries")
hasNot(t, one.Subscribe, "mesh.seat.node-hosts-file.tool.entries.two")
user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "three", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
has(t, user.Publish, "mesh.seat.node-hostname.tool.entries.*")
has(t, user.Publish, "mesh.seat.node-hosts-file.tool.entries.*")
}
// A mesh-scoped seat's tool stays flat: nothing about it changes.
internal/broker/streams.go
+1 -16
View File
@@ -226,23 +226,8 @@ var ControllerFollows = []string{
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
// with the retired seat row.
seatEventSubject("mesh-build-machine", "built"),
// **Every provider's standing** (novox/hq ADR 0224): a consumer it has failed for minutes, and
// that consumer recovered. The one pattern on this list, and a narrow one — two named events,
// from whichever module provides — because the rule is about every provider, and a list of
// providers here would be a list somebody forgets to extend. On 2026-10-05 the identity provider
// failed every consumer for a day and only its journal said so (issue 179). Appended, because
// the index is a name.
moduleEventSubject("*", ProvisionerFailing),
moduleEventSubject("*", ProvisionerRecovered),
}
// The provider standing events, by their local names. Written here as well as in the catalogue
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
const (
ProvisionerFailing = "provisioner.failing"
ProvisionerRecovered = "provisioner.recovered"
)
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
func moduleEventSubject(module, event string) string {
@@ -286,7 +271,7 @@ func MeshConsumers() []Consumer {
// client and come back to be acted on again.
MaxAckPending: 1,
FromNow: true,
Why: "the events the mesh's own controller reacts to, one at a time; after " +
Why: "the two events the mesh's own controller reacts to, one at a time; after " +
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
"become actionable",
},
internal/broker/testdata/composed.conf
Vendored
+1 -1
View File
@@ -25,7 +25,7 @@ accounts {
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
internal/catalogue/declaration.go
+1 -6
View File
@@ -159,11 +159,6 @@ type Rendering struct {
// 0199): the mesh's resolver forwards each one there.
Zones []ZoneAt
// Holders is, for each replicated mesh seat, every machine holding it, by internal name and
// private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver
// file lists: every holder of the mesh's resolver, this machine first if it is one.
Holders map[string]map[string]string
Settings SettingsBy
Generators map[string]Generator
// Grants are the credentials this node must create, for the provisions it offers. Passed in
@@ -985,7 +980,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
// this module's name, so they are applied, reported and removed exactly as anything else
// it declares.
given, err := FactsFrom(m, r, with)
given, err := FactsWithZonesInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix, with.Zones)
if err != nil {
return nil, err
}
internal/catalogue/events.go
-32
View File
@@ -3,7 +3,6 @@ package catalogue
import (
"fmt"
"regexp"
"slices"
"strings"
)
@@ -160,34 +159,3 @@ func consumePattern(pattern string) error {
}
return nil
}
// The events a provider says about its consumers (novox/hq ADR 0224): a consumer it has failed
// without one success for minutes, and that consumer succeeding again or being withdrawn. The
// controller follows them from every module and `status` names a consumer failing until it recovers.
const (
ProvisionerFailing = "provisioner.failing"
ProvisionerRecovered = "provisioner.recovered"
)
// ProvisionerEvents are both, in the order they are said.
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered}
// EmitsAll is every event a module may publish: what it declares and, for a module that receives
// contributions — a provider, running a provisioner over them — the provider's standing events.
//
// **Derived, not declared**, because they are the mesh's rule about every provider rather than
// anything one module chose to say: a provider whose manifest forgot them would fail its consumers
// as silently as on 2026-10-05, with its announcement refused by the bus (novox/hq issue 179). Every
// grant of a module's publishing reads this, never the declared list alone.
func (m Manifest) EmitsAll() []string {
out := append([]string(nil), m.Emits...)
if len(m.Receives) == 0 {
return out
}
for _, e := range ProvisionerEvents {
if !slices.Contains(out, e) {
out = append(out, e)
}
}
return out
}
internal/catalogue/hostname_seat_test.go
-98
View File
@@ -1,98 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// novox/hq ADR 0223 part 3: a machine's names are one seat's. The `hosts` module holding
// `node-hosts-file` became `hostname` holding `node-hostname`, which writes /etc/hostname beside the
// machine's own lines in /etc/hosts. The seat was renamed (ADR 0122), so what claims the old name — a
// manifest registered before the rename — still holds the one seat.
func withHostnameAlias(t *testing.T) {
t.Helper()
was := aliases
t.Cleanup(func() { aliases = was })
UseAliases(map[string]string{"node-hosts-file": "node-hostname"})
}
func TestTheHostsFilesFormerNameResolvesToTheHostnameSeat(t *testing.T) {
if _, known := SeatNamed("node-hostname"); !known {
t.Fatal("node-hostname is not in the mesh's set")
}
withHostnameAlias(t)
seat, known := SeatNamed("node-hosts-file")
if !known || seat.Name != "node-hostname" || seat.Scope != ScopeNode {
t.Fatalf("the former name did not resolve: %+v %v", seat, known)
}
var verbs []string
for _, v := range seat.Serves {
verbs = append(verbs, v.Name)
}
if strings.Join(verbs, " ") != "entries add remove" {
t.Errorf("the renamed seat serves %v; its verbs are unchanged", verbs)
}
}
// One seat under either name: the module registered before the rename and the one after cannot both
// hold it on one machine.
func TestTheOldAndTheNewClaimantAreOneSeatOnAMachine(t *testing.T) {
withHostnameAlias(t)
cat := shelf(
mod("hosts", nil, nil, nil, Claim{Name: "node-hosts-file"}),
mod("hostname", nil, nil, nil, Claim{Name: "node-hostname"}),
)
_, err := Resolve(cat, []string{"hosts", "hostname"}, workstation(), World{})
if err == nil || !strings.Contains(err.Error(), "node-hostname") {
t.Errorf("hosts and hostname both held the machine's names on one machine: %v", err)
}
if _, err := Resolve(cat, []string{"hosts"}, workstation(), World{}); err != nil {
t.Errorf("a machine still assigned hosts under the old name does not resolve: %v", err)
}
}
// The catalogue's module: /etc/hostname is the operator's `hostname` setting. Without it the module is
// left out, naming the key — the mesh never renames a machine on its own — and a mesh-wide setting
// naming ${machine:name} gives every machine its mesh name.
func TestTheMachinesNameIsItsSetting(t *testing.T) {
cat := map[string]Manifest{"hostname": catalogueManifest(t, "hostname")}
got, err := Resolve(cat, []string{"hostname"}, Node{Name: "ace", At: "ace.internal"}, World{})
if err != nil {
t.Fatal(err)
}
machines := map[string]string{"ace.internal": "10.42.0.2"}
nameOf := func(settings SettingsBy) (string, string) {
t.Helper()
composed, err := got.Compose(Rendering{Names: machines, Machines: machines, Suffix: "internal",
Settings: settings})
if err != nil {
t.Fatal(err)
}
if why := composed.LeftOut["hostname"]; why != "" {
return "", why
}
for _, r := range composed.Resources {
if r["path"] == "/etc/hostname" {
return r["content"].(string), ""
}
}
t.Fatal("no /etc/hostname composed")
return "", ""
}
if _, why := nameOf(nil); !strings.Contains(why, "hostname") {
t.Errorf("with no setting the machine's name was written, or left out for another reason: %q", why)
}
if name, why := nameOf(SettingsBy{"hostname": {{From: "ace", Values: map[string]any{"hostname": "Ace"}}}}); name != "Ace\n" {
t.Errorf("the operator's name for the machine gave %q (%s)", name, why)
}
mesh := Layer{From: "the mesh", Values: map[string]any{"hostname": "${machine:name}"}}
if name, why := nameOf(SettingsBy{"hostname": {mesh}}); name != "ace\n" {
t.Errorf("a mesh-wide ${machine:name} gave %q (%s)", name, why)
}
node := Layer{From: "ace", Values: map[string]any{"hostname": "Ace"}}
if name, why := nameOf(SettingsBy{"hostname": {mesh, node}}); name != "Ace\n" {
t.Errorf("a machine's own name over the mesh-wide one gave %q (%s)", name, why)
}
}
internal/catalogue/resolve.go
+10 -79
View File
@@ -767,27 +767,16 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
var problems []string
var held []Held
// onRecord is every recorded holder of a seat, if a handover ever named one: one for most seats,
// and as many as were added for a replicated one (novox/hq ADR 0223).
onRecord := func(claim, scope string) []Held {
var out []Held
// onRecord is the recorded holder of a seat, if a handover ever named one.
onRecord := func(claim, scope string) (Held, bool) {
for _, h := range holdings {
hs, ok := SeatNamed(h.Claim)
cs, cok := SeatNamed(claim)
if ok && cok && hs.Name == cs.Name && h.Scope == scope {
out = append(out, h)
return h, true
}
}
return out
}
// recordedHere says this node's module is one of a seat's holders on record.
recordedHere := func(claim, scope, module string) bool {
for _, rec := range onRecord(claim, scope) {
if rec.Node == node.Name && rec.Module == module {
return true
}
}
return false
return Held{}, false
}
byScope := map[string]map[string]string{} // scope → claim → module
@@ -798,35 +787,21 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
// the seat but is not the one on record is eligible, and that is all: it is not a second
// holder, so it is not refused, and it does not hold (novox/hq ADR 0131). This is what
// lets the next holder stand beside the current one until the seat is handed over.
if recs := onRecord(c.Name, scope); len(recs) > 0 {
// **A seat held once is on record once** (novox/hq ADR 0223). Only a replicated seat
// may have several holders on record; several for any other seat is a store that
// disagrees with the mesh's definition of the role, and it is refused, named, rather
// than letting two machines answer for what the mesh has one of.
if s, known := SeatNamed(c.Name); known && !s.Replicated && len(recs) > 1 {
problems = append(problems, fmt.Sprintf(
"%q is on record as held by %d assignments, and it is held once per %s — "+
"`seat %s --to <node>/<module>` records one", c.Name, len(recs), scope, c.Name))
continue
}
if !recordedHere(c.Name, scope, m.Module) {
if rec, recorded := onRecord(c.Name, scope); recorded {
if rec.Node != node.Name || rec.Module != m.Module {
continue
}
}
if byScope[scope] == nil {
byScope[scope] = map[string]string{}
}
// **One seat under either of its names** (novox/hq ADR 0122): a manifest registered before
// a rename claims the former name, and one written after it the current — two claimants of
// one seat, compared by the seat they resolve to and not by how each spelled it.
seat := canonicalSeat(c.Name)
if other, taken := byScope[scope][seat]; taken {
if other, taken := byScope[scope][c.Name]; taken {
problems = append(problems, fmt.Sprintf(
"%s and %s both claim %q, and only one thing may hold it per %s",
other, m.Module, seat, scope))
other, m.Module, c.Name, scope))
continue
}
byScope[scope][seat] = m.Module
byScope[scope][c.Name] = m.Module
held = append(held, Held{Claim: c.Name, Scope: scope, Node: node.Name,
Module: m.Module, Site: node.Site})
}
@@ -835,17 +810,11 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
// And against the rest of the mesh, for the scopes that reach past this machine.
for _, h := range held {
for _, e := range elsewhere {
if e.Node == node.Name || canonicalSeat(e.Claim) != canonicalSeat(h.Claim) || e.Scope != h.Scope {
if e.Node == node.Name || e.Claim != h.Claim || e.Scope != h.Scope {
continue
}
switch h.Scope {
case ScopeMesh:
// **A holder on record is never a second claimant** (novox/hq ADR 0223). Records are
// the mesh's settled answer: one for most seats, several only for a replicated seat,
// each added by an act. Two holders here are two records, and both hold.
if recordedHere(h.Claim, h.Scope, h.Module) {
continue
}
// Both claim and nobody is on record, or this refusal could not have happened.
// The remedy is the handover that records the holder (novox/hq ADR 0131,
// 04-ISSUES/170), so it is named here rather than left to be found.
@@ -866,15 +835,6 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
return held, problems
}
// canonicalSeat is the seat a claimed name refers to, by its current name: itself for a name the mesh
// does not know (a module's own seat), its seat's name for a former one.
func canonicalSeat(name string) string {
if s, known := SeatNamed(name); known {
return s.Name
}
return name
}
// checkResources refuses two modules writing the same thing.
//
// This costs no manifest field: the mesh already holds every resource of every module, so two
@@ -952,23 +912,6 @@ func checkResources(modules []Manifest) []string {
}
}
}
// **A file the mesh renders for a module is that module's path too** (novox/hq ADR 0223). The
// machine's resolver file is a fact the uplink's holder asks for, and a second module asking
// for it, or declaring it, would have the host write one path twice in every apply, the last
// one winning. A fact under the operator's home is placed per account and compared nowhere.
for _, name := range sortedFacts(m.Facts) {
fact := m.Facts[name]
if fact.Home || !strings.HasPrefix(fact.Path, "/") {
continue
}
key := "path " + fact.Path
if other, taken := owner[key]; taken && other != m.Module {
problems = append(problems, fmt.Sprintf(
"%s and %s both declare the path %q", other, m.Module, fact.Path))
}
owner[key] = m.Module
ownedPath[fact.Path] = m.Module
}
}
// An accessed path is the operator's, so no module may declare it as one of its own
@@ -1213,18 +1156,6 @@ func answeredElsewhere(want string, node Node, world World, brokered map[string]
if c, pinned := world.Pinned[want]; pinned {
return c.Node != node.Name
}
// **A machine holding the seat answers itself** (novox/hq ADR 0223). With a replicated seat
// another machine holds it too, and the first holder in the providers' order may be that one; a
// holder is still where this machine's own requirement is answered, so its resolver file lists
// itself first.
if seat, delivered := SeatDelivering(want); delivered {
for _, h := range append(append([]Held(nil), world.Holdings...), world.Held...) {
if hs, ok := SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope &&
h.Node == node.Name {
return false
}
}
}
holder, held := HolderAmong(want, world.Offered[want], world.Held)
return held && holder.Node != node.Name
}
internal/catalogue/resolver_file_is_the_uplinks_test.go
-57
View File
@@ -1,57 +0,0 @@
package catalogue
import (
"reflect"
"testing"
)
// novox/hq ADR 0223 part 2: /etc/resolv.conf belongs to the module holding node-uplink. The seat that
// wrote it, node-resolver-config, and ADR 0220's dependency of it on the uplink retire: one owner for
// the file, and it is the program that would otherwise rewrite it.
func TestTheResolverConfigSeatIsGone(t *testing.T) {
if _, known := SeatNamed("node-resolver-config"); known {
t.Error("node-resolver-config is still in the mesh's set; the uplink's holder writes the resolver file")
}
// An uplink's holder needs no seat beside it for the file: it is its own.
if got := DependsOn(mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"})); len(got) != 0 {
t.Errorf("an uplink holder with nothing declared depends on %v", got)
}
}
// The catalogue as it is: nothing claims the retired seat, and every manager the mesh knows holds the
// uplink and writes the resolver file.
func TestTheCataloguesUplinksWriteTheResolverFileAndNothingElseDoes(t *testing.T) {
cat := map[string]Manifest{}
for _, m := range theCatalogue(t) {
cat[m.Module] = m
}
if got := PossibleHolders(cat, "node-uplink"); !reflect.DeepEqual(got, uplinks) {
t.Errorf("node-uplink can be held by %v, not %v", got, uplinks)
}
for name, m := range cat {
for _, c := range m.Claims {
if c.Name == "node-resolver-config" {
t.Errorf("%s still claims node-resolver-config", name)
}
}
_, writes := m.Facts["resolvers"]
isUplink := false
for _, u := range uplinks {
isUplink = isUplink || u == name
}
for _, f := range m.Facts {
if f.Path == "/etc/resolv.conf" && !isUplink {
t.Errorf("%s writes /etc/resolv.conf and does not hold the uplink", name)
}
}
for _, r := range m.Resources {
if r["path"] == "/etc/resolv.conf" {
t.Errorf("%s declares /etc/resolv.conf as a file of its own", name)
}
}
if isUplink && !writes {
t.Errorf("%s holds the uplink and does not write the resolver file", name)
}
}
}
internal/catalogue/resolver_manifests_test.go
+40 -54
View File
@@ -15,8 +15,8 @@ import (
// same arrangement, and to the two things a resolver here must never do — read resolv.conf for
// its upstreams, or take an address systemd-resolved holds.
// resolverShelf is the resolver, an uplink module that writes what the machine asks (novox/hq ADR
// 0223), and the container runtime beside something that answers `mesh-addressing`.
// resolverShelf is the two resolver modules and the container runtime beside something that
// answers `mesh-addressing`.
// The networking module that really does is composed in the controller and cannot be imported
// here, so a stand-in offers the same word; what is under test is the manifests, not the network.
func resolverShelf(t *testing.T) map[string]Manifest {
@@ -24,7 +24,7 @@ func resolverShelf(t *testing.T) map[string]Manifest {
shelf := map[string]Manifest{
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
}
for _, name := range []string{"dnsmasq", "systemd-networkd", "docker"} {
for _, name := range []string{"dnsmasq", "resolv-conf", "docker"} {
shelf[name] = catalogueManifest(t, name)
}
return shelf
@@ -84,27 +84,26 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
t.Errorf("the mesh's resolver still reads or listens on %q", never)
}
}
// And the file that decides what the machine asks names every one of the mesh's resolvers, by
// address, from the seat's holders, and no public one (ADR 0223): a resolver library that asks
// every listed server at once takes the first reply, and a public "no such name" for a mesh name
// won it. Written by every uplink module, since the uplink's holder owns the file.
for _, uplink := range uplinks {
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
if !ok || fact.Path != "/etc/resolv.conf" {
t.Fatalf("%s's resolver file is not rendered from the roster: %+v", uplink, fact)
// And the file that decides what the machine asks names the mesh's resolver first, by address,
// and a public one second, asked only when the first is silent (ADR 0196).
var resolv string
for _, r := range catalogueManifest(t, "resolv-conf").Resources {
if r["path"] == "/etc/resolv.conf" {
resolv, _ = r["content"].(string)
}
if !strings.Contains(fact.Template, `{{range index .Holders "mesh-dns-resolver"}}nameserver {{.Address}}`) {
t.Errorf("%s's resolv.conf does not list every holder of the mesh's resolver:\n%s", uplink, fact.Template)
}
for _, line := range strings.Split(fact.Template, "\n") {
if strings.HasPrefix(line, "nameserver ") && !strings.Contains(line, "{{") {
t.Errorf("%s's resolv.conf names a resolver of its own beside the mesh's: %s", uplink, line)
}
}
if !strings.Contains(fact.Template, "options timeout:1 attempts:2 edns0\n") {
t.Errorf("%s: a silent resolver is not passed over after one short wait:\n%s", uplink, fact.Template)
}
var nameservers []string
for _, line := range strings.Split(resolv, "\n") {
if strings.HasPrefix(line, "nameserver ") {
nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
}
}
if len(nameservers) != 2 || nameservers[0] != "${bound:wildcard-resolution:address}" || nameservers[1] != "1.1.1.1" {
t.Errorf("resolv.conf names %v; the mesh's resolver by address first, a public one second", nameservers)
}
if !strings.Contains(resolv, "\noptions timeout:1 attempts:1") {
t.Errorf("the fallback is not reached after one short attempt:\n%s", resolv)
}
}
// The resolver and what points the machine at it compose on one machine, and what arrives is the
@@ -112,10 +111,9 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
// that file, the machine pointed at the resolver by address, and the runtime given no resolver of
// its own but kept running across a restart (ADR 0196).
func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "systemd-networkd", "docker"},
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf", "docker"},
Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
"package-manager": true, "service-manager": true, "privileged": true,
"uplink-systemd-networkd": true}}, World{})
"package-manager": true, "service-manager": true, "privileged": true}}, World{})
if err != nil {
t.Fatal(err)
}
@@ -128,7 +126,6 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
// happen to be the same map, since nothing routed is part of it.
Names: twoMachines, Machines: twoMachines, Suffix: "internal",
Zones: []ZoneAt{{Zone: "incus", Address: "10.42.0.2", Port: 5353}},
Holders: map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
})
@@ -173,7 +170,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
t.Errorf("the resolver still writes the runtime's dns: %v", ids["dnsmasq.runtime-dns"])
}
for id := range ids {
if strings.HasPrefix(id, "systemd-networkd.runtime") {
if strings.HasPrefix(id, "resolv-conf.runtime") {
t.Errorf("what the machine asks still writes the runtime's file: %s", id)
}
}
@@ -208,40 +205,29 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
}
resolv := ids["systemd-networkd.fact-resolvers"]
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\noptions ") {
t.Fatalf("the machine is not pointed at the resolver by address, and only at it: %v", resolv)
resolv := ids["resolv-conf.resolv"]
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\nnameserver 1.1.1.1\n") {
t.Fatalf("the machine is not pointed at the resolver by address, with the public fallback: %v", resolv)
}
}
// Two modules deciding what a machine asks are refused on one machine, as before — now that the file
// is the uplink's (novox/hq ADR 0223), by two things: a machine runs one network manager, and no other
// module may write the resolver file beside the uplink's, as a file or as a rendered fact.
// Two modules deciding what a machine asks are refused on one machine, as before — the claim
// exists so they never take turns overwriting each other.
//
// **The second is made up.** The catalogue's only other claimant, a systemd-resolved split-DNS
// module, was retired with the choice against a stub on every node (novox/hq ADR 0196, ADR 0220);
// what is under test is the claim, so any second module claiming it will do.
func TestTwoThingsDecidingWhatAMachineAsksAreRefused(t *testing.T) {
shelf := resolverShelf(t)
shelf["networkmanager"] = catalogueManifest(t, "networkmanager")
node := Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
"package-manager": true, "service-manager": true,
"uplink-systemd-networkd": true, "uplink-networkmanager": true}}
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "networkmanager"}, node, World{})
if err == nil || !strings.Contains(err.Error(), "node-uplink") {
t.Fatalf("two network managers were both assigned to one machine: %v", err)
shelf["other-resolver-config"] = Manifest{Module: "other-resolver-config", Version: "1",
Claims: []Claim{{Name: "node-resolver-config", Scope: ScopeNode}}}
_, err := Resolve(shelf, []string{"dnsmasq", "resolv-conf", "other-resolver-config"},
Node{Name: "anchor", At: "anchor.internal"}, World{})
if err == nil {
t.Fatal("resolv-conf and a second module deciding what the machine asks were both assigned to one machine")
}
// The second is made up: what is under test is that the resolver file has one owner, so any
// module asking the mesh to render it — or declaring it — will do.
for name, m := range map[string]Manifest{
"a-rendered-one": {Module: "a-rendered-one", Version: "1",
Facts: map[string]RosterFile{"mine": {Path: "/etc/resolv.conf", Template: "nameserver 10.42.0.1\n"}}},
"a-declared-one": {Module: "a-declared-one", Version: "1", Resources: []map[string]any{
{"id": "mine", "type": "file", "path": "/etc/resolv.conf", "content": "nameserver 10.42.0.1\n"}}},
} {
shelf := resolverShelf(t)
shelf[name] = m
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", name}, node, World{})
if err == nil || !strings.Contains(err.Error(), "/etc/resolv.conf") {
t.Errorf("%s wrote the resolver file beside the uplink's: %v", name, err)
}
if !strings.Contains(err.Error(), "node-resolver-config") {
t.Fatalf("the refusal does not say what was claimed: %v", err)
}
}
internal/catalogue/resolver_needs_uplink_test.go
+121
View File
@@ -0,0 +1,121 @@
package catalogue
import (
"errors"
"reflect"
"strings"
"testing"
)
// Defends novox/hq ADR 0220: what a machine asks for names needs the uplink held beside it.
//
// resolv.conf is the mesh's only while the program managing the machine's network is told to leave
// it alone, and the uplink's holder is what tells it (ADR 0117). Without one, the first connectivity
// change rewrites the file — so the dependency is checked at assignment, by the same mechanism as a
// service's on the service manager (ADR 0207), derived from the claim and never stated in a manifest.
// resolverAndUplinks is a resolver-config holder and two uplink holders, with no resources of their
// own so that nothing but the seats is judged.
func resolverAndUplinks() map[string]Manifest {
return shelf(
mod("resolv-conf", nil, nil, nil, Claim{Name: "node-resolver-config"}),
mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"}),
mod("systemd-networkd", nil, nil, nil, Claim{Name: "node-uplink"}),
)
}
func TestTheResolverConfigSeatNeedsTheUplink(t *testing.T) {
s, known := SeatNamed("node-resolver-config")
if !known {
t.Fatal("node-resolver-config is not in the mesh's set")
}
if !reflect.DeepEqual(s.Needs, []string{"node-uplink"}) {
t.Errorf("node-resolver-config needs %v, want [node-uplink] (ADR 0220)", s.Needs)
}
// Derived from the claim: a module claiming the seat depends on the uplink with nothing written.
got := DependsOn(mod("anything", nil, nil, nil, Claim{Name: "node-resolver-config"}))
if !reflect.DeepEqual(got, []string{"node-uplink"}) {
t.Errorf("a module claiming node-resolver-config depends on %v, want [node-uplink]", got)
}
// And the uplink's holders need nothing of the kind: the dependency runs one way.
if got := DependsOn(mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"})); len(got) != 0 {
t.Errorf("an uplink holder depends on %v; it needs no seat beside it", got)
}
}
// What the store loads has no column for it, so the compiled value survives a load.
func TestTheNeedSurvivesTheStoresRows(t *testing.T) {
defer UseSeats(DefaultSeats())
var rows []Seat
for _, s := range DefaultSeats() {
rows = append(rows, Seat{Name: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision})
}
UseSeats(rows)
if s, _ := SeatNamed("node-resolver-config"); !reflect.DeepEqual(s.Needs, []string{"node-uplink"}) {
t.Errorf("after loading the store's rows node-resolver-config needs %v", s.Needs)
}
}
func TestAssigningTheResolverConfigWithoutAnUplinkIsRefused(t *testing.T) {
cat := resolverAndUplinks()
_, err := AssignRefusal(cat, "laptop", nil, []string{"resolv-conf"})
var refusal *Refusal
if !errors.As(err, &refusal) {
t.Fatalf("resolv-conf was assigned to a machine nothing manages the network of: %v", err)
}
for _, want := range []string{"resolv-conf on laptop depends on node-uplink", "networkmanager", "systemd-networkd"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not say %q:\n%s", want, err)
}
}
// Beside a holder, or together with one in one act, it is let through.
if _, err := AssignRefusal(cat, "laptop", []string{"networkmanager"}, []string{"resolv-conf"}); err != nil {
t.Errorf("resolv-conf beside networkmanager was refused: %v", err)
}
if _, err := AssignRefusal(cat, "anchor", nil, []string{"systemd-networkd", "resolv-conf"}); err != nil {
t.Errorf("resolv-conf assigned with systemd-networkd was refused: %v", err)
}
// And a composition without one is refused once the switch is on.
if _, err := Resolve(cat, []string{"resolv-conf"}, workstation(), World{}); err == nil ||
!strings.Contains(err.Error(), "node-uplink") {
t.Errorf("a node with resolv-conf and no uplink composed: %v", err)
}
}
func TestUnassigningTheUplinkUnderTheResolverConfigIsRefused(t *testing.T) {
cat := resolverAndUplinks()
err := UnassignRefusal(cat, "laptop", []string{"networkmanager", "resolv-conf"}, []string{"networkmanager"})
if err == nil || !strings.Contains(err.Error(), "resolv-conf") || !strings.Contains(err.Error(), "node-uplink") {
t.Errorf("taking the uplink from under resolv-conf gave %v", err)
}
}
// The catalogue as it is: the module that writes resolv.conf depends on the uplink, and every manager
// the mesh knows can meet it — so the refusal always has a remedy to name.
func TestTheCataloguesResolverConfigHasUplinkHoldersToName(t *testing.T) {
cat := map[string]Manifest{}
for _, m := range theCatalogue(t) {
cat[m.Module] = m
}
deps := DependsOn(cat["resolv-conf"])
found := false
for _, d := range deps {
found = found || d == "node-uplink"
}
if !found {
t.Errorf("the catalogue's resolv-conf depends on %v, not on node-uplink", deps)
}
holders := PossibleHolders(cat, "node-uplink")
for _, want := range []string{"dhcpcd", "networkmanager", "systemd-networkd"} {
in := false
for _, h := range holders {
in = in || h == want
}
if !in {
t.Errorf("%s does not hold node-uplink in the catalogue; holders: %v", want, holders)
}
}
if got := PossibleHolders(cat, "node-resolver-config"); !reflect.DeepEqual(got, []string{"resolv-conf"}) {
t.Errorf("node-resolver-config can be held by %v; resolv-conf alone since ADR 0220", got)
}
}
internal/catalogue/roster.go
+9 -44
View File
@@ -64,12 +64,6 @@ type rosterView struct {
// Zones is every zone a module in the mesh answers itself, with where its answerer is (novox/hq
// ADR 0199) — what the mesh's resolver forwards. Ordered by zone.
Zones []rosterZone
// Holders is the machines holding each replicated mesh seat, by seat (novox/hq ADR 0223) — what
// a machine's resolver file lists for `mesh-dns-resolver`. **This machine first when it is one
// of them**, then the rest by name: the nearest holder is asked first, and two renderings of
// one mesh on one machine are one file. A template reads one seat's with
// `index .Holders "<seat>"`; a seat nobody holds ranges over nothing.
Holders map[string][]rosterEntry
}
// rosterZone is one zone as a template sees it: the zone, and the address and port answering it.
@@ -102,25 +96,21 @@ func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]st
// FactsWithZonesInto is FactsInto with the mesh's zones in the view, for a template that ranges them.
func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string,
zones []ZoneAt) ([]map[string]any, error) {
return FactsFrom(m, r, Rendering{Names: every, Machines: machines, Accounts: accounts, Suffix: suffix,
Zones: zones})
}
// FactsFrom renders the roster files a module asked for from everything the mesh composed for this
// machine: its machines, zones and the holders of each replicated seat.
func FactsFrom(m Manifest, r Resolution, with Rendering) ([]map[string]any, error) {
if len(m.Facts) == 0 {
return nil, nil
}
names := sortedFacts(m.Facts)
names := make([]string, 0, len(m.Facts))
for name := range m.Facts {
names = append(names, name)
}
sort.Strings(names)
view := rosterView{
Node: r.Node,
Suffix: strings.TrimPrefix(suffixOr(with.Suffix), "."),
Names: entriesFrom(with.Names, with.Accounts, with.Suffix),
Machines: entriesFrom(with.Machines, with.Accounts, with.Suffix),
Zones: zonesFrom(with.Zones),
Holders: holdersFrom(with.Holders, with.Accounts, with.Suffix, r.Node),
Suffix: strings.TrimPrefix(suffixOr(suffix), "."),
Names: entriesFrom(every, accounts, suffix),
Machines: entriesFrom(machines, accounts, suffix),
Zones: zonesFrom(zones),
}
out := make([]map[string]any, 0, len(names))
@@ -260,28 +250,3 @@ func zonesFrom(zones []ZoneAt) []rosterZone {
sort.Slice(out, func(i, j int) bool { return out[i].Zone < out[j].Zone })
return out
}
// holdersFrom is each replicated seat's holders as a template ranges them: this machine first when
// it holds the seat, then the others by name (novox/hq ADR 0223). A machine with no address is left
// out, as everywhere in the roster — a resolver named at nothing is a lookup that hangs.
func holdersFrom(holders map[string]map[string]string, accounts map[string]string, suffix, node string) map[string][]rosterEntry {
out := make(map[string][]rosterEntry, len(holders))
for seat, at := range holders {
entries := entriesFrom(at, accounts, suffix)
sort.SliceStable(entries, func(i, j int) bool {
return entries[i].Name == node && entries[j].Name != node
})
out[seat] = entries
}
return out
}
// sortedFacts is a module's fact names in order, so what is said about them is said the same way twice.
func sortedFacts(facts map[string]RosterFile) []string {
out := make([]string, 0, len(facts))
for name := range facts {
out = append(out, name)
}
sort.Strings(out)
return out
}
internal/catalogue/seat_dependencies.go
+20 -4
View File
@@ -6,10 +6,9 @@ import (
"strings"
)
// A module depends on the node seats that apply its resources (novox/hq ADR 0207) and on the
// seats it contributes to (novox/hq ADR 0210). A third source — what a seat it holds needs beside it
// (novox/hq ADR 0220) — had one user, node-resolver-config needing node-uplink, and went with that
// seat when the resolver file became the uplink's own (novox/hq ADR 0223).
// A module depends on the node seats that apply its resources (novox/hq ADR 0207), on the
// seats it contributes to (novox/hq ADR 0210), and on the seats a seat it holds needs beside it
// (novox/hq ADR 0220).
//
// Some of what a module declares is applied through software on the machine that is itself a
// module: a service through the service manager, a package through the package manager, a container
@@ -95,6 +94,9 @@ func DependsOn(m Manifest) []string {
for _, seat := range contributedTo(m) {
seen[seat] = true
}
for _, seat := range neededBesideClaims(m) {
seen[seat] = true
}
out := make([]string, 0, len(seen))
for s := range seen {
out = append(out, s)
@@ -126,6 +128,20 @@ func contributedTo(m Manifest) []string {
return out
}
// neededBesideClaims is every seat a seat the module claims at node scope needs held on the same
// node (novox/hq ADR 0220): the holder of node-resolver-config is only right while node-uplink's
// holder keeps the network manager off resolv.conf. Derived from the claim, as a resource's seat is
// derived from its type, so a module that claims the seat cannot leave the dependency out.
func neededBesideClaims(m Manifest) []string {
var out []string
for _, name := range nodeSeatsClaimed(m) {
if s, known := SeatNamed(name); known {
out = append(out, s.Needs...)
}
}
return out
}
// claimsSeat is whether a module claims a node seat, by its current name or one it used to have
// (ADR 0122), so a rename leaves the dependency met.
func claimsSeat(m Manifest, seat string) bool {
internal/catalogue/seats.go
+39 -45
View File
@@ -21,16 +21,8 @@ import (
type Seat struct {
// Name is what a manifest claims.
Name string
// Scope is where there may be only one holder — unless the seat is Replicated.
// Scope is where there may be only one holder.
Scope string
// Replicated says a mesh seat may be held on several machines at once, each holder answering the
// same thing (novox/hq ADR 0223): the mesh's resolver, held on the anchor and the home server so a
// machine's resolver file lists two that give one answer. Each holder is on record, added by an
// act (`seat <name> --add <node>/<module>`), never by being assigned: two claimants with nothing on
// record are refused exactly as for any mesh seat. One per machine still — two modules on one
// node claiming it are refused. Compiled, never stored, like Receives: it is the mesh's definition of
// the role, and the store's rows carry no column for it.
Replicated bool
// Delivers is the provision the seat's holder answers for, or empty. A seat that delivers a
// provision may only be held by a module providing it at the seat's scope, and its holder is
// what a requirement for that provision resolves to when several modules provide it.
@@ -53,6 +45,13 @@ type Seat struct {
// ${contribution:<seat>:<kind>}. Compiled, never stored: like the protocol, it is the mesh's
// definition of the role, and the store's rows carry no column for it.
Receives []Receivable
// Needs is every node seat this seat's holder needs held on its own node (novox/hq ADR 0220): a
// role whose holder is only right while another role is filled beside it. A module claiming this
// seat depends on each, exactly as a module declaring a service depends on the service manager
// (ADR 0207) — derived from the claim, never written in a manifest, and judged over the node's
// whole set of assignments. Compiled, never stored, like Receives: it is the mesh's definition of
// the role, and the store's rows carry no column for it.
Needs []string
// Decision is the record that made it a seat.
Decision string
}
@@ -141,20 +140,14 @@ var defaultSeats = append([]Seat{
// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
{Name: "mesh-build-machine", Scope: ScopeMesh,
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
// than one machine, each answering the same names from the same roster, and every machine's
// resolver file lists every holder — its own first — and no public resolver, so whichever answers
// first gives the one answer. Delivers what a machine's resolver configuration requires, so that
// requirement resolves to a holder wherever they are placed.
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Replicated: true,
Decision: "novox/hq ADR 0194, ADR 0223"},
// **A machine's names are one module's** (novox/hq ADR 0199, ADR 0223): its holder writes
// /etc/hostname and the machine's own lines in /etc/hosts, and keeps every other line of the hosts
// file as the operator's, changed through these three verbs on that machine alone. The controller
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
// alias on a mesh that knew it.
{Name: "node-hostname", Scope: ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
// **The mesh's one resolver** (novox/hq ADR 0194, 0196): every node's internal domain, held in one
// place, and every node and container asks it first. Delivers what a machine's resolver
// configuration requires, so that requirement resolves to the holder wherever it is placed.
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Decision: "novox/hq ADR 0194"},
// **A machine's /etc/hosts is one module's** (novox/hq ADR 0199): its holder writes the machine's
// own lines and keeps every other line as the operator's, changed through these three verbs on that
// machine alone. The controller holds none of it.
{Name: "node-hosts-file", Scope: ScopeNode, Decision: "novox/hq ADR 0199",
Serves: []Verb{
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
"the operator's, or the block of the module or tool that writes it.",
@@ -255,12 +248,18 @@ var defaultSeats = append([]Seat{
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
// model change, not a rename, so it stays until that is built.
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
// The program that manages the machine's own network. It delivers nothing: its holder keeps the
// manager and the mesh from contradicting each other — the private network's interface left
// alone — and writes the machine's resolver file itself, because the manager is what would
// otherwise rewrite it (novox/hq ADR 0223, which retired node-resolver-config into this seat).
// It never declares a link, an address or a wireless network, because the link is the only
// channel a fix could arrive on. A seat
// What a machine asks for names (novox/hq ADR 0121, ADR 0196): its holder writes resolv.conf.
// **And it needs the uplink held beside it** (novox/hq ADR 0220): resolv.conf stays the mesh's
// only while the program managing the machine's network is told to keep its hands off it, and
// that is what the uplink's holder says (ADR 0117). Without one, the first connectivity change
// rewrites the file and every surface of the mesh still reads green — so it is refused at
// assignment instead.
{Name: "node-resolver-config", Scope: ScopeNode, Decision: "novox/hq ADR 0121, ADR 0220",
Needs: []string{"node-uplink"}},
// The program that manages the machine's own network. It delivers nothing: its holder only
// keeps the manager and the mesh from contradicting each other — the resolver file left to the
// mesh, the private network's interface left alone — and never declares a link, an address or
// a wireless network, because the link is the only channel a fix could arrive on. A seat
// rather than a condition in the resolver's module, so a machine running two managers is
// refused at assignment instead of found by the resolver being rewritten (novox/hq ADR 0117).
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117"},
@@ -315,11 +314,11 @@ func UseSeats(s []Seat) {
row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
}
}
// What a seat receives and whether it is replicated are never stored (novox/hq ADR 0212, ADR
// 0223), so they are always the compiled ones.
// What a seat receives and what its holder needs are never stored (novox/hq ADR 0212, ADR
// 0220), so they are always the compiled ones.
if d, known := byName[row.Name]; known {
row.Receives = d.Receives
row.Replicated = d.Replicated
row.Needs = d.Needs
}
merged = append(merged, row)
}
@@ -499,24 +498,19 @@ func seatNames() string {
// two modules on one node could both provide a provision, and only the one holding the seat
// answers for it. Nothing when no seat delivers the provision, when nobody holds
// it, or when the holder is not among the providers offered.
//
// **The first holder in the providers' own order** (novox/hq ADR 0223). A replicated seat has
// several, and the answer must not depend on the order the mesh happened to resolve its machines
// in: the providers come sorted by machine, so every consumer is bound to the same one. A seat with
// one holder gets the same answer as before.
func HolderAmong(provision string, providers []Provider, held []Held) (Provider, bool) {
seat, delivered := SeatDelivering(provision)
if !delivered {
return Provider{}, false
}
for _, p := range providers {
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a
// seat's former name still matches it after a rename (novox/hq ADR 0122).
hs, ok := SeatNamed(h.Claim)
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
continue
}
for _, h := range held {
// Resolve the held claim to a seat rather than comparing names, so a record naming a seat's
// former name still matches it after a rename (novox/hq ADR 0122).
hs, ok := SeatNamed(h.Claim)
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
continue
}
for _, p := range providers {
if p.Node == h.Node && p.Module == h.Module {
return p, true
}
internal/catalogue/seats_test.go
+4 -6
View File
@@ -46,18 +46,16 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
delivered[s.Delivers] = s.Name
}
}
// Thirty-six since node-resolver-config retired into node-uplink (novox/hq ADR 0223); thirty-seven
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
// Thirty-seven since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
// node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215);
// thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199, now
// node-hostname); thirty-four
// thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199); thirty-four
// with node-hotkeys (ADR 0212); thirty-three with node-power (ADR 0211); thirty-two since the
// graphical session's eleven (ADR 0208); twenty-one with node-package-manager and
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
// mesh-build-machine row goes, when no registered manifest claims it.
if len(Seats()) != 36 {
t.Errorf("the mesh defines %d seats rather than 36; the set is closed, so a change here is "+
if len(Seats()) != 37 {
t.Errorf("the mesh defines %d seats rather than 37; the set is closed, so a change here is "+
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
}
}
internal/catalogue/two_resolvers_test.go
-270
View File
@@ -1,270 +0,0 @@
package catalogue
import (
"strings"
"testing"
)
// The mesh has two resolvers (novox/hq ADR 0223): `mesh-dns-resolver` is replicated, held on the
// anchor and on the home server, each answering the same names; every machine's resolver file lists
// every holder — its own first when it is one — and no public resolver. ADR 0196 listed the mesh's
// resolver then a public one, and musl asks both at once and takes the first reply: from the home
// server the public "no such name" for the anchor's mesh name won, every time, in every Alpine build.
// The file is written by the module holding the machine's uplink (ADR 0223 part 2).
// resolverMachines is the anchor and the home server holding the resolver, and a laptop holding nothing.
var resolverMachines = map[string]string{
"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "home.internal": "10.42.0.3"}
// bothResolvers is the two holders on record, as `seat mesh-dns-resolver --add` leaves them.
var bothResolvers = []Held{
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "anchor", Module: "dnsmasq"},
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "home", Module: "dnsmasq"},
}
// uplinks is every module in the catalogue holding node-uplink, and so writing the machine's resolver
// file (novox/hq ADR 0223 part 2): the program that would otherwise rewrite it is the one that writes it.
var uplinks = []string{"dhcpcd", "networkmanager", "systemd-networkd"}
// managing is a machine as each uplink module needs it: able to install, run a service and run the
// manager that module is for.
func managing(node string) Node {
caps := map[string]bool{"package-manager": true, "service-manager": true}
for _, u := range uplinks {
caps["uplink-"+u] = true
}
return Node{Name: node, At: node + ".internal", Capabilities: caps}
}
// twoResolverShelf is the resolver, the uplink modules that write what a machine asks, and a stand-in
// answering `mesh-addressing`.
func twoResolverShelf(t *testing.T) map[string]Manifest {
t.Helper()
out := map[string]Manifest{
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
"dnsmasq": catalogueManifest(t, "dnsmasq"),
}
for _, u := range uplinks {
out[u] = catalogueManifest(t, u)
}
return out
}
// worldWithout is the rest of the mesh as a plan for one machine sees it: every other holder's claim
// and offer, and both holders on record.
func worldWithout(node string) World {
w := World{Holdings: bothResolvers, Offered: map[string][]Provider{}}
for _, h := range bothResolvers {
if h.Node == node {
continue
}
w.Held = append(w.Held, h)
w.Offered["wildcard-resolution"] = append(w.Offered["wildcard-resolution"],
Provider{Node: h.Node, At: h.Node + ".internal", Module: h.Module})
}
return w
}
// resolvConfOn resolves and composes one machine and answers with the nameservers its resolver file
// lists, in order, and the file. The file is the uplink's — `uplink` is one of the assigned modules —
// and nothing else on the machine declares that path.
func resolvConfOn(t *testing.T, node, uplink string, assigned []string) ([]string, string) {
t.Helper()
got, err := Resolve(twoResolverShelf(t), assigned, managing(node), worldWithout(node))
if err != nil {
t.Fatalf("%s with %s does not resolve with two resolvers on record: %v", node, uplink, err)
}
out, err := got.Declaration(Rendering{
Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
Holders: map[string]map[string]string{"mesh-dns-resolver": {
"anchor.internal": "10.42.0.1", "home.internal": "10.42.0.3"}},
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
})
if err != nil {
t.Fatalf("%s with %s does not compose: %v", node, uplink, err)
}
var file map[string]any
for _, r := range out {
if r["path"] != "/etc/resolv.conf" {
continue
}
if file != nil {
t.Fatalf("%s declares /etc/resolv.conf twice: %v and %v", node, file["id"], r["id"])
}
file = r
}
if file == nil || file["id"] != uplink+".fact-resolvers" {
t.Fatalf("%s's resolver file is not %s's: %v", node, uplink, file)
}
content, _ := file["content"].(string)
var servers []string
for _, line := range strings.Split(content, "\n") {
if strings.HasPrefix(line, "nameserver ") {
servers = append(servers, strings.TrimPrefix(line, "nameserver "))
}
}
return servers, content
}
// Per uplink module: each writes a resolver file listing both holders, the machine's own first on a
// holder, and only the holders on a machine that is none.
func TestEveryUplinkListsBothResolversItsOwnFirst(t *testing.T) {
for _, uplink := range uplinks {
for node, want := range map[string][]string{
"anchor": {"10.42.0.1", "10.42.0.3"},
"home": {"10.42.0.3", "10.42.0.1"},
"laptop": {"10.42.0.1", "10.42.0.3"},
} {
assigned := []string{uplink}
if node != "laptop" {
assigned = append(assigned, "dnsmasq")
}
servers, content := resolvConfOn(t, node, uplink, assigned)
if strings.Join(servers, " ") != strings.Join(want, " ") {
t.Errorf("%s on %s lists %v; want %v\n%s", uplink, node, servers, want, content)
}
for _, public := range []string{"1.1.1.1", "8.8.8.8", "9.9.9.9"} {
if strings.Contains(content, public) {
t.Errorf("%s on %s lists a public resolver beside the mesh's (ADR 0223):\n%s", uplink, node, content)
}
}
if !strings.HasSuffix(content, "\noptions timeout:1 attempts:2 edns0\n") {
t.Errorf("%s on %s does not end with two short attempts:\n%s", uplink, node, content)
}
}
}
}
// One file, whichever manager the machine runs: the three modules carry the same template, so a
// machine changing its manager changes nothing in what it asks, and a fix made to one is made to all.
func TestEveryUplinkWritesTheSameResolverFile(t *testing.T) {
var first, firstOf string
for _, uplink := range uplinks {
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
if !ok || fact.Path != "/etc/resolv.conf" || fact.Shared || fact.Home {
t.Fatalf("%s does not write the machine's resolver file whole: %+v", uplink, fact)
}
if first == "" {
first, firstOf = fact.Template, uplink
continue
}
if fact.Template != first {
t.Errorf("%s's resolver file differs from %s's; the three are kept identical", uplink, firstOf)
}
}
}
// The requirement stays with what writes the file (ADR 0223 part 1): a machine is refused when nothing
// in the mesh resolves, rather than given a file listing nothing.
func TestEveryUplinkRequiresTheMeshsResolver(t *testing.T) {
for _, uplink := range uplinks {
found := false
for _, r := range catalogueManifest(t, uplink).Requires {
found = found || r == "wildcard-resolution"
}
if !found {
t.Errorf("%s writes the resolver file and does not require wildcard-resolution", uplink)
}
}
_, err := Resolve(twoResolverShelf(t), []string{"networkmanager"}, managing("laptop"), World{})
if err == nil || !strings.Contains(err.Error(), "wildcard-resolution") {
t.Errorf("an uplink was composed on a mesh with no resolver: %v", err)
}
}
// A holder answers its own requirement, even though the other holder sorts first (issue 258 kept).
func TestAHolderAnswersItsOwnRequirement(t *testing.T) {
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq", "networkmanager"},
managing("home"), worldWithout("home"))
if err != nil {
t.Fatal(err)
}
for _, n := range got.Needs {
if n.Name == "wildcard-resolution" && n.From != "home" {
t.Errorf("the home server's uplink is bound to %s; it holds the seat itself", n.From)
}
}
}
// A seat held once is still held once: a second claimant on another machine is refused while
// nothing is on record, and a store recording two holders of it is refused, naming the seat.
func TestASingleHolderMeshSeatStillRefusesASecondHolder(t *testing.T) {
store := shelf(mod("postgres", nil, nil, nil, Claim{Name: "mesh-store", Scope: ScopeMesh}))
other := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: "anchor", Module: "postgres"}
if _, err := Resolve(store, []string{"postgres"}, workstation(), World{Held: []Held{other}}); err == nil ||
!strings.Contains(err.Error(), "one per mesh") {
t.Errorf("a second claimant of a seat held once was not refused: %v", err)
}
here := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: workstation().Name, Module: "postgres"}
_, err := Resolve(store, []string{"postgres"}, workstation(),
World{Held: []Held{other}, Holdings: []Held{other, here}})
if err == nil || !strings.Contains(err.Error(), "on record as held by 2") {
t.Errorf("two holders on record for a seat held once were not refused: %v", err)
}
}
// Replicated is not "whoever is assigned": two claimants with nothing on record are refused, as for
// any mesh seat, and each holder is added by an act.
func TestTwoUnrecordedClaimantsOfTheReplicatedSeatAreRefused(t *testing.T) {
w := worldWithout("home")
w.Holdings = nil
_, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "home", At: "home.internal"}, w)
if err == nil || !strings.Contains(err.Error(), "seat mesh-dns-resolver --to") {
t.Errorf("a second resolver with nothing on record was not refused, naming the handover: %v", err)
}
}
// Only the mesh's resolver is replicated: a seat being replicated is a decision, recorded.
func TestOnlyTheResolverIsReplicated(t *testing.T) {
for _, s := range Seats() {
if s.Replicated != (s.Name == "mesh-dns-resolver") {
t.Errorf("%s replicated = %v; only mesh-dns-resolver is (ADR 0223)", s.Name, s.Replicated)
}
}
UseSeats([]Seat{{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution"}})
defer UseSeats(DefaultSeats())
if s, _ := SeatNamed("mesh-dns-resolver"); !s.Replicated {
t.Error("loading the set from the store, which has no column for it, lost the resolver's replication")
}
}
// Every consumer is bound to the same holder whatever order the mesh resolved its machines in.
func TestTheFirstHolderIsTheFirstProvider(t *testing.T) {
providers := []Provider{{Node: "anchor", Module: "dnsmasq"}, {Node: "home", Module: "dnsmasq"}}
for _, held := range [][]Held{bothResolvers, {bothResolvers[1], bothResolvers[0]}} {
if p, ok := HolderAmong("wildcard-resolution", providers, held); !ok || p.Node != "anchor" {
t.Errorf("held in order %v answered %v", held, p)
}
}
}
// One host record per machine, beside its wildcard (novox/hq issue 262): a name with a host record
// says it exists and has no IPv6 address, where the wildcard alone said there is no such name, and
// musl reads that as final.
func TestTheResolverHasOneHostRecordPerMachine(t *testing.T) {
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor", At: "anchor.internal"},
World{})
if err != nil {
t.Fatal(err)
}
out, err := got.Declaration(Rendering{Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}}})
if err != nil {
t.Fatal(err)
}
content, _ := byID(out)["dnsmasq.fact-node-zones"]["content"].(string)
records := map[string]int{}
for _, line := range strings.Split(content, "\n") {
if strings.HasPrefix(line, "host-record=") {
records[strings.TrimPrefix(line, "host-record=")]++
}
}
for name, at := range resolverMachines {
if records[name+","+at] != 1 {
t.Errorf("%s has %d host records at %s, and has one:\n%s", name, records[name+","+at], at, content)
}
}
if len(records) != len(resolverMachines) {
t.Errorf("%d host records for %d machines:\n%s", len(records), len(resolverMachines), content)
}
}
internal/catalogue/verbs.go
+11 -46
View File
@@ -73,13 +73,6 @@ var ControllerVerbs = []Verb{
{Name: "tools", Description: "Every seat's tools, from the mesh's own records: what each role " +
"answers, whether or not its holder is up. The mesh's own verbs are the mesh-controller seat's.",
Input: schema(nil, nil)},
{Name: "calls", Description: "The calls this controller answered lately and what came of each — " +
"one still running, one that finished after its caller was told it was running, one whose answer " +
"the bus refused — and, given a call's id, its whole answer (novox/hq issue 265). A call that has " +
"not finished within ten seconds answers that it is running, with its id; this is where it ends.",
Input: schema(map[string]string{
"call": "a call's id, as a running answer or `calls` gives it: that call and its whole answer",
}, nil)},
{Name: "status", Description: "What is wrong, what is quiet, what is out of date, and which " +
"machines are behind what the mesh would send them.",
Input: schema(nil, nil)},
@@ -97,7 +90,6 @@ var ControllerVerbs = []Verb{
Input: schema(map[string]string{
"module": "one module's name; every module when absent",
"log": "a build's id (as `builds` lists it): print what the build machine said, line by line",
"limit": "how many builds to list (default 20); not with log",
}, nil)},
{Name: "plans", Description: "What the last merges produced and where each stands (novox/hq ADR 0162): " +
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id.",
@@ -109,14 +101,9 @@ var ControllerVerbs = []Verb{
"repository": "owner/repository: the plan a merge there would produce, saving nothing (what-if); with paths or modules",
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
"modules": "with repository: or the modules it would change, comma-separated",
"limit": "how many plans to list (default 10); only when listing",
}, nil)},
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
"or, with files, the files it would be given.",
Input: schema(map[string]string{
"node": "the machine's name",
"files": "\"true\": the files this machine would be given, instead of the declaration as JSON",
}, []string{"node"}, "files")},
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it.",
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})},
{Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " +
"or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207).",
Input: schema(map[string]string{"node": "the machine's name",
@@ -134,14 +121,8 @@ var ControllerVerbs = []Verb{
}, []string{"node", "provision", "from", "module"})},
{Name: "unpin", Description: "Take that choice back, putting the question to the mesh again.",
Input: schema(map[string]string{"node": "the machine's name", "provision": "the provision"}, []string{"node", "provision"})},
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
"(a push can reload the bus, which then refuses any answer still to come).",
Input: schema(map[string]string{
"node": "the machine's name; without it, every machine that is behind",
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
}, nil, "behind")},
{Name: "push", Description: "Send a machine everything it should be — or every machine that is behind, when no machine is named.",
Input: schema(map[string]string{"node": "the machine's name; every machine behind when absent"}, nil)},
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine, " +
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
@@ -167,8 +148,8 @@ var ControllerVerbs = []Verb{
"module": "the module's name",
"values": "the settings as a JSON object, for set",
"node": "one machine; the whole mesh when absent",
"clear": "\"true\" to remove the layer instead of setting it; not with values",
}, []string{"module"}, "clear")},
"clear": "\"true\" to remove the layer instead of setting it",
}, []string{"module"})},
{Name: "command", Description: "Run one command line of the controller's own, as you would type it at its " +
"shell — `node account g14 jochen`, `node show ace`, `module list` — and answer what it printed. The " +
"generic verb beside the named ones (novox/hq ADR 0154): everything the binary can do, without a verb " +
@@ -186,7 +167,7 @@ var ControllerVerbs = []Verb{
Input: schema(map[string]string{"id": "the ask's build id, as `queue` lists it"}, []string{"id"})},
{Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " +
"recorded failed, cancelled by hand. Never touches one in flight.",
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil, "dead")},
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil)},
{Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " +
"build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.",
Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})},
@@ -197,7 +178,7 @@ var ControllerVerbs = []Verb{
"id": "the build's id",
"register": "\"true\" to register what it builds",
"older": "\"true\", with register: even though a newer build of the module is registered",
}, []string{"id"}, "register", "older")},
}, []string{"id"})},
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
"announces it failed, killed by hand — settled, never handed to another machine.",
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
@@ -216,27 +197,11 @@ var ControllerVerbs = []Verb{
}
// schema is a JSON schema for an object of string properties, which is every argument the verbs
// above take. Kept small on purpose: a schema an agent cannot read is a tool it cannot call. The
// switches are the properties that are "true" or "false" and nothing else, said in the schema as an
// enum so a caller sees it and the verb can refuse any other word rather than read it as false.
//
// **The schema is the whole of what a verb takes** (novox/hq issue 244): an argument it does not
// name is refused when the verb is called, never passed over.
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
isSwitch := map[string]bool{}
for _, s := range switches {
if _, declared := properties[s]; !declared {
panic("a switch that is not a property: " + s)
}
isSwitch[s] = true
}
// above take. Kept small on purpose: a schema an agent cannot read is a tool it cannot call.
func schema(properties map[string]string, required []string) map[string]any {
props := map[string]any{}
for name, description := range properties {
p := map[string]any{"type": "string", "description": description}
if isSwitch[name] {
p["enum"] = []string{"true", "false"}
}
props[name] = p
props[name] = map[string]any{"type": "string", "description": description}
}
out := map[string]any{"type": "object", "properties": props}
if len(required) > 0 {
internal/inventory/busrecords.go
+1 -1
View File
@@ -124,7 +124,7 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
d := broker.Declared{
Module: m.Module,
Emits: m.EmitsAll(),
Emits: m.Emits,
Consumes: fromModules,
Watches: watches,
// The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the
internal/inventory/holdings_test.go
-54
View File
@@ -97,57 +97,3 @@ func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T)
t.Fatalf("a re-told membership did not replace the first: %v", got)
}
}
// A replicated seat has several holders on record (novox/hq ADR 0223): each is added beside the
// others, adding one twice changes nothing, a handover still leaves exactly one, and unassigning one
// takes only its own row.
func TestAReplicatedSeatHasSeveralHoldersOnRecord(t *testing.T) {
resolver := catalogue.Manifest{Module: "resolver", Version: "1",
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}
inv, ctx := aMeshWith(t, resolver)
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
for _, n := range []string{"anchor", "home"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, n, "resolver"); err != nil {
t.Fatal(err)
}
}
if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "anchor", "resolver"); err != nil {
t.Fatal(err)
}
for range 2 {
if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
t.Fatal(err)
}
}
held, err := inv.Holdings(ctx)
if err != nil {
t.Fatal(err)
}
if len(held) != 2 || held[0].Node != "anchor" || held[1].Node != "home" {
t.Fatalf("the two holders are not both on record, once each: %+v", held)
}
if err := inv.Unassign(ctx, "home", "resolver"); err != nil {
t.Fatal(err)
}
if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "anchor" {
t.Fatalf("unassigning one holder took more or less than its own row: %+v", held)
}
if _, err := inv.Assign(ctx, "home", "resolver"); err != nil {
t.Fatal(err)
}
if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
t.Fatal(err)
}
if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
t.Fatal(err)
}
if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "home" {
t.Fatalf("a handover left other holders on record: %+v", held)
}
}
internal/inventory/migrations/0062-a-replicated-seat-has-several-holders-on-record.sql
-14
View File
@@ -1,14 +0,0 @@
-- A replicated seat has several holders on record (novox/hq ADR 0223).
--
-- 0039 recorded one holder per seat, keyed by the seat: a handover replaced the row, so the seat was
-- never without a holder in between. The mesh's resolver is now held on more than one machine, each
-- answering the same names, and each of those holders is recorded — added by `seat <name> --add`,
-- never by being assigned. So a holding is keyed by the seat and the assignment holding it.
--
-- Nothing else changes. A handover (`seat <name> --to`) still leaves exactly one row, replacing every
-- holder in one transaction; whether a seat may have more than one is the seat's compiled definition,
-- judged by the controller before a row is added, and a store holding two for any other seat is
-- refused at resolution, naming the seat. Every existing row is one per seat, so it satisfies the new
-- key as it stands.
alter table seat_holding drop constraint seat_holding_pkey;
alter table seat_holding add primary key (seat, node, module);
internal/inventory/migrations/0063-the-resolver-file-is-the-uplinks.sql
-17
View File
@@ -1,17 +0,0 @@
-- What a machine asks for names is the uplink's holder's to write (novox/hq ADR 0223, retiring what
-- ADR 0121 and ADR 0220 decided for node-resolver-config).
--
-- `/etc/resolv.conf` is written by the module holding `node-uplink` — the program that would otherwise
-- rewrite it — so the seat whose holder wrote it, and its need of the uplink beside it, go. Its only
-- claimant, `resolv-conf`, declared nothing for one release while every machine handed the file to its
-- uplink module in one apply, and was then unassigned everywhere and forgotten before this runs.
--
-- **The compiled defaults no longer carry it, and that alone would not remove it**: seeding adds a
-- seat a release ships and never takes one away (ADR 0122), as 0060 found for the per-node resolver.
-- A holding on record goes with it by cascade; a node seat has none. No alias is kept: nothing was
-- renamed, and a manifest still claiming the old name should be refused at registration, naming it.
--
-- Numbered after 0063 (node-hosts-file renamed to node-hostname), which is expected to merge first;
-- if this one lands first, the two are renumbered so the order they merge in is the order they run.
delete from seat_alias where seat = 'node-resolver-config' or alias = 'node-resolver-config';
delete from seat where name = 'node-resolver-config';
internal/inventory/migrations/0064-a-machines-names-are-one-seats.sql
-23
View File
@@ -1,23 +0,0 @@
-- A machine's names are one seat's (novox/hq ADR 0223 part 3): `node-hosts-file` is renamed
-- `node-hostname`, whose holder writes /etc/hostname beside the machine's own lines in /etc/hosts.
--
-- A rename is a database update (ADR 0122): the row keeps its verbs, the former name becomes an alias
-- that resolves to it, so a manifest registered under the old name — the `hosts` module still assigned
-- while machines move to `hostname` — goes on holding the one seat, and two claimants of it on one
-- machine are still refused.
--
-- **Both rows may exist when this runs**, as 0048 found for the artifact store: a controller whose
-- compiled defaults carry the new name may seed it before this migration. Then the old row's holding
-- moves to it and the old row goes; otherwise the old row is renamed. Either way the old name becomes
-- an alias.
update seat_holding set seat = 'node-hostname'
where seat = 'node-hosts-file'
and exists (select 1 from seat where name = 'node-hostname');
delete from seat
where name = 'node-hosts-file'
and exists (select 1 from seat where name = 'node-hostname');
update seat set name = 'node-hostname', decided = 'novox/hq ADR 0199, ADR 0223'
where name = 'node-hosts-file';
insert into seat_alias (alias, seat) values ('node-hosts-file', 'node-hostname')
on conflict (alias) do update set seat = excluded.seat;
update seat_alias set seat = 'node-hostname' where seat = 'node-hosts-file';
internal/inventory/migrations/0065-a-provider-says-which-consumer-it-keeps-failing.sql
-20
View File
@@ -1,20 +0,0 @@
-- A provider says which consumer it keeps failing (novox/hq ADR 0224).
--
-- On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a day and
-- only its journal said so (issue 179). A provider now announces a consumer it has failed for minutes
-- without one success, and the consumer recovering; the controller keeps the newest failing word per
-- provider module, the machine it runs on and the consumer, and removes it on recovery. `status` and
-- `node show` read this table: a row is a problem until it is gone.
create table provider_standing (
module text not null,
provider_node text not null,
consumer text not null,
consumer_node text not null default '',
provision text not null default '',
class text not null default '',
error text not null default '',
since timestamptz not null,
attempts integer not null default 0,
said_at timestamptz not null default now(),
primary key (module, provider_node, consumer)
);
internal/inventory/nodes.go
+2 -2
View File
@@ -851,9 +851,9 @@ func (i *Inventory) DoingOf(ctx context.Context, name string) (Doing, bool, erro
var d Doing
var failed []byte
err = i.store.Pool().QueryRow(ctx,
`select outcome, refused, failed, applied, at, failing_since, failures, coalesce(declared, '')
`select outcome, refused, failed, applied, at, failing_since, failures
from node_report where node = $1`, node.ID).
Scan(&d.Outcome, &d.Refused, &failed, &d.Applied, &d.At, &d.Since, &d.Times, &d.Declared)
Scan(&d.Outcome, &d.Refused, &failed, &d.Applied, &d.At, &d.Since, &d.Times)
if errors.Is(err, pgx.ErrNoRows) {
return Doing{}, false, nil
}
internal/inventory/seats.go
+10 -37
View File
@@ -215,50 +215,23 @@ func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error {
return nil
}
// HoldSeat records that one assignment holds a seat, replacing whoever held it — every holder, for a
// replicated seat (novox/hq ADR 0223) — as one transaction, so the seat is never without a holder in
// between (novox/hq ADR 0131, design 28 task 5.3). The assignment must exist; the store refuses
// otherwise, and that refusal is the right one: a seat cannot be handed to something that is not
// running anywhere.
// HoldSeat records that one assignment holds a seat, replacing whoever held it — as one write, so
// the seat is never without a holder in between (novox/hq ADR 0131, design 28 task 5.3). The
// assignment must exist; the store refuses otherwise, and that refusal is the right one: a seat
// cannot be handed to something that is not running anywhere.
func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module string) error {
node, err := i.NodeByName(ctx, nodeName)
if err != nil {
return err
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
if _, err := tx.Exec(ctx,
`delete from seat_holding where seat = $1 and not (node = $2 and module = $3)`,
seat, node.ID, module); err != nil {
return fmt.Errorf("handing %s to %s on %s: %w", seat, module, nodeName, err)
}
if _, err := tx.Exec(ctx,
_, err = i.store.Pool().Exec(ctx,
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
on conflict (seat, node, module) do update set scope = excluded.scope, since = now()`,
seat, scope, node.ID, module); err != nil {
on conflict (seat) do update set scope = excluded.scope, node = excluded.node,
module = excluded.module, since = now()`,
seat, scope, node.ID, module)
if err != nil {
return fmt.Errorf("recording %s on %s as the holder of %s: %w", module, nodeName, seat, err)
}
return tx.Commit(ctx)
}
// AddSeatHolder records one more assignment holding a replicated seat, beside those already on
// record (novox/hq ADR 0223). Whether the seat may have several holders is the caller's to judge —
// the seat's definition is compiled, and the store holds no column for it. Recording a holder
// already on record changes nothing.
func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, module string) error {
node, err := i.NodeByName(ctx, nodeName)
if err != nil {
return err
}
if _, err := i.store.Pool().Exec(ctx,
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
on conflict (seat, node, module) do nothing`,
seat, scope, node.ID, module); err != nil {
return fmt.Errorf("adding %s on %s as a holder of %s: %w", module, nodeName, seat, err)
}
return nil
}
@@ -267,7 +240,7 @@ func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, mo
func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
rows, err := i.store.Pool().Query(ctx,
`select h.seat, h.scope, n.name, h.module, coalesce(n.site, '')
from seat_holding h join node n on n.id = h.node order by h.seat, n.name, h.module`)
from seat_holding h join node n on n.id = h.node order by h.seat`)
if err != nil {
return nil, err
}
internal/inventory/standing.go
-76
View File
@@ -1,76 +0,0 @@
package inventory
import (
"context"
"time"
)
// ProviderStanding is a consumer a provider says it keeps failing (novox/hq ADR 0224).
type ProviderStanding struct {
// Module is the provider's module, and ProviderNode the machine it runs on.
Module string `json:"module"`
ProviderNode string `json:"provider-node"`
// Provision is the interface it provides, e.g. `oidc-client`.
Provision string `json:"provision"`
// Consumer is the identity the mesh derived for the consumer, ConsumerNode its machine.
Consumer string `json:"consumer"`
ConsumerNode string `json:"consumer-node"`
// Class is what kind of failure: credentials-rejected, unreachable, secret-unreadable, refused.
Class string `json:"class"`
Error string `json:"error"`
// Since is when the unbroken run of failures began; Attempts how many it has been.
Since time.Time `json:"since"`
Attempts int `json:"attempts"`
// SaidAt is when the controller last heard it. A provider says it again every quarter of an hour
// while it lasts, so an old one is a provider that stopped saying anything.
SaidAt time.Time `json:"said-at"`
}
// SayAgainWithin is how long a failing standing stays current without being said again: twice the
// quarter of an hour a provider repeats it at. Older, and status says the provider has gone quiet.
const SayAgainWithin = 30 * time.Minute
// Quiet says the provider has not repeated this standing for longer than it would while it lasts.
func (s ProviderStanding) Quiet(now time.Time) bool { return now.Sub(s.SaidAt) > SayAgainWithin }
// KeepStanding records a provider's newest word: failing keeps it, recovered removes it, and says
// whether a recovery removed anything.
func (i *Inventory) KeepStanding(ctx context.Context, failing bool, s ProviderStanding) (bool, error) {
if !failing {
tag, err := i.store.Pool().Exec(ctx,
`delete from provider_standing where module = $1 and provider_node = $2 and consumer = $3`,
s.Module, s.ProviderNode, s.Consumer)
return err == nil && tag.RowsAffected() > 0, err
}
_, err := i.store.Pool().Exec(ctx, `
insert into provider_standing
(module, provider_node, consumer, consumer_node, provision, class, error, since, attempts, said_at)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
on conflict (module, provider_node, consumer) do update set
consumer_node = excluded.consumer_node, provision = excluded.provision,
class = excluded.class, error = excluded.error, since = excluded.since,
attempts = excluded.attempts, said_at = excluded.said_at`,
s.Module, s.ProviderNode, s.Consumer, s.ConsumerNode, s.Provision, s.Class, s.Error, s.Since, s.Attempts)
return false, err
}
// FailingProviders is every consumer a provider last said it keeps failing, oldest run first.
func (i *Inventory) FailingProviders(ctx context.Context) ([]ProviderStanding, error) {
rows, err := i.store.Pool().Query(ctx, `
select module, provider_node, provision, consumer, consumer_node, class, error, since, attempts, said_at
from provider_standing order by since, module, consumer`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []ProviderStanding
for rows.Next() {
var s ProviderStanding
if err := rows.Scan(&s.Module, &s.ProviderNode, &s.Provision, &s.Consumer, &s.ConsumerNode,
&s.Class, &s.Error, &s.Since, &s.Attempts, &s.SaidAt); err != nil {
return nil, err
}
out = append(out, s)
}
return out, rows.Err()
}
internal/inventory/standing_test.go
-130
View File
@@ -1,130 +0,0 @@
package inventory
import (
"slices"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A provider's standing (novox/hq ADR 0224), from the grant that lets it say so to the row status
// reads.
// The broker spells the events itself because it cannot import the catalogue; the two agree.
func TestTheBrokerAndTheCatalogueNameTheSameStandingEvents(t *testing.T) {
if broker.ProvisionerFailing != catalogue.ProvisionerFailing ||
broker.ProvisionerRecovered != catalogue.ProvisionerRecovered {
t.Fatal("the broker and the catalogue disagree about what a provider's standing is called")
}
}
// **Every provider may say it, whatever its manifest lists**: a provider whose manifest forgot the
// events would have its announcement refused by the bus, and fail its consumers as silently as on
// 2026-10-05 (issue 179). A module that receives no contributions provides nothing and is given
// nothing.
func TestEveryProviderIsGrantedItsStandingAndNothingElseIs(t *testing.T) {
provider := catalogue.Manifest{Module: "keycloak", Version: "1",
Emits: []string{"client.created"}, Receives: map[string]string{"oidc-client": "/x/mesh.json"}}
consumer := catalogue.Manifest{Module: "grafana", Version: "1", Emits: []string{"dashboard.saved"}}
d := declaredFor(provider, nil)
for _, e := range []string{"client.created", catalogue.ProvisionerFailing, catalogue.ProvisionerRecovered} {
if !slices.Contains(d.Emits, e) {
t.Fatalf("a provider is not granted %s: %v", e, d.Emits)
}
}
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindModule, Node: "anchor",
Module: "keycloak", Emits: d.Emits})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(perms.Publish, "mesh.mod.keycloak.event.provisioner.failing") {
t.Fatalf("the bus would refuse a provider's standing: %v", perms.Publish)
}
if got := declaredFor(consumer, nil).Emits; slices.Contains(got, catalogue.ProvisionerFailing) {
t.Fatalf("a module that provides nothing was granted a provider's standing: %v", got)
}
// Declared by hand as well: said once.
provider.Emits = append(provider.Emits, catalogue.ProvisionerFailing)
n := 0
for _, e := range provider.EmitsAll() {
if e == catalogue.ProvisionerFailing {
n++
}
}
if n != 1 {
t.Fatalf("%v", provider.EmitsAll())
}
}
// And the controller may hear it from every provider, and only those two events.
func TestTheControllerHearsEveryProvidersStanding(t *testing.T) {
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"} {
if !slices.Contains(perms.Subscribe, want) {
t.Fatalf("the controller may not hear %s: %v", want, perms.Subscribe)
}
}
if slices.Contains(perms.Subscribe, "mesh.mod.*.event.>") {
t.Fatal("the controller hears every event in the mesh")
}
}
func TestAFailingStandingIsKeptUntilItRecovers(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
s := ProviderStanding{Module: "keycloak", ProviderNode: "anchor", Provision: "oidc-client",
Consumer: "mesh_home_grafana", ConsumerNode: "home-server", Class: "credentials-rejected",
Error: "401 invalid_grant", Since: since, Attempts: 60}
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
t.Fatal(err)
}
// Said again: one row, the newest word.
s.Attempts = 31000
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
t.Fatal(err)
}
got, err := inv.FailingProviders(ctx)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].Attempts != 31000 || !got[0].Since.Equal(since) || got[0].ConsumerNode != "home-server" ||
got[0].Class != "credentials-rejected" || got[0].SaidAt.IsZero() {
t.Fatalf("%+v", got)
}
if got[0].Quiet(time.Now()) {
t.Fatal("a standing just said reads as quiet")
}
if !got[0].Quiet(time.Now().Add(SayAgainWithin + time.Minute)) {
t.Fatal("a standing not said again for longer than a provider repeats it does not read as quiet")
}
// The same consumer from another machine's provider is its own row.
other := s
other.ProviderNode = "laptop"
if _, err := inv.KeepStanding(ctx, true, other); err != nil {
t.Fatal(err)
}
cleared, err := inv.KeepStanding(ctx, false, s)
if err != nil || !cleared {
t.Fatalf("recovered cleared nothing: %v %v", cleared, err)
}
cleared, err = inv.KeepStanding(ctx, false, s)
if err != nil || cleared {
t.Fatalf("a recovery for nothing kept said it cleared something: %v %v", cleared, err)
}
got, err = inv.FailingProviders(ctx)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].ProviderNode != "laptop" {
t.Fatalf("%+v", got)
}
}
internal/link/calls.go
-329
View File
@@ -1,329 +0,0 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"regexp"
"strconv"
"sync"
"time"
"github.com/nats-io/nats.go"
)
// A role's tool call, kept after it is answered (novox/hq issue 265).
//
// **A call that outlasts its caller must not end in silence.** A caller waits a bounded time (the
// console thirty seconds), and the bus lets a holder answer a request exactly once and only while the
// server still remembers it was asked (`allow_responses`). Before this, a push that ran longer than
// its caller waited did everything it was asked and its caller read "did not answer in time"; and a
// push whose first act sent the bus's own machine a changed user list had its answer refused
// outright — a broker reloading its users forgets every reply it was about to permit — so the
// answer went nowhere and the only trace was the client library's line on the controller's standard
// error. So every call is answered within AnswerWithin, with its whole answer when it has one by
// then and with "still running as call <id>" when it does not; and every call is kept here, with
// what came of it, including an answer the bus refused, so `calls` can say it.
// AnswerWithin is how long a call runs before its caller is answered that it is still running. Well
// inside the shortest wait of a caller the mesh ships (the console's thirty seconds) and the bus's
// own window for an answer (broker.ResponseTTL), so the one answer a call has is never late for
// either. A variable so a test need not wait.
var AnswerWithin = 10 * time.Second
// KeptCalls is how many calls are kept, newest first; keptAnswer the largest answer kept of a call
// whose caller was sent it. One its caller never had is kept whole.
const (
KeptCalls = 100
keptAnswer = 64 << 10
)
// The states a kept call is in.
const (
CallRunning = "running"
CallAnswered = "answered"
// CallFinishedAfter is a call that finished after its caller was told it was still running: its
// answer is here and nowhere else.
CallFinishedAfter = "finished after its caller was answered"
)
// Call is one call of a role's tool, as `calls` shows it.
type Call struct {
ID string `json:"call"`
Seat string `json:"seat"`
Verb string `json:"verb"`
Args json.RawMessage `json:"arguments,omitempty"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
State string `json:"state"`
// Failed is the call's own answer being an error — an answer, not a timeout.
Failed bool `json:"failed,omitempty"`
// Answer is what the call answered, or would have: kept for a call whose caller did not get it.
Answer json.RawMessage `json:"answer,omitempty"`
// Refused is the bus refusing the answer this holder sent: the caller got nothing, and this is
// the only place that says what it would have.
Refused string `json:"answer refused by the bus,omitempty"`
reply string // the subject the answer went to, which the bus names when it refuses it
}
// CallLog keeps the latest calls a holder served.
type CallLog struct {
mu sync.Mutex
calls []*Call // oldest first
next uint64
now func() time.Time
// Follow is the tool that reads this log back, named in a running answer — set by a holder that
// serves one (the controller's `calls`); without it, the answer points at the holder's journal.
Follow string
}
// Calls is this process's log: one holder process serves its seats on one connection.
var Calls = NewCallLog()
func NewCallLog() *CallLog { return &CallLog{now: time.Now} }
func (l *CallLog) begin(seat, verb string, args json.RawMessage, reply string) *Call {
l.mu.Lock()
defer l.mu.Unlock()
l.next++
c := &Call{ID: "call-" + strconv.FormatInt(l.now().UnixNano(), 10) + "-" + strconv.FormatUint(l.next, 10),
Seat: seat, Verb: verb, Args: args, Started: l.now(), State: CallRunning, reply: reply}
l.calls = append(l.calls, c)
if len(l.calls) > KeptCalls {
l.calls = l.calls[len(l.calls)-KeptCalls:]
}
return c
}
// finish records a call's answer; answeredAlready is its caller having been told it was running.
func (l *CallLog) finish(c *Call, answer []byte, failed, answeredAlready bool) {
l.mu.Lock()
defer l.mu.Unlock()
at := l.now()
c.Finished = &at
c.Failed = failed
c.Answer = append(json.RawMessage(nil), answer...)
if !answeredAlready && len(answer) > keptAnswer {
// Its caller has it; kept only in case the bus refuses it, and a whole declaration a
// hundred times over is memory nobody asked for.
c.Answer, _ = json.Marshal(map[string]any{"not kept": fmt.Sprintf(
"an answer of %d bytes, sent to its caller in full", len(answer))})
}
if answeredAlready {
c.State = CallFinishedAfter
} else {
c.State = CallAnswered
}
}
// Recent is the kept calls, newest first, as copies.
func (l *CallLog) Recent() []Call {
l.mu.Lock()
defer l.mu.Unlock()
out := make([]Call, 0, len(l.calls))
for i := len(l.calls) - 1; i >= 0; i-- {
out = append(out, *l.calls[i])
}
return out
}
// Get is one kept call.
func (l *CallLog) Get(id string) (Call, bool) {
l.mu.Lock()
defer l.mu.Unlock()
for _, c := range l.calls {
if c.ID == id {
return *c, true
}
}
return Call{}, false
}
// kept is what a call's arguments are kept as: each argument by name, a value only when it is short
// and not one that carries settings or a secret — `calls` answers anyone who may call the seat.
func kept(args json.RawMessage) json.RawMessage {
var given map[string]any
if json.Unmarshal(args, &given) != nil {
return nil
}
out := map[string]any{}
for k, v := range given {
s, isString := v.(string)
switch {
case k == "values" || k == "secret":
out[k] = "(given, not kept)"
case isString && len(s) <= 120:
out[k] = s
case isString:
out[k] = s[:120] + "…"
default:
out[k] = v
}
}
body, _ := json.Marshal(out)
return body
}
// refusedPublish is the bus's words for a publish it refused, with the subject.
var refusedPublish = regexp.MustCompile(`Permissions Violation for Publish to "([^"]+)"`)
// Refusal records the bus refusing an answer, from the error the client library hands the
// connection's error handler. It reports whether the error was the refusal of a kept call's answer.
func (l *CallLog) Refusal(err error, logger *log.Logger) bool {
if err == nil {
return false
}
m := refusedPublish.FindStringSubmatch(err.Error())
if m == nil {
return false
}
l.mu.Lock()
var hit *Call
for i := len(l.calls) - 1; i >= 0; i-- {
if c := l.calls[i]; c.reply != "" && c.reply == m[1] {
hit = c
break
}
}
if hit == nil {
l.mu.Unlock()
return false
}
at := l.now()
hit.Refused = fmt.Sprintf("%s: %s", at.Format(time.RFC3339), err)
id, verb, took := hit.ID, hit.Seat+"."+hit.Verb, at.Sub(hit.Started).Round(time.Second)
l.mu.Unlock()
if logger != nil {
// Said in the mesh's words, beside the library's own line: which call, and where its answer is.
logger.Printf("the bus refused the answer to %s (%s, asked %s ago): its caller got no answer. "+
"What it answered is kept under %s. A broker reloading its user list while a call runs "+
"forgets that it may be answered", id, verb, took, id)
}
return true
}
// WatchRefusals chains the connection's error handler so a refused answer is recorded against its
// call. The handler the dialler set — the library's default, which prints — still runs after it.
func (l *CallLog) WatchRefusals(conn *nats.Conn, logger *log.Logger) {
if conn == nil {
return
}
watched.Lock()
defer watched.Unlock()
if watched.conns == nil {
watched.conns = map[*nats.Conn]bool{}
}
if watched.conns[conn] {
return
}
watched.conns[conn] = true
before := conn.ErrorHandler()
conn.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, err error) {
if errors.Is(err, nats.ErrPermissionViolation) {
l.Refusal(err, logger)
}
if before != nil {
before(c, s, err)
}
})
}
var watched struct {
sync.Mutex
conns map[*nats.Conn]bool
}
// answerNow is how a handler asks for its caller to be answered before it goes on (Acknowledge).
type answerNowKey struct{}
// Acknowledge answers the call's caller now that it is running, rather than after AnswerWithin. For
// a handler about to do what makes its answer unsendable: a push sends the bus's own machine first,
// and a broker reloading its user list forgets every answer it was about to permit. Without effect
// outside a served call, and after the first time.
func Acknowledge(ctx context.Context) {
if f, ok := ctx.Value(answerNowKey{}).(func()); ok {
f()
}
}
// running is the interim answer: what a caller reads when the call has not finished.
func running(c *Call, within time.Duration, acknowledged bool, follow string) []byte {
why := fmt.Sprintf("it has not finished in %s", within)
if acknowledged {
why = "it answers before it starts, because what it does can leave the bus unable to carry a later answer"
}
next := "its holder's journal says how it ended."
if follow != "" {
next = fmt.Sprintf("%s with call %q says what came of it.", follow, c.ID)
}
said := fmt.Sprintf("%s.%s is running as %s — %s. This is not a failure, and it may already have "+
"done what was asked: %s", c.Seat, c.Verb, c.ID, why, next)
body, _ := json.Marshal(map[string]any{"result": map[string]any{
"running": true, "call": c.ID, "started": c.Started, "output": said,
}})
return body
}
// serveCall runs one call and answers it once: in full when the handler returns within AnswerWithin
// and has not acknowledged, and otherwise that it is running — its answer then kept, and logged.
func (l *CallLog) serveCall(seat, verb string, args json.RawMessage, reply string, handle ToolHandler,
respond func([]byte) error, logger *log.Logger) {
ctx, cancel := context.WithTimeout(context.Background(), HandlerTimeout)
defer cancel()
if len(args) == 0 {
args = json.RawMessage(`{}`)
}
c := l.begin(seat, verb, kept(args), reply)
acknowledged := make(chan struct{})
var once sync.Once
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
type outcome struct {
body []byte
failed bool
}
done := make(chan outcome, 1)
go func() {
var body []byte
result, err := handle(ctx, args)
failed := err != nil
if err != nil {
body, _ = json.Marshal(map[string]any{"error": err.Error()})
} else if body, err = json.Marshal(map[string]any{"result": result}); err != nil {
failed = true
body, _ = json.Marshal(map[string]any{"error": "the answer could not be written as JSON: " + err.Error()})
}
done <- outcome{body, failed}
}()
say := func(body []byte) {
if err := respond(body); err != nil && logger != nil {
logger.Printf("%s.%s: could not answer %s: %v", seat, verb, c.ID, err)
}
}
timer := time.NewTimer(AnswerWithin)
defer timer.Stop()
select {
case o := <-done:
l.finish(c, o.body, o.failed, false)
say(o.body)
return
case <-acknowledged:
say(running(c, AnswerWithin, true, l.Follow))
case <-timer.C:
say(running(c, AnswerWithin, false, l.Follow))
}
o := <-done
l.finish(c, o.body, o.failed, true)
if logger != nil {
how := "and it succeeded"
if o.failed {
how = "and it answered an error"
}
logger.Printf("%s (%s.%s) finished after %s, after its caller was told it was running, %s — its answer is kept under %s", c.ID,
seat, verb, time.Since(c.Started).Round(time.Second), how, c.ID)
}
}
internal/link/calls_test.go
-174
View File
@@ -1,174 +0,0 @@
package link
import (
"bytes"
"context"
"encoding/json"
"errors"
"log"
"strings"
"sync"
"testing"
"time"
)
// answers collects what a call answered, and fails a second answer: the bus permits one.
type answers struct {
t *testing.T
mu sync.Mutex
got [][]byte
sent chan struct{}
}
func newAnswers(t *testing.T) *answers { return &answers{t: t, sent: make(chan struct{}, 4)} }
func (a *answers) respond(body []byte) error {
a.mu.Lock()
defer a.mu.Unlock()
a.got = append(a.got, body)
if len(a.got) > 1 {
a.t.Errorf("a call was answered %d times; the bus refuses every answer after the first", len(a.got))
}
a.sent <- struct{}{}
return nil
}
func (a *answers) only() map[string]any {
a.mu.Lock()
defer a.mu.Unlock()
if len(a.got) != 1 {
a.t.Fatalf("%d answers, want exactly one", len(a.got))
}
var out map[string]any
if err := json.Unmarshal(a.got[0], &out); err != nil {
a.t.Fatal(err)
}
return out
}
func shortWindow(t *testing.T, d time.Duration) {
t.Helper()
was := AnswerWithin
AnswerWithin = d
t.Cleanup(func() { AnswerWithin = was })
}
// A call that finishes in time answers in full, once, and is kept as answered.
func TestACallThatFinishesInTimeAnswersInFull(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
l.serveCall("mesh-controller", "status", nil, "_INBOX.x.1", func(context.Context, json.RawMessage) (any, error) {
return "all well", nil
}, a.respond, nil)
if got := a.only()["result"]; got != "all well" {
t.Fatalf("answered %v", got)
}
recent := l.Recent()
if len(recent) != 1 || recent[0].State != CallAnswered || string(recent[0].Args) != "{}" {
t.Fatalf("kept %+v", recent)
}
}
// **A call that outlasts AnswerWithin is answered that it is running, with its id** — before its
// caller gives up — and what it finally answered is kept under that id, not dropped (issue 265).
func TestACallThatOutlastsTheWindowSaysItIsRunningAndKeepsItsAnswer(t *testing.T) {
shortWindow(t, 20*time.Millisecond)
l, a := NewCallLog(), newAnswers(t)
var logged bytes.Buffer
release := make(chan struct{})
finished := make(chan struct{})
go func() {
l.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor"}`), "_INBOX.x.2",
func(context.Context, json.RawMessage) (any, error) {
<-release
return "anchor told", nil
}, a.respond, log.New(&logged, "", 0))
close(finished)
}()
<-a.sent
got := a.only()["result"].(map[string]any)
id, _ := got["call"].(string)
if got["running"] != true || id == "" || !strings.Contains(got["output"].(string), id) {
t.Fatalf("the running answer does not name its call: %v", got)
}
if c, _ := l.Get(id); c.State != CallRunning {
t.Fatalf("while it runs it is kept as %q", c.State)
}
close(release)
<-finished
c, ok := l.Get(id)
if !ok || c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "anchor told") {
t.Fatalf("its answer was not kept: %+v", c)
}
if !strings.Contains(logged.String(), id) {
t.Errorf("finishing late was not said: %q", logged.String())
}
}
// A handler that acknowledges is answered then, not when it ends: a push answers before it sends.
func TestAnAcknowledgedCallIsAnsweredBeforeItGoesOn(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
done := make(chan struct{})
go func() {
l.serveCall("mesh-controller", "push", nil, "_INBOX.x.3", func(ctx context.Context, _ json.RawMessage) (any, error) {
Acknowledge(ctx)
Acknowledge(ctx) // a second time is nothing
select {
case <-a.sent: // the caller was answered before the work goes on
case <-time.After(5 * time.Second):
t.Error("acknowledging did not answer the caller")
}
return "sent", nil
}, a.respond, nil)
close(done)
}()
<-done
if got := a.only()["result"].(map[string]any); got["running"] != true {
t.Fatalf("an acknowledged call answered %v", got)
}
if c := l.Recent()[0]; c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "sent") {
t.Fatalf("kept %+v", c)
}
}
// **A refused answer is recorded against its call, in the mesh's words** — not only as the client
// library's line on standard error, which is all there was on 2026-10-05.
func TestARefusedAnswerIsKeptAgainstItsCall(t *testing.T) {
l, a := NewCallLog(), newAnswers(t)
reply := "_INBOX.laptop.node-tools.jJ4DRJFnZYvkPUBKYwUG5v.LNRyztuX"
l.serveCall("mesh-controller", "push", nil, reply, func(context.Context, json.RawMessage) (any, error) {
return "told", nil
}, a.respond, nil)
var logged bytes.Buffer
refusal := errors.New(`nats: permissions violation: Permissions Violation for Publish to "` + reply + `" on connection [838]`)
if !l.Refusal(refusal, log.New(&logged, "", 0)) {
t.Fatal("the refusal of a kept call's answer was not recognised")
}
c := l.Recent()[0]
if c.Refused == "" || !strings.Contains(logged.String(), c.ID) {
t.Fatalf("the refusal is not kept or not said: %+v / %q", c, logged.String())
}
other := errors.New(`nats: permissions violation: Permissions Violation for Publish to "mesh.node.x" on connection [1]`)
if l.Refusal(other, nil) || l.Refusal(errors.New("nats: timeout"), nil) {
t.Error("an unrelated error was taken for a refused answer")
}
}
// What `calls` keeps of the arguments never carries settings or a secret.
func TestACallKeepsNoSettingsOrSecrets(t *testing.T) {
got := string(kept(json.RawMessage(`{"module":"m","values":"{\"token\":\"s3cret\"}","secret":"s3cret"}`)))
if strings.Contains(got, "s3cret") || !strings.Contains(got, `"module":"m"`) {
t.Fatalf("kept %s", got)
}
}
// Only the newest KeptCalls are kept.
func TestTheLogKeepsTheNewest(t *testing.T) {
l := NewCallLog()
for i := 0; i < KeptCalls+5; i++ {
l.begin("s", "v", nil, "")
}
recent := l.Recent()
if len(recent) != KeptCalls || !strings.HasSuffix(recent[0].ID, "-105") || !strings.HasSuffix(recent[KeptCalls-1].ID, "-6") {
t.Fatalf("kept %d, newest %s, oldest %s", len(recent), recent[0].ID, recent[len(recent)-1].ID)
}
}
internal/link/enrolment.go
-19
View File
@@ -410,25 +410,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
if err := e.Inventory.RecordCarried(ctx, report.Node, report.Carried); err != nil {
return false, err
}
// **An account of a declaration the mesh has moved past never replaces the account it keeps**
// (novox/hq issue 267). The machine-facts half of such a report is kept above, whenever it
// arrives; this half is the machine's word on what it did with what it was sent, and the release
// plan reads it as such. A reconcile that held a machine to the older declaration a moment before
// the newer one arrived reported *after* the newer apply's report, and stored last, it read as the
// machine never having applied what it was sent — the plan waited until somebody pushed by hand.
// One row per node means the last write wins, so the order of arrival must not decide it.
if report.Declared != "" {
sent, err := e.Inventory.Outstanding(ctx, report.Node)
if err != nil {
return false, err
}
if Superseded(report.Declared, sent) {
log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
return false, e.Inventory.Seen(ctx, node.ID)
}
}
// **Whether this is news** is the store's answer: it holds the previous report, and a machine
// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
internal/link/heard_test.go
-64
View File
@@ -260,67 +260,3 @@ func TestWhatFiltersAMachineIsKeptFromItsReport(t *testing.T) {
t.Fatalf("the next report did not replace what filters the machine: %+v", again)
}
}
// Measured on the home server (novox/hq issue 267): the mesh sent d2; the machine applied it and
// reported, and a reconcile's report about d1 — made just before d2 arrived — reached the mesh after
// it. Stored last, it read as the machine never having applied d2, and the release plan waited on a
// report it had already been given.
func TestAnAccountOfAnOlderDeclarationDoesNotReplaceTheNewer(t *testing.T) {
inv := inventory.ForTest(t)
ctx := context.Background()
node, err := inv.AddNode(ctx, "home-server")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
t.Fatal(err)
}
heard := link.Enrolment{Inventory: inv}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d2",
Applied: []string{"a", "b"}, Outward: []string{"eth0"}}); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d1",
Applied: []string{"a"}, Outward: []string{"eth1"}}); err != nil {
t.Fatal(err)
}
doing, said, err := inv.DoingOf(ctx, "home-server")
if err != nil || !said {
t.Fatalf("no account kept: %v", err)
}
if doing.Declared != "d2" || doing.Applied != 2 {
t.Fatalf("the older report replaced the account of what was sent: %+v", doing)
}
// What it says about the machine is kept whenever it arrives, as before.
if links, err := inv.OutwardLinksOf(ctx, "home-server"); err != nil || len(links) != 1 || links[0] != "eth1" {
t.Fatalf("what the older report said about the machine was not kept: %v %v", links, err)
}
}
// The account of the declaration that is outstanding replaces whatever was kept, and so does one
// from a machine nothing was ever recorded as sent to.
func TestAnAccountOfTheSentDeclarationReplacesTheKeptOne(t *testing.T) {
inv := inventory.ForTest(t)
ctx := context.Background()
node, err := inv.AddNode(ctx, "home-server")
if err != nil {
t.Fatal(err)
}
heard := link.Enrolment{Inventory: inv}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d1", Applied: []string{"a"}}); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d2", Applied: []string{"a", "b"}}); err != nil {
t.Fatal(err)
}
doing, _, err := inv.DoingOf(ctx, "home-server")
if err != nil {
t.Fatal(err)
}
if doing.Declared != "d2" || doing.Applied != 2 {
t.Fatalf("the account of what was sent was not kept: %+v", doing)
}
}
internal/link/merges.go
-79
View File
@@ -1,79 +0,0 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// AnnouncedMerge is one merge the forge announced, as the events stream holds it: what it said, and
// when the bus took it.
type AnnouncedMerge struct {
SourceMoved
// At is when the bus took the announcement — the stream's own time, not the forge's.
At time.Time
// Seq is its place in the events stream.
Seq uint64
}
// MergedSubject is where the forge's merges land: the controller's own follow of them.
var MergedSubject = broker.ControllerFollows[3]
// readQuiet is how long a read of the stream waits for one more message before it takes the stream
// as read to its end. The stream answers at once when it holds something; this is only the wait at
// the end.
const readQuiet = 2 * time.Second
// AnnouncedMerges is every merge the forge announced since a moment, oldest first, read back from
// the events stream (novox/hq issue 266).
//
// **Read on a consumer of its own, filtered on the one subject.** The controller's durable consumer
// carries several filters, and the bus server the mesh ran when this was written (2.10) skips a
// message now and then on a consumer with more than one filter: it moves its delivered pointer past
// the message without ever handing it over, so the controller never hears of it and nothing says so.
// A consumer filtered on a single subject reads the stream another way and was not seen to skip. This
// one is ordered, ephemeral and acknowledges nothing, so reading it changes nothing on the bus.
func (s *Server) AnnouncedMerges(ctx context.Context, since time.Time) ([]AnnouncedMerge, error) {
if s.js == nil {
return nil, errors.New("this control plane is not on the bus, so it cannot read what the forge announced")
}
sub, err := s.js.Context().SubscribeSync(MergedSubject, nats.OrderedConsumer(), nats.StartTime(since))
if err != nil {
return nil, fmt.Errorf("reading the forge's merges from the events stream: %w", err)
}
defer func() { _ = sub.Unsubscribe() }()
var out []AnnouncedMerge
for {
wait, cancel := context.WithTimeout(ctx, readQuiet)
msg, err := sub.NextMsgWithContext(wait)
cancel()
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
// Nothing more within the quiet wait: the stream has been read to its end.
break
}
meta, err := msg.Metadata()
if err != nil {
continue
}
var moved SourceMoved
if json.Unmarshal(msg.Data, &moved) == nil && moved.Commit != "" && moved.Repo != "" {
out = append(out, AnnouncedMerge{SourceMoved: moved, At: meta.Timestamp, Seq: meta.Sequence.Stream})
}
if meta.NumPending == 0 {
break
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Seq < out[j].Seq })
return out, nil
}
internal/link/receive.go
-8
View File
@@ -34,9 +34,6 @@ const (
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
KindSourceMoved = "source-moved"
KindCatchUp = "catch-up"
// KindProvisioner is a provider saying a consumer has failed for minutes, or recovered
// (novox/hq ADR 0224).
KindProvisioner = "provisioner"
)
// Control is one thing a node or a module said, as the controller must act on it.
@@ -57,11 +54,6 @@ type Control interface {
// Body is the message itself — the payload alone, never the envelope.
Body() []byte
// Subject is where it was published. For a module's event it names the emitter, which the bus
// enforces (only a module may publish into its own namespace), so who said it is read from here
// and never from the body.
Subject() string
// Redelivered says the bus has handed this message over before. An enrolment cares and
// nothing else does: one already spent is not finished a second time.
Redelivered() bool
internal/link/receive_fake_test.go
-2
View File
@@ -61,7 +61,6 @@ func (c *fakeInbound) retries(ctx context.Context, s *Server) {
type fakeControl struct {
kind string
subject string
body []byte
tag uint64
to *settled
@@ -72,7 +71,6 @@ type fakeControl struct {
func (m *fakeControl) Kind() string { return m.kind }
func (m *fakeControl) Body() []byte { return m.body }
func (m *fakeControl) Subject() string { return m.subject }
func (m *fakeControl) Redelivered() bool { return m.redelivered }
func (m *fakeControl) About(string) {}
func (m *fakeControl) Answer(context.Context, []byte) error { return nil }
internal/link/receive_nats.go
+3 -25
View File
@@ -53,7 +53,7 @@ func Nats(js *broker.JetStream) Inbound {
// whatever was asked for — and not at all when nothing was.
func (n *natsInbound) Also(kind string) error {
switch kind {
case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner:
case KindModuleMoved, KindCatchUp, KindSourceMoved:
n.follows[kind] = true
return nil
default:
@@ -241,30 +241,9 @@ func kindOfSubject(subject string) (string, bool) {
// runs (ADR 0190): the old builder still answers on the retired seat until it is unassigned.
return KindBuilt, true
}
if _, ok := ProvisionerEmitter(subject); ok {
return KindProvisioner, true
}
return "", false
}
// ProvisionerEmitter is the module a provider's standing event came from, read from its subject
// (`mesh.mod.<module>.event.provisioner.<failing|recovered>`); false for any other subject. The
// controller's own follow pattern, with `*` for the module, decodes too.
func ProvisionerEmitter(subject string) (string, bool) {
rest, ok := strings.CutPrefix(subject, "mesh.mod.")
if !ok {
return "", false
}
module, event, ok := strings.Cut(rest, ".event.")
if !ok || module == "" || strings.Contains(module, ".") {
return "", false
}
if event != broker.ProvisionerFailing && event != broker.ProvisionerRecovered {
return "", false
}
return module, true
}
// natsControl is one message from the bus being built, as the controller reads it.
type natsControl struct {
kind string
@@ -277,9 +256,8 @@ type natsControl struct {
delivered uint64
}
func (m *natsControl) Kind() string { return m.kind }
func (m *natsControl) Body() []byte { return m.msg.Data }
func (m *natsControl) Subject() string { return m.msg.Subject }
func (m *natsControl) Kind() string { return m.kind }
func (m *natsControl) Body() []byte { return m.msg.Data }
// Redelivered is what the server counted, not what the controller remembers. Which is the answer to
// a question the AMQP side could only guess at across a restart: an enrolment redelivered because
internal/link/seattools.go
+21 -7
View File
@@ -25,8 +25,8 @@ type ToolHandler func(ctx context.Context, args json.RawMessage) (any, error)
// SeatToolSubject is where a mesh-scoped seat's tool is asked (design 33 §4).
func SeatToolSubject(seat, verb string) string { return "mesh.seat." + seat + ".tool." + verb }
// HandlerTimeout bounds one call. Its caller is answered within AnswerWithin either way; this is how
// long the call itself may run before it is stopped.
// HandlerTimeout bounds one answer. A verb that runs a command — a push, a build with no wait —
// answers in seconds; anything that has not in this long is said to have not answered.
const HandlerTimeout = 5 * time.Minute
// RebindAfter is how long a refused subscription waits before it is tried again.
@@ -62,17 +62,31 @@ func (b OverNATS) serveTools(seat string, subjectOf func(string) string, handler
_ = s.Unsubscribe()
}
}
// A refused answer is recorded against its call, not only printed by the library.
Calls.WatchRefusals(b.Conn, logger)
for verb, handle := range handlers {
verb, handle := verb, handle
subject := subjectOf(verb)
bind := func() (*nats.Subscription, error) {
return b.Conn.QueueSubscribe(subject, "seat."+seat, func(msg *nats.Msg) {
// Its own goroutine per call: a slow `push` must not hold up a `status` asked beside it,
// and the library would otherwise run handlers one after another. Answered once, within
// AnswerWithin, and kept (novox/hq issue 265).
go Calls.serveCall(seat, verb, json.RawMessage(msg.Data), msg.Reply, handle, msg.Respond, logger)
// and the library would otherwise run handlers one after another.
go func() {
ctx, cancel := context.WithTimeout(context.Background(), HandlerTimeout)
defer cancel()
args := json.RawMessage(msg.Data)
if len(args) == 0 {
args = json.RawMessage(`{}`)
}
var reply []byte
result, err := handle(ctx, args)
if err != nil {
reply, _ = json.Marshal(map[string]any{"error": err.Error()})
} else if reply, err = json.Marshal(map[string]any{"result": result}); err != nil {
reply, _ = json.Marshal(map[string]any{"error": "the answer could not be written as JSON: " + err.Error()})
}
if err := msg.Respond(reply); err != nil && logger != nil {
logger.Printf("%s: could not answer: %v", subject, err)
}
}()
})
}
sub, err := bind()
internal/link/serve.go
-7
View File
@@ -79,8 +79,6 @@ type Server struct {
recorder Recorder
upgrader Upgrader
replayer Replayer
// standings keeps what providers say about their consumers (novox/hq ADR 0224).
standings Standings
log *log.Logger
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
@@ -155,9 +153,6 @@ func (s *Server) Serve(ctx context.Context) error {
if s.replayer != nil {
s.log.Printf("answering %s", KindCatchUp)
}
if s.standings != nil {
s.log.Printf("keeping every provider's %s", KindProvisioner)
}
return s.inbound.Receive(ctx, s.act)
}
@@ -178,8 +173,6 @@ func (s *Server) act(ctx context.Context, m Control) {
s.sourceMoved(ctx, m)
case KindCatchUp:
s.catchingUp(ctx, m)
case KindProvisioner:
s.provisioner(ctx, m)
default:
// Dropped: a message nothing understands will not be understood on the next attempt
// either, and asking for it again would spin.
internal/link/standing.go
-114
View File
@@ -1,114 +0,0 @@
package link
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
)
// A provider's standing (novox/hq ADR 0224).
//
// **A provider that keeps failing a consumer is a problem the controller reports**, not a line in a
// journal. On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a
// day — its admin no longer took the mesh's secret once its database was moved — and every surface
// the mesh has called the mesh well (novox/hq issue 179). A provider now says, as an event, a
// consumer it has failed for minutes without one success, and the consumer recovering; the
// controller keeps the newest word per provider, machine and consumer, and `status` names each one
// still failing.
// Standing is one provider's word about one consumer.
type Standing struct {
// Module is the emitter, read from the subject the bus let it publish on — never from the body.
Module string `json:"-"`
// Failing is which of the two it said: failing, or recovered.
Failing bool `json:"-"`
Provider string `json:"provider"`
ProviderNode string `json:"provider-node"`
Consumer string `json:"consumer"`
Node string `json:"node"`
Class string `json:"class,omitempty"`
Error string `json:"error,omitempty"`
Since time.Time `json:"since"`
Attempts int `json:"attempts"`
// Why is said with a recovery that is not a success: `withdrawn`, a consumer no longer asked for.
Why string `json:"why,omitempty"`
}
// Standings keeps what providers say about their consumers.
type Standings interface {
// Stood records a provider's newest word about a consumer: a failing one kept, a recovered one
// cleared — and says whether a recovery cleared anything, since a provider announces its first
// success for every consumer after it starts. An error the store is away for is held and asked
// again, like a report.
Stood(ctx context.Context, s Standing) (cleared bool, err error)
}
// Watches says where providers' standings are kept, and asks for them to be delivered.
func (s *Server) Watches(st Standings) error {
if err := s.inbound.Also(KindProvisioner); err != nil {
return err
}
s.standings = st
return nil
}
// ReadStanding is one standing event as the controller understands it, from its subject and body.
func ReadStanding(subject string, body []byte) (Standing, error) {
module, ok := ProvisionerEmitter(subject)
if !ok {
return Standing{}, fmt.Errorf("%s is not a provider's standing", subject)
}
var st Standing
if err := json.Unmarshal(body, &st); err != nil {
return Standing{}, fmt.Errorf("%s's standing could not be read: %w", module, err)
}
if st.Consumer == "" {
return Standing{}, fmt.Errorf("%s's standing named no consumer", module)
}
st.Module = module
st.Failing = strings.HasSuffix(subject, "."+broker.ProvisionerFailing)
return st, nil
}
// provisioner acts on one standing event.
//
// **A recovery must not be lost.** A failing standing is said again every quarter of an hour while
// it lasts, so one dropped is replaced; a recovery is said once, and dropping it would leave status
// naming a consumer that is fine. So a store that is away holds the message, as a report is held.
func (s *Server) provisioner(ctx context.Context, m Control) {
if s.standings == nil {
// Delivered because the consumer's filter names it, with nothing here keeping it: taken,
// because handing it back would not give it anywhere to go.
_ = m.Took()
return
}
st, err := ReadStanding(m.Subject(), m.Body())
if err != nil {
s.log.Printf("%v; ignored", err)
_ = m.Took()
return
}
cleared, err := s.standings.Stood(ctx, st)
what := fmt.Sprintf("%s's standing for %s", st.Module, st.Consumer)
switch s.decide(ctx, m, what, "", "", err) {
case Hold:
return
case Stale, GiveUp:
_ = m.Took()
return
}
if err != nil {
s.log.Printf("%s could not be kept: %v", what, err)
} else if st.Failing {
s.log.Printf("%s on %s is FAILING %s on %s (%s, %d attempts since %s): %s", st.Module,
st.ProviderNode, st.Consumer, st.Node, st.Class, st.Attempts, st.Since.Format(time.RFC3339), st.Error)
} else if cleared {
s.log.Printf("%s on %s recovered %s", st.Module, st.ProviderNode, st.Consumer)
}
_ = m.Took()
}
internal/link/standing_test.go
-203
View File
@@ -1,203 +0,0 @@
package link
import (
"context"
"sync"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
)
// A provider's standing (novox/hq ADR 0224): who said it is read from the subject the bus let it
// publish on, a failing one is kept and a recovery cleared, and a recovery is never lost to a store
// that is away — said once, it would leave status naming a consumer that is fine.
type keptStandings struct {
kept []Standing
err error
}
func (k *keptStandings) Stood(_ context.Context, s Standing) (bool, error) {
if k.err != nil {
return false, k.err
}
k.kept = append(k.kept, s)
return !s.Failing, nil
}
func standingSays(t *testing.T, in *fakeInbound, to *settled, subject string, body map[string]any) Control {
t.Helper()
m := in.sends(t, to, KindProvisioner, body).(*fakeControl)
m.subject = subject
return m
}
func TestTheControllerFollowsEveryProvidersStandingAndNothingElse(t *testing.T) {
for subject, want := range map[string]string{
"mesh.mod.keycloak.event.provisioner.failing": "keycloak",
"mesh.mod.postgres.event.provisioner.recovered": "postgres",
"mesh.mod.*.event.provisioner.failing": "*",
} {
got, ok := ProvisionerEmitter(subject)
if !ok || got != want {
t.Errorf("%s: %q %v", subject, got, ok)
}
if kind, _ := kindOfSubject(subject); kind != KindProvisioner {
t.Errorf("%s decodes to %q", subject, kind)
}
}
for _, subject := range []string{
"mesh.mod.keycloak.event.provisioner.other",
"mesh.mod.keycloak.event.client.created",
"mesh.mod.a.b.event.provisioner.failing",
"mesh.seat.keycloak.event.provisioner.failing",
} {
if _, ok := ProvisionerEmitter(subject); ok {
t.Errorf("%s read as a provider's standing", subject)
}
}
var follows int
for _, s := range broker.ControllerFollows {
if kind, _ := kindOfSubject(s); kind == KindProvisioner {
follows++
}
}
if follows != 2 {
t.Fatalf("the controller follows %d standing subjects, want failing and recovered", follows)
}
}
func TestAFailingStandingIsKeptNamingTheEmitterFromTheSubject(t *testing.T) {
s, in := serving()
kept := &keptStandings{}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
to := &settled{}
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing", map[string]any{
"provider": "oidc-client", "provider-node": "anchor", "consumer": "mesh_home_grafana",
"node": "home-server", "class": "credentials-rejected", "error": "401 invalid_grant",
"since": since, "attempts": 31000,
// A body naming another module is not believed: the subject is the bus's word.
"module": "postgres",
}))
if !to.acked || len(kept.kept) != 1 {
t.Fatalf("settled %+v, kept %+v", to, kept.kept)
}
got := kept.kept[0]
if got.Module != "keycloak" || !got.Failing || got.Consumer != "mesh_home_grafana" || got.Node != "home-server" ||
got.ProviderNode != "anchor" || got.Class != "credentials-rejected" || got.Attempts != 31000 || !got.Since.Equal(since) {
t.Fatalf("%+v", got)
}
to = &settled{}
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.recovered", map[string]any{
"provider": "oidc-client", "provider-node": "anchor", "consumer": "mesh_home_grafana",
}))
if !to.acked || len(kept.kept) != 2 || kept.kept[1].Failing {
t.Fatalf("settled %+v, kept %+v", to, kept.kept)
}
}
func TestARecoveryIsHeldWhileTheStoreIsAway(t *testing.T) {
s, in := serving()
kept := &keptStandings{err: restarting}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
to := &settled{}
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.recovered",
map[string]any{"consumer": "mesh_home_grafana"}))
if !to.unsettled() || len(in.held) != 1 {
t.Fatalf("a recovery was settled while the store was away: %+v", to)
}
kept.err = nil
in.retries(t.Context(), s)
if !to.acked || len(kept.kept) != 1 {
t.Fatalf("the held recovery was not kept when the store came back: %+v %+v", to, kept.kept)
}
}
func TestAStandingThatNamesNoConsumerIsTakenAndForgotten(t *testing.T) {
s, in := serving()
kept := &keptStandings{}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
to := &settled{}
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing", map[string]any{}))
if !to.acked || len(kept.kept) != 0 {
t.Fatalf("%+v %+v", to, kept.kept)
}
}
func TestAStandingWithNothingKeepingItIsTaken(t *testing.T) {
s, in := serving()
to := &settled{}
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing",
map[string]any{"consumer": "x"}))
if !to.acked {
t.Fatal("a standing nothing keeps was left for the bus to hand over again")
}
}
// Over a real bus: a provider's standing published under its own module's namespace reaches the
// controller through the events consumer's filter — the one wildcard filter on it — names the emitter
// from the subject, and is acknowledged.
func TestNatsAProvidersStandingReachesTheController(t *testing.T) {
js := aBus(t)
kept := &lockedStandings{}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(&toldAbout{}); err != nil {
t.Fatal(err)
}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
eventually(t, "the controller's event consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil
})
for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} {
if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event,
[]byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil {
t.Fatal(err)
}
}
// Somebody else's event under the same prefix is not the controller's to hear.
if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil {
t.Fatal(err)
}
eventually(t, "both standings being kept, in order, naming the emitter", func() bool {
got := kept.all()
return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing
})
eventually(t, "both being acknowledged and nothing else delivered", func() bool {
info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2
})
}
type lockedStandings struct {
mu sync.Mutex
kept []Standing
}
func (l *lockedStandings) Stood(_ context.Context, s Standing) (bool, error) {
l.mu.Lock()
defer l.mu.Unlock()
l.kept = append(l.kept, s)
return !s.Failing, nil
}
func (l *lockedStandings) all() []Standing {
l.mu.Lock()
defer l.mu.Unlock()
return append([]Standing(nil), l.kept...)
}
internal/overlay/generator.go
+2 -2
View File
@@ -32,8 +32,8 @@ const Requirement = "private-network"
// Name is the module that answers it with WireGuard.
//
// **It writes no names.** A machine's mesh names are answered by the mesh's one resolver (novox/hq
// ADR 0194), and /etc/hosts is the file of one module, the holder of `node-hostname` (ADR 0199,
// ADR 0223): the controller writes into no file another seat's holder owns. If the mesh ever needs a line there, it
// ADR 0194), and /etc/hosts is the file of one module, the holder of `node-hosts-file` (ADR 0199): the
// controller writes into no file another seat's holder owns. If the mesh ever needs a line there, it
// asks that holder to register it. This module asked for a `node-names` fact written into /etc/hosts
// until 2026-10-05; the host gives that region back at the first push without it.
const Name = "mesh-wireguard"