From 1c32af6a22d68e3324fc57504ce91bd7bc51e73d Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:14:05 +0200 Subject: [PATCH 01/33] Record whether a node is adopted and which modules were taken on it (hq ADR 0100) --- cmd/mesh-controller/adoption.go | 44 ++++++ cmd/mesh-controller/board.go | 9 +- cmd/mesh-controller/main.go | 3 +- cmd/mesh-controller/nodes.go | 97 +++++++++--- cmd/mesh-controller/nodes_test.go | 50 ++++++ cmd/mesh-controller/readable.go | 4 +- cmd/mesh-controller/status.go | 7 + internal/inventory/adoption.go | 149 ++++++++++++++++++ internal/inventory/adoption_test.go | 134 ++++++++++++++++ .../0029-a-node-is-adopted-or-converged.sql | 21 +++ internal/inventory/nodes.go | 66 +++++--- internal/token/token.go | 5 + internal/token/token_test.go | 16 ++ 13 files changed, 559 insertions(+), 46 deletions(-) create mode 100644 cmd/mesh-controller/adoption.go create mode 100644 internal/inventory/adoption.go create mode 100644 internal/inventory/adoption_test.go create mode 100644 internal/inventory/migrations/0029-a-node-is-adopted-or-converged.sql diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go new file mode 100644 index 0000000..e898dbf --- /dev/null +++ b/cmd/mesh-controller/adoption.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// A node is adopted or converged (novox/hq ADR 0100), and it is said to be adopted wherever the +// mesh reports a node's state: node list, node show, status and the board. + +// showMode is the node show lines about a node's mode and what was taken on it. +func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node) error { + if !node.Adopted { + fmt.Printf(" mode converged\n") + return nil + } + fmt.Printf(" mode adopted since %s\n", + node.AdoptedSince.Local().Format(time.DateTime)) + taken, err := inv.Taken(ctx, node.Name) + if err != nil { + return err + } + if len(taken) == 0 { + fmt.Printf(" taken nothing yet\n") + } else { + fmt.Printf(" taken %s\n", strings.Join(taken, ", ")) + } + return nil +} + +// adoptedNodes are the names of every adopted node, in the order given. +func adoptedNodes(nodes []inventory.Node) []string { + var out []string + for _, n := range nodes { + if n.Adopted { + out = append(out, n.Name) + } + } + return out +} diff --git a/cmd/mesh-controller/board.go b/cmd/mesh-controller/board.go index 49702ac..3557b8d 100644 --- a/cmd/mesh-controller/board.go +++ b/cmd/mesh-controller/board.go @@ -137,6 +137,9 @@ type view struct { Unresolved []blockedMachine // Network is why the private network could not be computed, when it could not. Network string + // Adopted is every node still adopted (novox/hq ADR 0100). Not broken: nothing forces the + // flip, so a node left adopted is shown rather than read as converged. + Adopted []string At string } @@ -181,7 +184,7 @@ type staleModule struct { func viewOf(asked answers) view { out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"), - Network: asked.network} + Network: asked.network, Adopted: adoptedNodes(asked.nodes)} var blocked []string for name := range asked.refused { blocked = append(blocked, name) @@ -311,6 +314,10 @@ new, switched off, or unreachable.

Never told is not out of date: nobody has asked that machine to be anything yet. Both are sent by push --behind.

{{else}}

Every machine is running what the mesh would send it.

{{end}} +{{if .Adopted}} +

Which machines are adopted?

+ +{{end}} {{end}} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index f03cd59..00cc99b 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -126,7 +126,7 @@ func usage() { fmt.Fprint(os.Stderr, `mesh-controller — the control plane migrate bring each context's schema up to date - node add create a node record + node add [--adopted] create a node record; --adopted: the machine is in use node list the nodes this mesh knows about node show what one machine reported it can do, and why node public-domain the domain it composes its routed names under @@ -134,6 +134,7 @@ func usage() { node public-domain --clear ...it faces the outside no longer token issue --node a one-time right to join, for an existing record token issue --new create the record and issue for it + token issue ... --adopted ...for a machine in use, which joins adopted identity show this control plane's signing key broker show where the broker is, and what to expect there serve consume what nodes say, and answer diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index e6dcfe0..1ca2257 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -38,15 +38,7 @@ func nodeCommand(ctx context.Context, args []string) error { } return showNode(ctx, inv, args[1]) case "add": - if len(args) != 2 { - return errors.New("node add ") - } - node, err := inv.AddNode(ctx, args[1]) - if err != nil { - return err - } - fmt.Printf("added %s (%s)\n", node.Name, node.ID) - return nil + return addNode(ctx, inv, args[1:]) case "list": nodes, err := inv.Nodes(ctx) @@ -61,7 +53,7 @@ func nodeCommand(ctx context.Context, args []string) error { return nil } for _, n := range nodes { - fmt.Printf("%-20s %-14s %s\n", n.Name, heardFrom(n), n.ID) + fmt.Printf("%-20s %-14s %-9s %s\n", n.Name, heardFrom(n), modeOf(n), n.ID) } return nil @@ -80,6 +72,39 @@ func nodeCommand(ctx context.Context, args []string) error { } } +// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100). +func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error { + set := flag.NewFlagSet("node add", flag.ContinueOnError) + adopted := set.Bool("adopted", false, + "the machine is in use: keep what is found on it until each module is taken") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 1 { + return errors.New("node add [--adopted]") + } + node, err := inv.AddNodeAs(ctx, positionals[0], *adopted) + if err != nil { + return err + } + fmt.Printf("added %s (%s)", node.Name, node.ID) + if node.Adopted { + fmt.Print(", adopted") + } + fmt.Println() + return nil +} + +// modeOf is a node's mode as a word (novox/hq ADR 0100): an adopted node is said to be adopted +// wherever the mesh reports a node's state. +func modeOf(n inventory.Node) string { + if n.Adopted { + return "adopted" + } + return "converged" +} + // publicDomainUsage is the one description of the three forms, so a refusal and the help agree. const publicDomainUsage = "node public-domain — what it is now; " + " to set it; --clear to take it away" @@ -153,6 +178,8 @@ func tokenCommand(ctx context.Context, args []string) error { existing := set.String("node", "", "issue for a node record that already exists") fresh := set.String("new", "", "create the node record, then issue for it") validFor := set.Duration("for", time.Hour, "how long the token may be used") + adopted := set.Bool("adopted", false, + "the machine joining is in use: it is adopted, and keeps what is found on it") if err := set.Parse(args[1:]); err != nil { return err } @@ -171,16 +198,7 @@ func tokenCommand(ctx context.Context, args []string) error { defer open.Close() inv := open.inventory - name := *existing - if *fresh != "" { - node, err := inv.AddNode(ctx, *fresh) - if err != nil { - return err - } - name = node.Name - } - - issued, err := inv.IssueToken(ctx, name, *validFor) + issued, err := issueFor(ctx, inv, *existing, *fresh, *adopted, *validFor) if err != nil { return err } @@ -211,7 +229,8 @@ func tokenCommand(ctx context.Context, args []string) error { return err } - made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret} + made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret, + Adopted: issued.Node.Adopted} // Absent is a state, not a failure: a control plane can hold records and a key before it has // a broker. What it cannot do is issue a token anybody could use, and Missing() says so. @@ -228,8 +247,12 @@ func tokenCommand(ctx context.Context, args []string) error { return err } - fmt.Printf("token for %s, usable once, until %s\n\n %s\n\n", - issued.Node.Name, issued.Expires.Format(time.RFC3339), encoded) + joins := "" + if made.Adopted { + joins = ", joining adopted" + } + fmt.Printf("token for %s%s, usable once, until %s\n\n %s\n\n", + issued.Node.Name, joins, issued.Expires.Format(time.RFC3339), encoded) fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.") if missing := made.Missing(); len(missing) > 0 { @@ -243,6 +266,31 @@ func tokenCommand(ctx context.Context, args []string) error { return nil } +// issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted +// when the operator says so, and the one-time secret for it. The node in what it returns carries +// its mode, which is what the token says. +func issueFor(ctx context.Context, inv *inventory.Inventory, existing, fresh string, adopted bool, + validFor time.Duration) (inventory.Issued, error) { + name := existing + if fresh != "" { + node, err := inv.AddNodeAs(ctx, fresh, adopted) + if err != nil { + return inventory.Issued{}, err + } + name = node.Name + } + // Saying adopted makes the node adopted. Not saying it leaves the node as it is: re-issuing a + // token for an adopted node does not converge it — converging is its own act, previewed + // (novox/hq ADR 0100). + if adopted { + if err := inv.SetAdopted(ctx, name, true); err != nil { + return inventory.Issued{}, err + } + } + + return inv.IssueToken(ctx, name, validFor) +} + func identityCommand(ctx context.Context, args []string) error { if len(args) == 0 || args[0] != "show" { return errors.New("identity show") @@ -334,6 +382,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error } fmt.Printf("%s\n", node.Name) fmt.Printf(" last heard from %s\n", heardFrom(node)) + if err := showMode(ctx, inv, node); err != nil { + return err + } // The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown // only when set: a machine that serves nothing to the outside has no domain, and saying so of diff --git a/cmd/mesh-controller/nodes_test.go b/cmd/mesh-controller/nodes_test.go index a27d3ba..c9f55a7 100644 --- a/cmd/mesh-controller/nodes_test.go +++ b/cmd/mesh-controller/nodes_test.go @@ -3,6 +3,7 @@ package main import ( "strings" "testing" + "time" ) // **A read-shaped invocation is never a destructive write.** @@ -97,3 +98,52 @@ func TestAskingAboutAMachineTheMeshHasNeverHeardOfIsRefused(t *testing.T) { t.Fatal("a name the mesh does not know was answered as if it were a machine") } } + +// novox/hq ADR 0100: the operator says a node is adopted — `node add --adopted` for a record, and +// the token for a machine joining. +func TestANodeAddedAdoptedIsAdopted(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + if err := addNode(ctx, open.inventory, []string{"joiner", "--adopted"}); err != nil { + t.Fatal(err) + } + n, err := open.inventory.NodeByName(ctx, "joiner") + if err != nil { + t.Fatal(err) + } + if !n.Adopted { + t.Fatal("node add --adopted made a converged node") + } + if err := addNode(ctx, open.inventory, []string{"plain"}); err != nil { + t.Fatal(err) + } + if n, _ := open.inventory.NodeByName(ctx, "plain"); n.Adopted { + t.Fatal("node add without --adopted made an adopted node") + } +} + +func TestATokenIssuedAdoptedSaysSoAndReissuingDoesNotConverge(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + issued, err := issueFor(ctx, open.inventory, "", "joiner", true, time.Hour) + if err != nil { + t.Fatal(err) + } + if !issued.Node.Adopted { + t.Fatal("a token issued --adopted is for a node that is not adopted") + } + again, err := issueFor(ctx, open.inventory, "joiner", "", false, time.Hour) + if err != nil { + t.Fatal(err) + } + if !again.Node.Adopted { + t.Fatal("re-issuing without --adopted converged the node; converging is its own act") + } + existing, err := issueFor(ctx, open.inventory, "laptop", "", true, time.Hour) + if err != nil { + t.Fatal(err) + } + if !existing.Node.Adopted { + t.Fatal("--adopted on an existing record did not make it adopted") + } +} diff --git a/cmd/mesh-controller/readable.go b/cmd/mesh-controller/readable.go index 891bfcd..0bda5b5 100644 --- a/cmd/mesh-controller/readable.go +++ b/cmd/mesh-controller/readable.go @@ -54,6 +54,8 @@ type meshStatus struct { // Machines is how many the mesh knows about, so a reader can tell "none wrong" from // "none at all". Machines int `json:"machines"` + // Adopted is every node still adopted (novox/hq ADR 0100); absent when none is. + Adopted []string `json:"adopted,omitempty"` } type machineUnresolved struct { @@ -133,7 +135,7 @@ func statusAsJSON(asked answers) ([]byte, error) { out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{}, Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{}, Reported: []machineReported{}, Unresolved: []machineUnresolved{}, - Network: asked.network} + Network: asked.network, Adopted: adoptedNodes(nodes)} for name := range asked.refused { out.Unresolved = append(out.Unresolved, machineUnresolved{ Node: name, Problem: asked.refused[name]}) diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index cdc7bc7..e5bbae9 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -171,6 +171,13 @@ func statusCommand(ctx context.Context, args []string) error { fmt.Printf("\n `push --behind` sends them\n\n") } + if adopted := adoptedNodes(nodes); len(adopted) > 0 { + // Said, because nothing forces the flip: a node left adopted is visible here rather than + // read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well". + fmt.Printf("%d machine(s) adopted: %s\n", len(adopted), strings.Join(adopted, ", ")) + fmt.Printf("\n `converge ` previews the flip\n\n") + } + if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 && len(asked.refused) == 0 && asked.network == "" { // Said plainly. "Nothing to report" and "nothing was checked" must never look the same, diff --git a/internal/inventory/adoption.go b/internal/inventory/adoption.go new file mode 100644 index 0000000..031bf52 --- /dev/null +++ b/internal/inventory/adoption.go @@ -0,0 +1,149 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + "sort" +) + +// A node is adopted or converged (novox/hq ADR 0100). +// +// Adopted: what is found on the machine is kept until its module is taken, and the firewall found +// there stays in force. Converged: the machine is what the mesh declares, as every node was before +// adoption existed. The controller is authoritative, and every declaration it sends says which. + +// ErrNotAdopted is taking a module on a node that is converged. On a converged node every assigned +// module converges already; there is nothing to take. +var ErrNotAdopted = errors.New("the node is converged, so every module on it is taken already") + +// ErrNotAssigned is taking a module that is not on the node. Taking is the cutover of a module +// the node runs; one it does not run has nothing to cut over. +var ErrNotAssigned = errors.New("that module is not assigned to the node") + +// SetAdopted makes a node adopted or converged. Becoming adopted stamps when; converging stamps +// when too. Neither touches what was taken: what was taken stays taken when a node returns to +// adopted, and converging takes the rest by its own act. +func (i *Inventory) SetAdopted(ctx context.Context, name string, adopted bool) error { + node, err := i.NodeByName(ctx, name) + if err != nil { + return err + } + if node.Adopted == adopted { + return nil + } + if adopted { + _, err = i.store.Pool().Exec(ctx, + `update node set adopted = true, adopted_since = now() where id = $1`, node.ID) + return err + } + _, err = i.store.Pool().Exec(ctx, + `update node set adopted = false, adopted_since = null, converged_at = now() where id = $1`, + node.ID) + return err +} + +// Take records that a module has been taken on an adopted node: its cutover. From then on the +// module's resources converge on that node like any other, replacing what was found. +// +// Refused on a converged node and for a module not assigned there. Taking again is not an error; +// the first time it was taken is kept. +func (i *Inventory) Take(ctx context.Context, nodeName, module string) error { + node, err := i.NodeByName(ctx, nodeName) + if err != nil { + return err + } + if !node.Adopted { + return fmt.Errorf("%w: %s", ErrNotAdopted, nodeName) + } + return i.take(ctx, node, module) +} + +// take is Take without the adopted check, for converging, which takes every assigned module in +// the same act that makes the node converged. +func (i *Inventory) take(ctx context.Context, node Node, module string) error { + var assigned bool + if err := i.store.Pool().QueryRow(ctx, + `select exists (select 1 from assignment where node = $1 and module = $2)`, + node.ID, module).Scan(&assigned); err != nil { + return err + } + if !assigned { + return fmt.Errorf("%w: %s is not on %s; assign it first", ErrNotAssigned, module, node.Name) + } + _, err := i.store.Pool().Exec(ctx, + `insert into taken (node, module) values ($1, $2) on conflict do nothing`, node.ID, module) + return err +} + +// Converge makes an adopted node converged in one act: every module assigned there is taken, and +// the node is recorded converged. Returned is what this act took, in name order. +func (i *Inventory) Converge(ctx context.Context, nodeName string) ([]string, error) { + node, err := i.NodeByName(ctx, nodeName) + if err != nil { + return nil, err + } + if !node.Adopted { + return nil, fmt.Errorf("%s is converged already", nodeName) + } + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + rows, err := tx.Query(ctx, + `insert into taken (node, module) + select node, module from assignment where node = $1 + on conflict do nothing + returning module`, node.ID) + if err != nil { + return nil, err + } + var took []string + for rows.Next() { + var m string + if err := rows.Scan(&m); err != nil { + rows.Close() + return nil, err + } + took = append(took, m) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + if _, err := tx.Exec(ctx, + `update node set adopted = false, adopted_since = null, converged_at = now() where id = $1`, + node.ID); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, err + } + sort.Strings(took) + return took, nil +} + +// Taken is every module taken on a node, in name order — including one no longer assigned there: +// unassigning does not un-take. +func (i *Inventory) Taken(ctx context.Context, nodeName string) ([]string, error) { + node, err := i.NodeByName(ctx, nodeName) + if err != nil { + return nil, err + } + rows, err := i.store.Pool().Query(ctx, + `select module from taken where node = $1 order by module`, node.ID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var m string + if err := rows.Scan(&m); err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +} diff --git a/internal/inventory/adoption_test.go b/internal/inventory/adoption_test.go new file mode 100644 index 0000000..9e505a7 --- /dev/null +++ b/internal/inventory/adoption_test.go @@ -0,0 +1,134 @@ +package inventory + +import ( + "errors" + "reflect" + "testing" +) + +// novox/hq ADR 0100: a node is adopted or converged, and the controller records which. + +func TestANodeAddedWithoutSayingIsConverged(t *testing.T) { + inv := fresh(t) + if _, err := inv.AddNode(t.Context(), "anchor"); err != nil { + t.Fatal(err) + } + n, err := inv.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if n.Adopted || !n.AdoptedSince.IsZero() { + t.Fatalf("a node nobody said anything about reads as adopted: %+v", n) + } +} + +func TestAnAdoptedNodeRoundTripsThroughEveryReading(t *testing.T) { + inv := fresh(t) + made, err := inv.AddNodeAs(t.Context(), "anchor", true) + if err != nil { + t.Fatal(err) + } + if !made.Adopted || made.AdoptedSince.IsZero() { + t.Fatalf("added adopted, got %+v", made) + } + byName, err := inv.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + all, err := inv.Nodes(t.Context()) + if err != nil { + t.Fatal(err) + } + if !byName.Adopted || len(all) != 1 || !all[0].Adopted { + t.Fatalf("adoption did not survive reading back: %+v %+v", byName, all) + } + + // And through a token: enrolment reads the node from the token it spends. + issued, err := inv.IssueToken(t.Context(), "anchor", 60e9) + if err != nil { + t.Fatal(err) + } + claimed, err := inv.Claim(t.Context(), issued.Secret, "a-key", false) + if err != nil { + t.Fatal(err) + } + if !claimed.Adopted { + t.Fatal("the node a token claims lost its mode") + } +} + +func TestTakingIsRefusedOnAConvergedNodeAndForAnUnassignedModule(t *testing.T) { + inv := fresh(t) + if err := inv.RegisterModule(t.Context(), manifest("hello-web", nil, nil), Source{}); err != nil { + t.Fatal(err) + } + if _, err := inv.AddNode(t.Context(), "converged"); err != nil { + t.Fatal(err) + } + if err := inv.Assign(t.Context(), "converged", "hello-web"); err != nil { + t.Fatal(err) + } + if err := inv.Take(t.Context(), "converged", "hello-web"); !errors.Is(err, ErrNotAdopted) { + t.Fatalf("taking on a converged node gave %v", err) + } + + if _, err := inv.AddNodeAs(t.Context(), "anchor", true); err != nil { + t.Fatal(err) + } + if err := inv.Take(t.Context(), "anchor", "hello-web"); !errors.Is(err, ErrNotAssigned) { + t.Fatalf("taking an unassigned module gave %v", err) + } +} + +func TestATakenModuleOutlivesItsAssignmentAndReturningToAdopted(t *testing.T) { + inv := fresh(t) + for _, m := range []string{"hello-web", "postgres"} { + if err := inv.RegisterModule(t.Context(), manifest(m, nil, nil), Source{}); err != nil { + t.Fatal(err) + } + } + if _, err := inv.AddNodeAs(t.Context(), "anchor", true); err != nil { + t.Fatal(err) + } + for _, m := range []string{"hello-web", "postgres"} { + if err := inv.Assign(t.Context(), "anchor", m); err != nil { + t.Fatal(err) + } + } + if err := inv.Take(t.Context(), "anchor", "postgres"); err != nil { + t.Fatal(err) + } + if err := inv.Take(t.Context(), "anchor", "postgres"); err != nil { + t.Fatalf("taking twice is not an error: %v", err) + } + if err := inv.Unassign(t.Context(), "anchor", "postgres"); err != nil { + t.Fatal(err) + } + taken, err := inv.Taken(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(taken, []string{"postgres"}) { + t.Fatalf("unassigning un-took it: %v", taken) + } + + took, err := inv.Converge(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(took, []string{"hello-web"}) { + t.Fatalf("converging took %v; it takes every assigned module not yet taken", took) + } + n, _ := inv.NodeByName(t.Context(), "anchor") + if n.Adopted { + t.Fatal("converged node still reads adopted") + } + + if err := inv.SetAdopted(t.Context(), "anchor", true); err != nil { + t.Fatal(err) + } + taken, _ = inv.Taken(t.Context(), "anchor") + if !reflect.DeepEqual(taken, []string{"hello-web", "postgres"}) { + t.Fatalf("returning to adopted lost what was taken: %v", taken) + } +} diff --git a/internal/inventory/migrations/0029-a-node-is-adopted-or-converged.sql b/internal/inventory/migrations/0029-a-node-is-adopted-or-converged.sql new file mode 100644 index 0000000..a5d2fb9 --- /dev/null +++ b/internal/inventory/migrations/0029-a-node-is-adopted-or-converged.sql @@ -0,0 +1,21 @@ +-- A node is adopted or converged, and the mesh records which (novox/hq ADR 0100). +-- +-- A machine already serving a predecessor's services is adopted: what is found on it is kept +-- until its module is taken, and the firewall found on it stays in force. It stays adopted until +-- the operator converges it. False by default, so every node that exists is converged, as it was. + +alter table node add column adopted boolean not null default false; +-- When it was last made adopted, and when it last converged. Both kept: a node returned to adopted +-- after converging is a different history from one that never converged. +alter table node add column adopted_since timestamptz; +alter table node add column converged_at timestamptz; + +-- The modules taken on a node: its cutover, the operator's act, done when the module's data has +-- moved. A row per node and module, and it outlives the assignment on purpose -- unassigning a +-- module does not un-take it, and what was taken stays taken when a node returns to adopted. +create table taken ( + node uuid not null references node(id) on delete cascade, + module text not null references module(name) on delete cascade, + taken_at timestamptz not null default now(), + primary key (node, module) +); diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 2b822ce..fb736b1 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -49,6 +49,12 @@ type Node struct { // month's assignments, and until this existed those looked the same as a node that is // current (novox/hq 09-the-node-lifecycle). LastSeen time.Time + + // Adopted is whether this node is adopted rather than converged (novox/hq ADR 0100): what is + // found on it is kept until its module is taken, and the firewall found on it stays in force. + // AdoptedSince is when it last became so; zero for a converged node. + Adopted bool + AdoptedSince time.Time } // Silent is how long since this node was last heard from, and whether it ever was. @@ -74,28 +80,59 @@ var ErrNameTaken = errors.New("a node of that name already exists") // (novox/hq 09-the-node-lifecycle), so the record is what a token binds to and must exist before // there is anything to join. func (i *Inventory) AddNode(ctx context.Context, name string) (Node, error) { + return i.AddNodeAs(ctx, name, false) +} + +// AddNodeAs creates a node record, adopted or converged (novox/hq ADR 0100). The operator says +// which; a node added without saying is converged, as every node was before adoption existed. +func (i *Inventory) AddNodeAs(ctx context.Context, name string, adopted bool) (Node, error) { name = strings.TrimSpace(name) if name == "" { return Node{}, errors.New("a node needs a name: it is how a token is issued for it") } var n Node + var since *time.Time err := i.store.Pool().QueryRow(ctx, - `insert into node (name) values ($1) returning id, name, created`, - name).Scan(&n.ID, &n.Name, &n.Created) + `insert into node (name, adopted, adopted_since) + values ($1, $2, case when $2 then now() end) + returning id, name, created, adopted, adopted_since`, + name, adopted).Scan(&n.ID, &n.Name, &n.Created, &n.Adopted, &since) if err != nil { if strings.Contains(err.Error(), "node_name_key") { return Node{}, fmt.Errorf("%w: %s", ErrNameTaken, name) } return Node{}, err } + if since != nil { + n.AdoptedSince = *since + } + return n, nil +} + +// nodeColumns and scanNode are the one reading of a node row, so every way of finding a node +// says whether it is adopted. +const nodeColumns = `id, name, created, last_seen, adopted, adopted_since` + +func scanNode(row pgx.Row) (Node, error) { + var n Node + var seen, since *time.Time + if err := row.Scan(&n.ID, &n.Name, &n.Created, &seen, &n.Adopted, &since); err != nil { + return Node{}, err + } + if seen != nil { + n.LastSeen = *seen + } + if since != nil { + n.AdoptedSince = *since + } return n, nil } // Nodes are every node record, oldest first. func (i *Inventory) Nodes(ctx context.Context) ([]Node, error) { rows, err := i.store.Pool().Query(ctx, - `select id, name, created, last_seen from node order by created, name`) + `select `+nodeColumns+` from node order by created, name`) if err != nil { return nil, err } @@ -103,14 +140,10 @@ func (i *Inventory) Nodes(ctx context.Context) ([]Node, error) { var nodes []Node for rows.Next() { - var n Node - var seen *time.Time - if err := rows.Scan(&n.ID, &n.Name, &n.Created, &seen); err != nil { + n, err := scanNode(rows) + if err != nil { return nil, err } - if seen != nil { - n.LastSeen = *seen - } nodes = append(nodes, n) } return nodes, rows.Err() @@ -118,9 +151,8 @@ func (i *Inventory) Nodes(ctx context.Context) ([]Node, error) { // NodeByName finds one node record. func (i *Inventory) NodeByName(ctx context.Context, name string) (Node, error) { - var n Node - err := i.store.Pool().QueryRow(ctx, - `select id, name, created from node where name = $1`, name).Scan(&n.ID, &n.Name, &n.Created) + n, err := scanNode(i.store.Pool().QueryRow(ctx, + `select `+nodeColumns+` from node where name = $1`, name)) if errors.Is(err, pgx.ErrNoRows) { return Node{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name) } @@ -248,10 +280,7 @@ func (i *Inventory) Claim(ctx context.Context, secret, by string, again bool) (N if err != nil { return Node{}, err } - var n Node - err = i.store.Pool().QueryRow(ctx, - `select id, name, created from node where id = $1`, id).Scan(&n.ID, &n.Name, &n.Created) - return n, err + return scanNode(i.store.Pool().QueryRow(ctx, `select `+nodeColumns+` from node where id = $1`, id)) } // Spend makes a claimed token used, only for the presenter holding the claim. The last write to the @@ -293,10 +322,7 @@ func (i *Inventory) Redeem(ctx context.Context, secret string) (Node, error) { return Node{}, err } - var n Node - err = i.store.Pool().QueryRow(ctx, - `select id, name, created from node where id = $1`, id).Scan(&n.ID, &n.Name, &n.Created) - return n, err + return scanNode(i.store.Pool().QueryRow(ctx, `select `+nodeColumns+` from node where id = $1`, id)) } // RecordProfile keeps the last thing a node said about what it can do. diff --git a/internal/token/token.go b/internal/token/token.go index ef99fbe..ee7ab49 100644 --- a/internal/token/token.go +++ b/internal/token/token.go @@ -50,6 +50,11 @@ type Token struct { // Secret is the one-time right to join. Useless once used, useless after it expires. Secret string `json:"secret"` + + // Adopted says the node joins adopted (novox/hq ADR 0100): the host checks, before enrolling, + // that it speaks the firewall found on the machine, because an adopted node keeps that firewall + // in force. Absent for a converged node, so a converged token is byte for byte what it was. + Adopted bool `json:"adopted,omitempty"` } // Missing names the parts that are not filled in. diff --git a/internal/token/token_test.go b/internal/token/token_test.go index a7860a1..ecf3f78 100644 --- a/internal/token/token_test.go +++ b/internal/token/token_test.go @@ -140,6 +140,22 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { if len(fields) != 6 { t.Errorf("the token has %d fields, expected 6: %v", len(fields), fields) } + + // An adopted node's token says so, under exactly this name, and a converged one does not + // carry it at all (novox/hq ADR 0100). + adopted := complete(t) + adopted.Adopted = true + raw, err = json.Marshal(adopted) + if err != nil { + t.Fatal(err) + } + fields = nil + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + if fields["adopted"] != true || len(fields) != 7 { + t.Errorf("an adopted token does not carry \"adopted\": true: %v", fields) + } } func TestATokenWithNoNameIsRefused(t *testing.T) { From a86a6c2974a63ae1d45be105c4def3487d1a76ef Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:17:54 +0200 Subject: [PATCH 02/33] Carry an adopted node's mode and taken modules in every declaration, from one marshaller (hq ADR 0100) --- cmd/mesh-controller/plan.go | 55 ++++++--- cmd/mesh-controller/push.go | 47 ++++---- cmd/mesh-controller/push_test.go | 8 +- cmd/mesh-controller/sendable.go | 96 +++++++++++++++ cmd/mesh-controller/sendable_test.go | 170 +++++++++++++++++++++++++++ internal/catalogue/declaration.go | 30 +++++ 6 files changed, 357 insertions(+), 49 deletions(-) create mode 100644 cmd/mesh-controller/sendable.go create mode 100644 cmd/mesh-controller/sendable_test.go diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 55d09fc..aeef125 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "errors" @@ -271,10 +272,10 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory, // silently produced a declaration missing them — a difference between what `plan` showed and what // `plan --json` handed to anything reading it. func declarationFor(ctx context.Context, open *stores, node string, - plan catalogue.Resolution, settings catalogue.SettingsBy) ([]map[string]any, error) { + plan catalogue.Resolution, settings catalogue.SettingsBy) (sendable, error) { gens, err := generators(ctx, open) if err != nil { - return nil, err + return sendable{}, err } // **Allocating, because `plan` is the send without the sending.** It is one machine, named by // a person, who is asking what a push would do — so the port it shows and the secret it seals @@ -316,11 +317,11 @@ const ( func declarationWith(ctx context.Context, open *stores, node string, plan catalogue.Resolution, settings catalogue.SettingsBy, - gens map[string]catalogue.Generator, choosing Choosing) ([]map[string]any, error) { + gens map[string]catalogue.Generator, choosing Choosing) (sendable, error) { inv := open.inventory grants, err := grantsFor(ctx, open, node) if err != nil { - return nil, err + return sendable{}, err } // Where this machine puts what each module needs reachable (novox/hq ADR 0038). // @@ -333,7 +334,7 @@ func declarationWith(ctx context.Context, open *stores, node string, if choosing == Reading { held, err := inv.PortsFor(ctx, node) if err != nil { - return nil, err + return sendable{}, err } for _, a := range held { if already[a.Module] == nil { @@ -357,7 +358,7 @@ func declarationWith(ctx context.Context, open *stores, node string, case mayAssign && choosing == Allocating: at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed) if err != nil { - return nil, fmt.Errorf( + return sendable{}, fmt.Errorf( "%s needs %d reachable on %s and it could not be assigned: %w", m.Module, l.Port, node, err) } @@ -398,7 +399,7 @@ func declarationWith(ctx context.Context, open *stores, node string, } } if err != nil { - return nil, err + return sendable{}, err } if needed[m.Module] == nil { needed[m.Module] = map[string]string{} @@ -416,7 +417,7 @@ func declarationWith(ctx context.Context, open *stores, node string, } issued, meshCA, err := certificateFor(ctx, open, node) if err != nil { - return nil, err + return sendable{}, err } certificate, authority = issued, meshCA break @@ -429,11 +430,11 @@ func declarationWith(ctx context.Context, open *stores, node string, // One reading of the catalogue for the three questions below that resolve the whole mesh. shelf, err := inv.Catalogue(ctx) if err != nil { - return nil, err + return sendable{}, err } private, err := onThePrivateNetwork(ctx, inv, shelf) if err != nil { - return nil, err + return sendable{}, err } // And every machine's name, so a container can reach one. The same set that writes the @@ -441,7 +442,7 @@ func declarationWith(ctx context.Context, open *stores, node string, // about where another machine is. names, err := namesInTheMesh(ctx, inv, shelf) if err != nil { - return nil, err + return sendable{}, err } // And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the @@ -450,7 +451,7 @@ func declarationWith(ctx context.Context, open *stores, node string, // to serve and knows nothing about what they mean. routes, err := routeNamesInTheMesh(ctx, open) if err != nil { - return nil, err + return sendable{}, err } for name, at := range routes { names[name] = at @@ -479,15 +480,25 @@ func declarationWith(ctx context.Context, open *stores, node string, continue } if kept, err = inv.OperatorExport(ctx); err != nil { - return nil, err + return sendable{}, err } break } - return plan.Declaration(catalogue.Rendering{ + composed, err := plan.Compose(catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept}) + if err != nil { + return sendable{}, err + } + // Whether this node is adopted, and what was taken on it, said in every declaration it is + // sent from this one place (novox/hq ADR 0100). + adoption, err := adoptionOf(ctx, inv, node, plan, composed) + if err != nil { + return sendable{}, err + } + return sendable{Resources: composed.Resources, Adoption: adoption}, nil } // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq @@ -736,16 +747,21 @@ func planCommand(ctx context.Context, args []string) error { return nil } if *asJSON { - resources, err := declarationFor(ctx, open, args[0], plan, settings) + declared, err := declarationFor(ctx, open, args[0], plan, settings) if err != nil { return err } - body, err := json.MarshalIndent( - map[string]any{"declaration": 1, "resources": resources}, "", " ") + // The bytes a push would send, indented: one marshaller, so `plan --json` cannot show an + // envelope other than the one sent. + body, err := declared.Body() if err != nil { return err } - fmt.Println(string(body)) + var indented bytes.Buffer + if err := json.Indent(&indented, body, "", " "); err != nil { + return err + } + fmt.Println(indented.String()) return nil } @@ -769,10 +785,11 @@ func planCommand(ctx context.Context, args []string) error { for _, n := range plan.Needs { fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For) } - resources, err := declarationFor(ctx, open, args[0], plan, settings) + declared, err := declarationFor(ctx, open, args[0], plan, settings) if err != nil { return err } + resources := declared.Resources for module, layers := range settings { for _, layer := range layers { fmt.Printf(" %-20s settings from %s\n", module, layer.From) diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 7a21c0e..86ac3d9 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "errors" "flag" "fmt" @@ -283,10 +282,10 @@ func pushCommand(ctx context.Context, args []string) error { asked = append(asked, n.Name) } - sending, refusals := composeEach(asked, func(node string) ([]map[string]any, error) { + sending, refusals := composeEach(asked, func(node string) (sendable, error) { plan, settings, err := planFor(ctx, open, node) if err != nil { - return nil, err + return sendable{}, err } // A module assigned here that this machine cannot host is said and left out, not fatal: the // healthy modules beside it are still resolved and sent. Reported so it is not silently @@ -300,7 +299,7 @@ func pushCommand(ctx context.Context, args []string) error { sentDigest := map[string]string{} for _, s := range sending { - body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources}) + body, err := s.declared.Body() if err != nil { return err } @@ -318,7 +317,7 @@ func pushCommand(ctx context.Context, args []string) error { return err } sentDigest[s.node] = digest - fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources)) + fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources)) } fmt.Printf("\n%d node(s) told\n", len(sending)) @@ -374,17 +373,17 @@ func pushCommand(ctx context.Context, args []string) error { // (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is // all-or-nothing — so one swept machine's compose error failed the operator's named // push and skipped its --wait, the very intolerance the main path exists to avoid. - sending, refused := composeEach(also, func(node string) ([]map[string]any, error) { + sending, refused := composeEach(also, func(node string) (sendable, error) { plan, settings, err := planFor(ctx, open, node) if err != nil { - return nil, err + return sendable{}, err } reportUnhostable(node, plan) return declarationWith(ctx, open, node, plan, settings, gens, Allocating) }) refusals = append(refusals, refused...) for _, s := range sending { - body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources}) + body, err := s.declared.Body() if err != nil { return err } @@ -398,7 +397,7 @@ func pushCommand(ctx context.Context, args []string) error { if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { return err } - fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources)) + fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources)) } // Every candidate this round is marked handled — the sent ones so they are not // re-listed, and the refused ones so a machine that cannot be composed does not make @@ -458,8 +457,8 @@ func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest // readyNode is one machine and the declaration it would be sent. type readyNode struct { - node string - resources []map[string]any + node string + declared sendable } // composeEach works out what each named machine should be, and never lets one machine's answer @@ -480,21 +479,21 @@ type readyNode struct { // The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential // across two machines that must agree — and dropped here, where it never did. func composeEach(names []string, - compose func(node string) ([]map[string]any, error)) ([]readyNode, []string) { + compose func(node string) (sendable, error)) ([]readyNode, []string) { var sending []readyNode var refusals []string for _, name := range names { - resources, err := compose(name) + declared, err := compose(name) if err != nil { refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) continue } - if len(resources) == 0 { + if len(declared.Resources) == 0 { fmt.Printf("%s is assigned nothing — skipped\n", name) continue } - sending = append(sending, readyNode{name, resources}) + sending = append(sending, readyNode{name, declared}) } return sending, refusals } @@ -533,11 +532,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error { return err } - type ready struct { - node string - resources []map[string]any - } - var sending []ready + var sending []readyNode var refusals []string for _, name := range names { plan, settings, err := planFor(ctx, open, name) @@ -546,12 +541,12 @@ func sendTo(ctx context.Context, open *stores, names []string) error { continue } reportUnhostable(name, plan) - resources, err := declarationWith(ctx, open, name, plan, settings, gens, Allocating) + declared, err := declarationWith(ctx, open, name, plan, settings, gens, Allocating) if err != nil { refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) continue } - sending = append(sending, ready{name, resources}) + sending = append(sending, readyNode{name, declared}) } if len(refusals) > 0 { return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s", @@ -565,7 +560,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error { defer server.Close() for _, s := range sending { - body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources}) + body, err := s.declared.Body() if err != nil { return err } @@ -579,7 +574,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error { if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { return err } - fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources)) + fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.declared.Resources)) } return nil } @@ -610,11 +605,11 @@ func wouldSend(ctx context.Context, open *stores, if err != nil { continue } - resources, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading) + declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading) if err != nil { continue } - body, err := json.Marshal(map[string]any{"declaration": 1, "resources": resources}) + body, err := declared.Body() if err != nil { return nil, err } diff --git a/cmd/mesh-controller/push_test.go b/cmd/mesh-controller/push_test.go index 8448169..3e6dc9d 100644 --- a/cmd/mesh-controller/push_test.go +++ b/cmd/mesh-controller/push_test.go @@ -18,11 +18,11 @@ import ( func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) { sending, refusals := composeEach( []string{"anchor", "home-server", "laptop"}, - func(node string) ([]map[string]any, error) { + func(node string) (sendable, error) { if node == "anchor" { - return nil, errors.New(`nothing provides "acme-ca", wanted by route-proxy`) + return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`) } - return []map[string]any{{"id": node + ".thing"}}, nil + return sendable{Resources: []map[string]any{{"id": node + ".thing"}}}, nil }) var told []string @@ -42,7 +42,7 @@ func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) { // And a machine assigned nothing is neither sent nor a refusal — it is nothing to say. func TestAMachineAssignedNothingIsNotARefusal(t *testing.T) { sending, refusals := composeEach([]string{"spare"}, - func(string) ([]map[string]any, error) { return nil, nil }) + func(string) (sendable, error) { return sendable{}, nil }) if len(sending) != 0 || len(refusals) != 0 { t.Errorf("a machine assigned nothing was treated as something: %v / %v", sending, refusals) } diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go new file mode 100644 index 0000000..d3a88da --- /dev/null +++ b/cmd/mesh-controller/sendable.go @@ -0,0 +1,96 @@ +package main + +import ( + "context" + "encoding/json" + "sort" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// sendable is a declaration as a machine is sent it: its resources and, for an adopted node, its +// mode and which modules were taken on it (novox/hq ADR 0100). +// +// **Body is the only place the envelope is marshalled.** It was written by hand at every send site, +// in the digest the mesh compares, and in `plan --json`; a key added at one and not another would +// make a machine look out of date for ever, or send something `plan` never showed. +type sendable struct { + Resources []map[string]any + // Adoption is nil for a converged node, and then the body is byte for byte what it was before + // adoption existed: an older host parses the envelope strictly and would refuse the key. + Adoption *adoptionEnvelope +} + +// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on +// it that it runs; Untaken is, for every module it runs that is not taken, the ids of that module's +// files and containers — the resources the host keeps as found until the module is taken. Ids +// rather than a rule to split them by, because a module's name may contain a dot. +type adoptionEnvelope struct { + Taken []string `json:"taken"` + Untaken map[string][]string `json:"untaken,omitempty"` +} + +// Body is the declaration's bytes, as sent and as digested. +func (s sendable) Body() ([]byte, error) { + envelope := map[string]any{"declaration": 1, "resources": s.Resources} + if s.Adoption != nil { + envelope["adoption"] = s.Adoption + } + return json.Marshal(envelope) +} + +// adoptionOf is the envelope for a node, nil when it is converged. +func adoptionOf(ctx context.Context, inv *inventory.Inventory, node string, + plan catalogue.Resolution, composed catalogue.Composed) (*adoptionEnvelope, error) { + record, err := inv.NodeByName(ctx, node) + if err != nil { + return nil, err + } + if !record.Adopted { + return nil, nil + } + taken, err := inv.Taken(ctx, node) + if err != nil { + return nil, err + } + return adoptionFor(plan, taken, composed), nil +} + +// adoptionFor is the envelope computed from what was taken and who owns each resource. +// +// Every module the node runs that is not taken is untaken — including one pulled in by another +// rather than assigned: what is found is kept until its module is taken, whoever put it there. +func adoptionFor(plan catalogue.Resolution, taken []string, + composed catalogue.Composed) *adoptionEnvelope { + isTaken := map[string]bool{} + for _, m := range taken { + isTaken[m] = true + } + out := &adoptionEnvelope{Taken: []string{}} + runs := map[string]bool{} + for _, m := range plan.Modules { + runs[m.Module] = true + if isTaken[m.Module] { + out.Taken = append(out.Taken, m.Module) + } + } + sort.Strings(out.Taken) + for _, r := range composed.Resources { + id, _ := r["id"].(string) + module, owned := composed.Owner[id] + if !owned || !runs[module] || isTaken[module] { + continue + } + switch r["type"] { + case "file", "container": + default: + continue + } + if out.Untaken == nil { + out.Untaken = map[string][]string{} + } + out.Untaken[module] = append(out.Untaken[module], id) + } + return out +} diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go new file mode 100644 index 0000000..bf052b9 --- /dev/null +++ b/cmd/mesh-controller/sendable_test.go @@ -0,0 +1,170 @@ +package main + +import ( + "bytes" + "encoding/json" + "io" + "os" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq ADR 0100: every declaration an adopted node is sent says it is adopted and which modules +// were taken on it; a converged node's declaration is byte for byte what it was. + +// helloWeb is a module the predecessor also runs: a page and a server under names it uses. +func helloWeb() catalogue.Manifest { + return catalogue.Manifest{Module: "hello-web", Version: "1", + Resources: []map[string]any{ + {"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hello"}, + {"id": "server", "type": "container", "name": "hello-web", + "image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)}, + {"id": "served", "type": "directory", "path": "/var/lib/hello-web"}, + }} +} + +// composed is what node would be sent now, as push composes it. +func composed(t *testing.T, open *stores, node string) sendable { + t.Helper() + plan, settings, err := planFor(t.Context(), open, node) + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(t.Context(), open, node, plan, settings) + if err != nil { + t.Fatal(err) + } + return declared +} + +func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) { + open := aMesh(t) + register(t, open, helloWeb()) + if _, err := assign(t.Context(), open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + declared := composed(t, open, "anchor") + if declared.Adoption != nil { + t.Fatal("a converged node was given an adoption envelope") + } + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + // The envelope exactly as every send site marshalled it before adoption existed. + before, err := json.Marshal(map[string]any{"declaration": 1, "resources": declared.Resources}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(body, before) { + t.Fatalf("a converged declaration changed:\n%s\n%s", body, before) + } + if bytes.Contains(body, []byte(`"adoption"`)) { + t.Fatal("a converged declaration names adoption; an older host would refuse it") + } +} + +func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, helloWeb()) + if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + + declared := composed(t, open, "anchor") + if declared.Adoption == nil { + t.Fatal("an adopted node's declaration does not say it is adopted") + } + untaken := declared.Adoption.Untaken["hello-web"] + if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server"}) { + t.Fatalf("hello-web's files and containers are not named untaken: %v", declared.Adoption) + } + if len(declared.Adoption.Taken) != 0 { + t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken) + } + + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + var envelope map[string]any + if err := json.Unmarshal(body, &envelope); err != nil { + t.Fatal(err) + } + adoption, _ := envelope["adoption"].(map[string]any) + if _, ok := adoption["taken"].([]any); !ok { + t.Fatalf("taken is not a list on the wire, even empty: %s", body) + } + + // The digest the mesh compares is the digest of what is sent: status and push agree. + would, err := wouldSend(ctx, open, mustNodes(t, open)) + if err != nil { + t.Fatal(err) + } + if would["anchor"] != digestOf(body) { + t.Fatal("the digest the mesh compares is not of the declaration push sends") + } + + // plan --json prints that same envelope. + printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) }) + var compact bytes.Buffer + if err := json.Compact(&compact, []byte(printed)); err != nil { + t.Fatalf("plan --json is not JSON: %v\n%s", err, printed) + } + if digestOf(compact.Bytes()) != digestOf(body) { + t.Fatalf("plan --json shows something other than what push sends:\n%s", printed) + } + + // Taking the module moves its resources out of untaken. + if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + declared = composed(t, open, "anchor") + if _, still := declared.Adoption.Untaken["hello-web"]; still { + t.Fatalf("a taken module is still untaken: %v", declared.Adoption) + } + if !reflect.DeepEqual(declared.Adoption.Taken, []string{"hello-web"}) { + t.Fatalf("taken is %v", declared.Adoption.Taken) + } +} + +func mustNodes(t *testing.T, open *stores) []inventory.Node { + t.Helper() + nodes, err := open.inventory.Nodes(t.Context()) + if err != nil { + t.Fatal(err) + } + return nodes +} + +// stdoutOf is what run printed. +func stdoutOf(t *testing.T, run func() error) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + saved := os.Stdout + os.Stdout = w + done := make(chan string) + go func() { + all, _ := io.ReadAll(r) + done <- string(all) + }() + runErr := run() + os.Stdout = saved + w.Close() + out := <-done + if runErr != nil { + t.Fatalf("%v\n%s", runErr, out) + } + return out +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 11d983c..39d8fa3 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -146,6 +146,34 @@ func (r Rendering) machinePort(module string, wanted int) int { // both call something "config", and without this the second would silently replace the first — // the node applying one of them and reporting success. func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { + composed, err := r.Compose(with) + if err != nil { + return nil, err + } + return composed.Resources, nil +} + +// Composed is a declaration's resources and which module each came from. +// +// Owner is kept beside the resources because a resource id cannot be split back into its module: +// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard — +// has no owner. +type Composed struct { + Resources []map[string]any + Owner map[string]string +} + +// Compose is Declaration with the owner of every resource said. +func (r Resolution) Compose(with Rendering) (Composed, error) { + owner := map[string]string{} + resources, err := r.compose(with, owner) + if err != nil { + return Composed{}, err + } + return Composed{Resources: resources, Owner: owner}, nil +} + +func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) { // Where each provision's credentials land, so a contribution can name the file rather than // carry a value the mesh does not have. directories := map[string]string{} @@ -521,6 +549,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil { copied["restart-on"] = renamed } + owner[fmt.Sprint(copied["id"])] = m.Module out = append(out, copied) } @@ -534,6 +563,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { } for _, fact := range given { fact["id"] = m.Module + "." + fmt.Sprint(fact["id"]) + owner[fmt.Sprint(fact["id"])] = m.Module out = append(out, fact) } } From c3b1617693578239365b47c115c17f6071b88b66 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:21:44 +0200 Subject: [PATCH 03/33] Declare openings and a refusal-only guard on an adopted node in place of the filter (hq ADR 0100) --- cmd/mesh-controller/plan.go | 19 ++- cmd/mesh-controller/sendable.go | 8 +- internal/catalogue/adoption.go | 218 +++++++++++++++++++++++++++ internal/catalogue/adoption_test.go | 223 ++++++++++++++++++++++++++++ internal/catalogue/declaration.go | 57 +++++++ internal/catalogue/manifest.go | 14 ++ 6 files changed, 526 insertions(+), 13 deletions(-) create mode 100644 internal/catalogue/adoption.go create mode 100644 internal/catalogue/adoption_test.go diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index aeef125..3781267 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -485,16 +485,21 @@ func declarationWith(ctx context.Context, open *stores, node string, break } - composed, err := plan.Compose(catalogue.Rendering{ - Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, - Certificate: certificate, Authority: authority, Mesh: private, Names: names, - Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept}) + // Whether this node is adopted (novox/hq ADR 0100): then the found firewall stays in force, and + // the declaration carries openings and the mesh's guard in place of a filter. + record, err := inv.NodeByName(ctx, node) if err != nil { return sendable{}, err } - // Whether this node is adopted, and what was taken on it, said in every declaration it is - // sent from this one place (novox/hq ADR 0100). - adoption, err := adoptionOf(ctx, inv, node, plan, composed) + composed, err := plan.Compose(catalogue.Rendering{ + Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, + Certificate: certificate, Authority: authority, Mesh: private, Names: names, + Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted}) + if err != nil { + return sendable{}, err + } + // And what was taken on it, said in every declaration it is sent from this one place. + adoption, err := adoptionOf(ctx, inv, record, plan, composed) if err != nil { return sendable{}, err } diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index d3a88da..fcf182c 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -41,16 +41,12 @@ func (s sendable) Body() ([]byte, error) { } // adoptionOf is the envelope for a node, nil when it is converged. -func adoptionOf(ctx context.Context, inv *inventory.Inventory, node string, +func adoptionOf(ctx context.Context, inv *inventory.Inventory, record inventory.Node, plan catalogue.Resolution, composed catalogue.Composed) (*adoptionEnvelope, error) { - record, err := inv.NodeByName(ctx, node) - if err != nil { - return nil, err - } if !record.Adopted { return nil, nil } - taken, err := inv.Taken(ctx, node) + taken, err := inv.Taken(ctx, record.Name) if err != nil { return nil, err } diff --git a/internal/catalogue/adoption.go b/internal/catalogue/adoption.go new file mode 100644 index 0000000..5e31ef0 --- /dev/null +++ b/internal/catalogue/adoption.go @@ -0,0 +1,218 @@ +package catalogue + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +// What an adopted node is declared in place of a filter (novox/hq ADR 0100). +// +// On an adopted node the firewall found on the machine stays in force: the mesh loads no table +// that drops by default or holds an accept. What the mesh needs reachable is declared as +// openings, which the host converges through the found firewall in its own terms; and the mesh +// guards its own foundation ports itself, in a table that only refuses. + +// AdoptionPrefix is the id prefix of what the mesh declares of its own on an adopted node. It is +// never a module's, so none of it is ever held as found. +const AdoptionPrefix = "adoption." + +// Where an opening admits from, on the wire. +const ( + OpeningFromEverywhere = "everywhere" + OpeningFromMesh = "mesh" +) + +// The two paths a packet reaches a port by: received by the machine, or forwarded to a +// container that publishes it. +const ( + PathIncoming = "incoming" + PathForwarded = "forwarded" +) + +// Guard resources: the refusal-only table, the unit that loads it, and that unit running. +const ( + GuardPath = "/etc/mesh/guard.nft" + GuardUnit = "mesh-guard.service" + // GuardUnitPath is where the unit is written. + GuardUnitPath = "/etc/systemd/system/" + GuardUnit +) + +// GuardID, GuardUnitID and GuardRunningID are the guard's resource identities. The installer +// raises the same three on an adopted genesis, so the first push finds them already there. +func GuardID() string { return AdoptionPrefix + "guard" } +func GuardUnitID() string { return AdoptionPrefix + "guard-unit" } +func GuardRunningID() string { return AdoptionPrefix + "guard-running" } + +// OpeningID is an opening's resource identity: its protocol, port and path say what it is. +func OpeningID(protocol string, port int, path string) string { + return fmt.Sprintf("%sopening-%s-%d-%s", AdoptionPrefix, protocol, port, path) +} + +// Openings are what the mesh needs reachable on an adopted node, from the same inputs as the +// filter it would load were the node converged, each from where that filter would admit it. +// +// `rules` is Filtering's answer — every module's listens, the hub's port, the per-node exposure — +// and `foundation` is the ports the mesh itself needs, from everywhere. A rule for this machine +// only opens nothing. `published` maps a machine port a container publishes to the container's +// port: a published port is forwarded, not received, so its opening names the forwarded path and +// the port the packet is forwarded to. +func Openings(rules []Rule, foundation []int, published map[string]map[int]int) []map[string]any { + type key struct { + protocol string + port int + } + from := map[key]string{} + var order []key + widen := func(k key, f string) { + was, seen := from[k] + if !seen { + order = append(order, k) + } + if !seen || was != OpeningFromEverywhere { + from[k] = f + } + } + for _, rule := range rules { + switch rule.From { + case FromEverywhere: + widen(key{rule.Protocol, rule.Port}, OpeningFromEverywhere) + case FromMesh: + widen(key{rule.Protocol, rule.Port}, OpeningFromMesh) + } + } + for _, port := range foundation { + widen(key{"tcp", port}, OpeningFromEverywhere) + } + sort.Slice(order, func(a, b int) bool { + if order[a].port != order[b].port { + return order[a].port < order[b].port + } + return order[a].protocol < order[b].protocol + }) + out := make([]map[string]any, 0, len(order)) + for _, k := range order { + opening := map[string]any{"type": "opening", "port": k.port, "protocol": k.protocol, + "from": from[k]} + if to, forwarded := published[k.protocol][k.port]; forwarded { + opening["id"] = OpeningID(k.protocol, k.port, PathForwarded) + opening["path"] = PathForwarded + opening["to"] = to + } else { + opening["id"] = OpeningID(k.protocol, k.port, PathIncoming) + opening["path"] = PathIncoming + } + out = append(out, opening) + } + return out +} + +// Published is every port the given containers publish on the machine, by protocol and machine +// port, mapped to the container's own port. A mapping bound to loopback is left out: nothing off +// the machine reaches it, forwarded or not. +func Published(resources []map[string]any) map[string]map[int]int { + out := map[string]map[int]int{} + for _, r := range resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + listed, _ := r["ports"].([]any) + for _, entry := range listed { + written := strings.TrimSpace(fmt.Sprint(entry)) + protocol := "tcp" + if cut := strings.LastIndex(written, "/"); cut >= 0 { + protocol = written[cut+1:] + written = written[:cut] + } + parts := strings.Split(written, ":") + if len(parts) < 2 { + continue + } + if len(parts) == 3 && (parts[0] == "127.0.0.1" || parts[0] == "localhost" || + parts[0] == "[::1]") { + continue + } + outer, err := strconv.Atoi(parts[len(parts)-2]) + if err != nil { + continue + } + inner, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + continue + } + if out[protocol] == nil { + out[protocol] = map[int]int{} + } + out[protocol][outer] = inner + } + } + return out +} + +// AsGuard renders the mesh's refusal-only table for the given machine ports. +// +// It passes everything by default and holds nothing but a refusal, so it cannot close anything +// the machine serves; and it is the mesh's own table, so the found firewall reloading does not +// touch it. It refuses the ports except from the machine itself — its loopback and the container +// runtime's own networks — and from the private network, known by the interface a packet arrives +// on and never by its source address. At prerouting, ahead of the runtime's destination +// translation, so it matches the port the packet was sent to; in the inet family, so both address +// families. +// +// The same text the installer raises on an adopted genesis; a test holds both to it. +func AsGuard(ports []int) string { + sorted := append([]int{}, ports...) + sort.Ints(sorted) + listed := make([]string, len(sorted)) + for i, p := range sorted { + listed[i] = strconv.Itoa(p) + } + var b strings.Builder + b.WriteString("table inet mesh_guard {}\n") + b.WriteString("delete table inet mesh_guard\n") + b.WriteString("table inet mesh_guard {\n") + b.WriteString("\tchain prerouting {\n") + b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n") + fmt.Fprintf(&b, "\t\tiifname != \"lo\" iifname != \"docker0\" iifname != \"br-*\" "+ + "iifname != \"mesh0\" tcp dport { %s } drop\n", strings.Join(listed, ", ")) + b.WriteString("\t}\n") + b.WriteString("}\n") + return b.String() +} + +// GuardUnitText is the unit that loads the guard. Stopping it deletes only its own table: never +// a flush, which would take the container runtime's rules and the found firewall with it. +func GuardUnitText() string { + return "[Unit]\n" + + "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + + "After=network-pre.target\n" + + "Wants=network-pre.target\n" + + "\n" + + "[Service]\n" + + "Type=oneshot\n" + + "RemainAfterExit=yes\n" + + "ExecStart=nft -f " + GuardPath + "\n" + + "ExecReload=nft -f " + GuardPath + "\n" + + "ExecStop=nft delete table inet mesh_guard\n" + + "\n" + + "[Install]\n" + + "WantedBy=multi-user.target\n" +} + +// GuardResources are the guard as three resources of the existing kinds: the table, the unit, and +// the unit running, restarted when the table changes. Nothing when there is nothing to guard: an +// empty set is not a table nft loads. +func GuardResources(ports []int) []map[string]any { + if len(ports) == 0 { + return nil + } + return []map[string]any{ + {"id": GuardID(), "type": "file", "path": GuardPath, "content": AsGuard(ports), + "mode": "0644"}, + {"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(), + "mode": "0644"}, + {"id": GuardRunningID(), "type": "service", "unit": GuardUnit, "state": "running", + "boot": "enabled", "restart-on": []any{GuardID(), GuardUnitID()}}, + } +} diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go new file mode 100644 index 0000000..20569bd --- /dev/null +++ b/internal/catalogue/adoption_test.go @@ -0,0 +1,223 @@ +package catalogue + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +// novox/hq ADR 0100: on an adopted node the found firewall stays in force. The mesh declares +// openings where it would have loaded a filter, and guards its own ports in a table that only +// refuses. + +// hub is the private network's generator on the hub: it opens the hub's port from anywhere. +type hub struct{} + +func (hub) Resources(string) ([]map[string]any, bool, error) { + return []map[string]any{{"id": "config", "type": "file", "path": "/etc/wireguard/mesh0.conf", + "content": "[Interface]\n"}}, true, nil +} +func (hub) Listens(string) ([]Listening, error) { + return []Listening{{Port: 51820, Protocol: "udp", From: FromEverywhere}}, nil +} + +// anAdoptedAnchor is the control-node's set: the store, the bus, the registry, the private network, +// a served module and the filter module. +func anAdoptedAnchor() Resolution { + return Resolution{Node: "anchor", Modules: []Manifest{ + {Module: "network", Computed: "overlay"}, + {Module: "postgres", Guards: []int{5432}, + Listens: []Listening{{Port: 5432, From: FromMesh}}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}}}}, + {Module: "lavinmq", Guards: []int{15672}, + Listens: []Listening{{Port: 5671, From: FromMesh}, {Port: 5672, From: FromMesh}}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker", + "ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}}}}, + {Module: "distribution", + Listens: []Listening{{Port: 5000, From: FromMesh}}, + Resources: []map[string]any{{"id": "store", "type": "container", "name": "registry", + "ports": []any{"5000"}}}}, + {Module: "hello-web", + Listens: []Listening{{Port: 8080, From: FromEverywhere}}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "hello-web", + "ports": []any{"8080"}}}}, + {Module: "helper", Listens: []Listening{{Port: 9000, From: FromMachine}}}, + {Module: "nftables", Filtering: &Filtering{Into: "/etc/nftables.conf"}, + Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service", + "state": "running", "restart-on": []any{"filtering"}}}}, + }} +} + +func anchorRendering(adopted bool) Rendering { + return Rendering{ + Generators: map[string]Generator{"overlay": hub{}}, + Ports: map[string]map[int]int{"distribution": {5000: 5000}, "hello-web": {8080: 20001}}, + Settings: SettingsBy{"distribution": {{From: "node anchor", + Values: map[string]any{ExposeSetting: map[string]any{"5000": FromEverywhere}}}}}, + Mesh: []string{"10.42.0.1"}, + Foundation: []int{5671}, + Adopted: adopted, + } +} + +func byID(resources []map[string]any) map[string]map[string]any { + out := map[string]map[string]any{} + for _, r := range resources { + out[r["id"].(string)] = r + } + return out +} + +func TestAnAdoptedNodeIsDeclaredOpeningsFromTheSameInputsAsTheFilter(t *testing.T) { + composed, err := anAdoptedAnchor().Compose(anchorRendering(true)) + if err != nil { + t.Fatal(err) + } + got := byID(composed.Resources) + want := map[string]map[string]any{ + // The hub's port, from anywhere, received. + "adoption.opening-udp-51820-incoming": {"port": 51820, "protocol": "udp", + "from": "everywhere", "path": "incoming"}, + // The store's port from the private network only, and forwarded: a container publishes it. + "adoption.opening-tcp-5432-forwarded": {"port": 5432, "protocol": "tcp", "from": "mesh", + "path": "forwarded", "to": 5432}, + // The bus from anywhere: a node enrols over it before it has a private address. + "adoption.opening-tcp-5671-forwarded": {"port": 5671, "protocol": "tcp", + "from": "everywhere", "path": "forwarded", "to": 5671}, + "adoption.opening-tcp-5672-forwarded": {"port": 5672, "protocol": "tcp", "from": "mesh", + "path": "forwarded", "to": 5672}, + // The registry from anywhere, by its node's exposure setting. + "adoption.opening-tcp-5000-forwarded": {"port": 5000, "protocol": "tcp", + "from": "everywhere", "path": "forwarded", "to": 5000}, + // A published port names the machine port and the container port it is forwarded to. + "adoption.opening-tcp-20001-forwarded": {"port": 20001, "protocol": "tcp", + "from": "everywhere", "path": "forwarded", "to": 8080}, + } + for id, fields := range want { + opening, ok := got[id] + if !ok { + t.Errorf("no %s among %v", id, keys(got)) + continue + } + if opening["type"] != "opening" { + t.Errorf("%s is a %v", id, opening["type"]) + } + for k, v := range fields { + if opening[k] != v { + t.Errorf("%s: %s is %v, want %v", id, k, opening[k], v) + } + } + } + for id := range got { + if strings.HasPrefix(id, "adoption.opening-") && want[id] == nil { + t.Errorf("an opening nothing asked for: %s", id) + } + } + // A port for this machine only opens nothing, and the management port is not opened at all. + for id := range got { + if strings.Contains(id, "-9000-") || strings.Contains(id, "-15672-") { + t.Errorf("%s is opened", id) + } + } + + // And openings come first, in the order the machine applies them. + if !strings.HasPrefix(composed.Resources[0]["id"].(string), "adoption.opening-") { + t.Errorf("openings are not first: %v", composed.Resources[0]["id"]) + } +} + +func TestAnAdoptedNodeLoadsNoFilterOfTheMeshs(t *testing.T) { + composed, err := anAdoptedAnchor().Compose(anchorRendering(true)) + if err != nil { + t.Fatal(err) + } + for _, r := range composed.Resources { + if r["path"] == "/etc/nftables.conf" || strings.HasPrefix(r["id"].(string), "nftables.") { + t.Fatalf("an adopted node is declared the filter module's %v", r["id"]) + } + if content, _ := r["content"].(string); strings.Contains(content, "policy drop") { + t.Fatalf("an adopted node is declared a table that drops by default: %v", r["id"]) + } + // Nothing but refusals: the only accept in the guard is its policy. + if content, _ := r["content"].(string); r["id"] == GuardID() && + strings.Count(content, "accept") != 1 { + t.Fatalf("the guard holds an accept:\n%s", content) + } + } + got := byID(composed.Resources) + guard := got[GuardID()] + if guard == nil || got[GuardUnitID()] == nil || got[GuardRunningID()] == nil { + t.Fatalf("no guard: %v", keys(got)) + } + if guard["content"] != AsGuard([]int{5432, 15672}) { + t.Fatalf("the guard does not guard the store and the management port:\n%s", guard["content"]) + } + if !reflect.DeepEqual(got[GuardRunningID()]["restart-on"], []any{GuardID(), GuardUnitID()}) { + t.Fatalf("the guard is not reloaded when its table changes: %v", got[GuardRunningID()]) + } + // Nothing of the mesh's own is anybody's to hold. + for id, module := range composed.Owner { + if strings.HasPrefix(id, AdoptionPrefix) { + t.Fatalf("%s is owned by %s", id, module) + } + } +} + +func TestAConvergedNodeIsDeclaredItsFilterAndNoOpenings(t *testing.T) { + composed, err := anAdoptedAnchor().Compose(anchorRendering(false)) + if err != nil { + t.Fatal(err) + } + got := byID(composed.Resources) + if got["nftables.filtering"] == nil || got["nftables.load"] == nil { + t.Fatalf("a converged node lost its filter: %v", keys(got)) + } + for id := range got { + if strings.HasPrefix(id, AdoptionPrefix) { + t.Fatalf("a converged node is declared %s", id) + } + } + plain, err := anAdoptedAnchor().Declaration(anchorRendering(false)) + if err != nil { + t.Fatal(err) + } + a, _ := json.Marshal(plain) + b, _ := json.Marshal(composed.Resources) + if string(a) != string(b) { + t.Fatal("Compose and Declaration disagree on a converged node") + } +} + +// The table the installer raises and the controller declares, character for character. +func TestTheGuardIsExactlyThisTable(t *testing.T) { + const golden = `table inet mesh_guard {} +delete table inet mesh_guard +table inet mesh_guard { + chain prerouting { + type filter hook prerouting priority raw; policy accept; + iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop + } +} +` + if got := AsGuard([]int{15672, 5432}); got != golden { + t.Fatalf("the guard changed:\n%s", got) + } + if GuardResources(nil) != nil { + t.Fatal("a guard with nothing to guard is an empty set nft refuses to load") + } +} + +func TestAGuardedPortMustBeAPort(t *testing.T) { + if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[5432]}`)); err != nil { + t.Fatalf("guards is refused: %v", err) + } + if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[0]}`)); err == nil { + t.Fatal("guarding port 0 was accepted") + } +} + +func keys[V any](m map[string]V) []string { + return sortedKeys(m) +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 39d8fa3..d4f5ffa 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -128,6 +128,11 @@ type Rendering struct { // set, for what a consumer is told, and for what the runtime publishes — and nothing checked // that the three agreed. They are all derived from this. Ports map[string]map[int]int + + // Adopted says the node is adopted (novox/hq ADR 0100): the firewall found on it stays in + // force, so no module that loads a filter is declared there, and what the mesh needs + // reachable is declared as openings, with its own ports guarded by a table that only refuses. + Adopted bool } // machinePort is where a module's port lives on this machine, or the port itself when the mesh has @@ -257,6 +262,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri var out []map[string]any for _, m := range r.Modules { + if with.Adopted && m.Filtering != nil { + // Nothing of a module that loads a filter, on an adopted node: its table would drop + // by default and hold accepts, and the found firewall stays in force. Every resource, + // not only the rule set — its service must not run, and a node returned to adopted + // stops it by the ordinary removal of what is no longer declared. + continue + } resources := m.Resources // What the mesh computes for this module goes FIRST, before the module's own resources. @@ -567,9 +579,54 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri out = append(out, fact) } } + if with.Adopted { + // First, before anything a module declares: what the mesh needs reachable, then its guard. + // The order a machine applies is the order written here. + ours := Openings(rules, with.Foundation, Published(out)) + ours = append(ours, GuardResources(r.guarded(out, owner, with))...) + out = append(ours, out...) + } return out, nil } +// guarded is the machine ports of every guarded port of the modules here: where each module's +// container publishes it, as composed — or where the machine put it when no container does. +func (r Resolution) guarded(out []map[string]any, owner map[string]string, with Rendering) []int { + seen := map[int]bool{} + var ports []int + for _, m := range r.Modules { + for _, want := range m.Guards { + at := with.machinePort(m.Module, want) + for _, resource := range out { + if owner[fmt.Sprint(resource["id"])] != m.Module || + fmt.Sprint(resource["type"]) != "container" { + continue + } + listed, _ := resource["ports"].([]any) + for _, entry := range listed { + parts := strings.Split(strings.TrimSpace(fmt.Sprint(entry)), ":") + if len(parts) < 2 { + continue + } + inner, err := strconv.Atoi(strings.SplitN(parts[len(parts)-1], "/", 2)[0]) + if err != nil || inner != want { + continue + } + if outer, err := strconv.Atoi(parts[len(parts)-2]); err == nil { + at = outer + } + } + } + if !seen[at] { + seen[at] = true + ports = append(ports, at) + } + } + } + sort.Ints(ports) + return ports +} + // Contribution is one module telling the answer to a requirement what it needs from it. type Contribution struct { // From is the module that said it, so the provider and a person reading the file can tell diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 601c647..e39fa17 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -346,6 +346,14 @@ type Manifest struct { // that could only see its own ports would write a rule set that closed everything else. Filtering *Filtering `json:"filtering,omitempty"` + // Guards are ports of this module's the mesh refuses on an adopted node except from the + // private network and from the machine itself (novox/hq ADR 0100) — the store's port and the + // broker's management port. The ports the software uses; the mesh guards where the machine + // publishes them. On an adopted node the found firewall stays in force and the mesh loads no + // filter of its own, so this is what keeps them unreachable from outside whatever that + // firewall does. Ignored on a converged node, whose derived filter already closes them. + Guards []int `json:"guards,omitempty"` + // Facts are things only the mesh knows, written where this module asks for them. // // **The graph is the control plane's; how a machine uses it is the module's.** The mesh knows @@ -950,6 +958,12 @@ func ParseManifest(raw []byte) (Manifest, error) { "%s listens on %d over %q, which is tcp or udp", m.Module, l.Port, p)) } } + for _, port := range m.Guards { + if port < 1 || port > 65535 { + problems = append(problems, fmt.Sprintf( + "%s guards port %d, which is not a port", m.Module, port)) + } + } if c := m.Certificate; c != nil { if !strings.HasPrefix(c.Into, "/") { problems = append(problems, fmt.Sprintf( From 28894fa5bdf1921083289705cadedf5e4c755720 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:23:09 +0200 Subject: [PATCH 04/33] Keep what an adopted node reports holding, the firewall it found and what is reachable on it (hq ADR 0100) --- cmd/mesh-controller/adoption.go | 31 ++++++ internal/inventory/adoption.go | 99 ++++++++++++++++++- .../0030-what-an-adopted-node-reports.sql | 12 +++ internal/link/enrolment.go | 21 ++++ internal/link/heard_test.go | 43 ++++++++ internal/link/protocol.go | 42 +++++++- internal/link/protocol_test.go | 7 ++ 7 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 internal/inventory/migrations/0030-what-an-adopted-node-reports.sql diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index e898dbf..eec29d4 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -29,9 +29,40 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node } else { fmt.Printf(" taken %s\n", strings.Join(taken, ", ")) } + said, err := inv.AdoptionOf(ctx, node.Name) + if err != nil { + return err + } + if said.At.IsZero() { + fmt.Printf(" it has not yet said what it found\n") + return nil + } + fmt.Printf(" firewall found %s\n", orNone(said.Firewall)) + if len(said.Held) == 0 { + fmt.Printf(" holding nothing found\n") + } + for _, h := range said.Held { + // A held thing that changed is how a predecessor still writing is caught: said first. + line := fmt.Sprintf(" holds %-11s %s %s, for %s", h.Kind, h.Target, h.ID, h.Module) + if h.Changed != "" { + line += " — " + strings.ToUpper(h.Changed) + " by something other than the mesh" + } + fmt.Println(line) + if h.Kept != "" { + fmt.Printf(" %-17s original kept at %s\n", "", h.Kept) + } + } + fmt.Printf(" as of %s\n", said.At.Local().Format(time.DateTime)) return nil } +func orNone(s string) string { + if s == "" { + return "none reported" + } + return s +} + // adoptedNodes are the names of every adopted node, in the order given. func adoptedNodes(nodes []inventory.Node) []string { var out []string diff --git a/internal/inventory/adoption.go b/internal/inventory/adoption.go index 031bf52..feae77c 100644 --- a/internal/inventory/adoption.go +++ b/internal/inventory/adoption.go @@ -2,9 +2,13 @@ package inventory import ( "context" + "encoding/json" "errors" "fmt" "sort" + "time" + + "github.com/jackc/pgx/v5" ) // A node is adopted or converged (novox/hq ADR 0100). @@ -113,7 +117,9 @@ func (i *Inventory) Converge(ctx context.Context, nodeName string) ([]string, er return nil, err } if _, err := tx.Exec(ctx, - `update node set adopted = false, adopted_since = null, converged_at = now() where id = $1`, + `update node set adopted = false, adopted_since = null, converged_at = now(), + held = null, reachable = null + where id = $1`, node.ID); err != nil { return nil, err } @@ -147,3 +153,94 @@ func (i *Inventory) Taken(ctx context.Context, nodeName string) ([]string, error } return out, rows.Err() } + +// Held is one file or container an adopted node found and keeps as it was until its module is +// taken. The node's own account, kept as it said it. +type Held struct { + ID string `json:"id"` + Module string `json:"module"` + Kind string `json:"kind"` + Target string `json:"target"` + Since time.Time `json:"since"` + Changed string `json:"changed,omitempty"` + Kept string `json:"kept,omitempty"` +} + +// Reach is one thing reachable on an adopted node: a listening socket or a published port. +type Reach struct { + Protocol string `json:"protocol"` + Address string `json:"address"` + Port int `json:"port"` + By string `json:"by,omitempty"` + Published bool `json:"published,omitempty"` + ContainerPort int `json:"container-port,omitempty"` +} + +// Adoption is what an adopted node last said about adoption, and when. +type Adoption struct { + Held []Held + Firewall string + Reachable []Reach + // At is when it said so; zero when it never has. + At time.Time +} + +// RecordAdoption keeps what a node last reported about adoption, replacing what was there: the +// question is the machine as it is now. +func (i *Inventory) RecordAdoption(ctx context.Context, node string, held []Held, firewall string, + reachable []Reach) error { + heldRaw, err := json.Marshal(nonNil(held)) + if err != nil { + return err + } + reachRaw, err := json.Marshal(nonNil(reachable)) + if err != nil { + return err + } + _, err = i.store.Pool().Exec(ctx, + `update node set held = $2, firewall = nullif($3, ''), reachable = $4, + adoption_reported = now(), last_seen = now() + where id = $1`, node, heldRaw, firewall, reachRaw) + return err +} + +func nonNil[T any](s []T) []T { + if s == nil { + return []T{} + } + return s +} + +// AdoptionOf is what a node last reported about adoption. +func (i *Inventory) AdoptionOf(ctx context.Context, name string) (Adoption, error) { + var heldRaw, reachRaw []byte + var firewall *string + var at *time.Time + err := i.store.Pool().QueryRow(ctx, + `select held, firewall, reachable, adoption_reported from node where name = $1`, name). + Scan(&heldRaw, &firewall, &reachRaw, &at) + if errors.Is(err, pgx.ErrNoRows) { + return Adoption{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name) + } + if err != nil { + return Adoption{}, err + } + var out Adoption + if firewall != nil { + out.Firewall = *firewall + } + if at != nil { + out.At = *at + } + if len(heldRaw) > 0 { + if err := json.Unmarshal(heldRaw, &out.Held); err != nil { + return Adoption{}, err + } + } + if len(reachRaw) > 0 { + if err := json.Unmarshal(reachRaw, &out.Reachable); err != nil { + return Adoption{}, err + } + } + return out, nil +} diff --git a/internal/inventory/migrations/0030-what-an-adopted-node-reports.sql b/internal/inventory/migrations/0030-what-an-adopted-node-reports.sql new file mode 100644 index 0000000..047d457 --- /dev/null +++ b/internal/inventory/migrations/0030-what-an-adopted-node-reports.sql @@ -0,0 +1,12 @@ +-- What an adopted node last reported about adoption (novox/hq ADR 0100): the files and containers +-- it found and holds until their module is taken, the firewall it found, and what is reachable on +-- the machine now — which converging it previews, so nothing closes without being named first. +-- +-- The last report, replaced, like what a node owns: the question is the machine as it is now. +-- Kept apart from node_report because a node reports it on its own schedule, when what it holds +-- changes, and not only after an apply. + +alter table node add column held jsonb; +alter table node add column firewall text; +alter table node add column reachable jsonb; +alter table node add column adoption_reported timestamptz; diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index f39bc52..d4d71aa 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -199,6 +199,27 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (err error) { return err } + // What an adopted node holds, which firewall it found, and what is reachable on it (novox/hq + // ADR 0100). Recorded whenever a report carries any of it — a node reports these on its own + // schedule, when what it holds changes, not only after an apply — and never cleared by a + // report that carries none, which is every bare word that the node is there. An adopted node + // always names its firewall, so a report from one replaces all three, emptied held included. + if len(report.Held) > 0 || report.Firewall != "" || len(report.Reachable) > 0 { + held := make([]inventory.Held, 0, len(report.Held)) + for _, h := range report.Held { + held = append(held, inventory.Held{ID: h.ID, Module: h.Module, Kind: h.Kind, + Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept}) + } + reachable := make([]inventory.Reach, 0, len(report.Reachable)) + for _, r := range report.Reachable { + reachable = append(reachable, inventory.Reach{Protocol: r.Protocol, Address: r.Address, + Port: r.Port, By: r.By, Published: r.Published, ContainerPort: r.ContainerPort}) + } + if err := e.Inventory.RecordAdoption(ctx, node.ID, held, report.Firewall, reachable); err != nil { + return err + } + } + // A bare word that a node is there is not an account of what the machine did or holds: it // moves last_seen and touches nothing else. This arrives every minute (link.AliveEvery), // while a real report is rare, so recording it as one would overwrite the node's last real diff --git a/internal/link/heard_test.go b/internal/link/heard_test.go index 01fb10b..3a89b21 100644 --- a/internal/link/heard_test.go +++ b/internal/link/heard_test.go @@ -175,3 +175,46 @@ func TestAFailureDoesNotBecomeTheAccountOfWhatTheMachineHolds(t *testing.T) { t.Fatalf("a partial report replaced the account of what the machine holds: %v", owned) } } + +// novox/hq ADR 0100: what an adopted node holds, the firewall it found and what is reachable on it +// are kept from the report that carries them, and a bare word that the node is there wipes none. +func TestWhatAnAdoptedNodeHoldsIsKeptAndAnAliveWordDoesNotWipeIt(t *testing.T) { + inv, _, _ := heardFrom(t, link.Report{ + Node: "anchor", Applied: []string{"hello-web.served"}, Firewall: "ufw", + Held: []link.Held{{ID: "hello-web.page", Module: "hello-web", Kind: "file", + Target: "/var/lib/hello-web/index.html", Changed: "rewritten", Kept: "/var/lib/mesh/kept/x"}}, + Reachable: []link.Reach{{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", + Published: true, ContainerPort: 80}}, + }) + ctx := context.Background() + check := func(when string) { + t.Helper() + got, err := inv.AdoptionOf(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + if got.Firewall != "ufw" || len(got.Held) != 1 || got.Held[0].Changed != "rewritten" || + len(got.Reachable) != 1 || got.Reachable[0].ContainerPort != 80 || got.At.IsZero() { + t.Fatalf("%s: what the node said is not what was kept: %+v", when, got) + } + } + check("after the report") + + if err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{Node: "anchor"}); err != nil { + t.Fatal(err) + } + check("after an alive word") + + // A reconcile report carrying only adoption is recorded, though it applied nothing. + if err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{Node: "anchor", + Firewall: "ufw"}); err != nil { + t.Fatal(err) + } + got, err := inv.AdoptionOf(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + if len(got.Held) != 0 || got.Firewall != "ufw" { + t.Fatalf("a report from an adopted node holding nothing did not empty held: %+v", got) + } +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index ba6dd7c..0a7227d 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -6,7 +6,10 @@ // traffic between them, each receiving half of what it expects. That has happened here before. package link -import "encoding/base64" +import ( + "encoding/base64" + "time" +) // Exchange is where nodes publish everything they have to say. const Exchange = "mesh" @@ -116,6 +119,43 @@ type Report struct { // Declared is the digest of the declaration this report is about — the same bytes, hashed // the same way, as the `sent` digest the mesh recorded. Which declaration, not when. Declared string `json:"declared,omitempty"` + + // Held is what an adopted node found and is keeping as it was until its module is taken + // (novox/hq ADR 0100). Without it an adopted node reads as converged. + Held []Held `json:"held,omitempty"` + // Firewall is the firewall found on the machine — "ufw" or "none" — and empty on a node that + // was never asked, which is every converged one. + Firewall string `json:"firewall,omitempty"` + // Reachable is what can be reached on the machine now: every listening socket and every + // published container port. Only an adopted node reports it; it is what converging previews. + Reachable []Reach `json:"reachable,omitempty"` +} + +// Held is one file or container found on an adopted node and kept as it was. +type Held struct { + ID string `json:"id"` + Module string `json:"module"` + Kind string `json:"kind"` + Target string `json:"target"` + Since time.Time `json:"since"` + // Changed is what something other than the mesh did to it since — rewritten, stopped, + // replaced or gone — and empty while it is as found. + Changed string `json:"changed,omitempty"` + // Kept is where a file's original was kept. + Kept string `json:"kept,omitempty"` +} + +// Reach is one thing reachable on the machine: a listening socket, or a published container port. +type Reach struct { + Protocol string `json:"protocol"` + Address string `json:"address"` + Port int `json:"port"` + // By is what holds it — a process, or a container's name. + By string `json:"by,omitempty"` + // Published is a container port the runtime publishes, reached on the forwarded path; its + // container's own port is ContainerPort. + Published bool `json:"published,omitempty"` + ContainerPort int `json:"container-port,omitempty"` } // EnrolReply is what the mesh says back. diff --git a/internal/link/protocol_test.go b/internal/link/protocol_test.go index 6c0ff37..53ef4ce 100644 --- a/internal/link/protocol_test.go +++ b/internal/link/protocol_test.go @@ -16,6 +16,13 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { {Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}}, {Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"}, []string{"node", "applied", "failed", "refused"}}, + {Report{Node: "n", Held: []Held{{ID: "m.f"}}, Firewall: "ufw", Reachable: []Reach{{Port: 1}}}, + []string{"node", "held", "firewall", "reachable"}}, + {Held{ID: "m.f", Module: "m", Kind: "file", Target: "/f", Changed: "rewritten", Kept: "/k"}, + []string{"id", "module", "kind", "target", "since", "changed", "kept"}}, + {Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, + ContainerPort: 80}, + []string{"protocol", "address", "port", "by", "published", "container-port"}}, {EnrolRequest{Node: "n", Secret: "s", PublicKey: []byte("k")}, []string{"node", "secret", "public_key"}}, } { From 1ea84f8b8f20e5ef8c4155ffb3df596b8df1c698 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:27:35 +0200 Subject: [PATCH 05/33] Take a module, converge a node after a preview, and return it to adopted, from the command line and the API (hq ADR 0100) --- cmd/mesh-controller/adopting_test.go | 255 ++++++++++++++++++++ cmd/mesh-controller/adoption.go | 342 +++++++++++++++++++++++++++ cmd/mesh-controller/api.go | 31 ++- cmd/mesh-controller/main.go | 9 + cmd/mesh-controller/plan.go | 56 +++-- internal/catalogue/declaration.go | 29 ++- 6 files changed, 683 insertions(+), 39 deletions(-) create mode 100644 cmd/mesh-controller/adopting_test.go diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go new file mode 100644 index 0000000..1936b9a --- /dev/null +++ b/cmd/mesh-controller/adopting_test.go @@ -0,0 +1,255 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/overlay" +) + +// novox/hq ADR 0100: taking a module is its cutover; converging a node is one act, previewed, and +// refused while a found container is held; returning to adopted keeps what was taken. + +// anAdoptedAnchor is aMesh with the anchor adopted, running a served module the predecessor also +// runs and a module with only a file, and a filter module in the catalogue. +func anAdoptedAnchor(t *testing.T) (*stores, *[]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "hello-web", Version: "1", + Listens: []catalogue.Listening{{Port: 8080, From: catalogue.FromEverywhere}}, + Resources: []map[string]any{ + {"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hi"}, + {"id": "server", "type": "container", "name": "hello-web", "ports": []any{"8080:80"}, + "image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)}, + }}) + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{ + {"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"}, + }}) + register(t, open, catalogue.Manifest{Module: "nftables", Version: "1", + Filtering: &catalogue.Filtering{Into: "/etc/nftables.conf"}, + Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service", + "state": "running", "restart-on": []any{"filtering"}}}}) + if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + for _, m := range []string{"hello-web", "notes"} { + if _, err := assign(ctx, open, "anchor", m); err != nil { + t.Fatal(err) + } + } + sent := &[]string{} + saved := sendNodes + sendNodes = func(_ context.Context, _ *stores, names []string) error { + *sent = append(*sent, names...) + return nil + } + t.Cleanup(func() { sendNodes = saved }) + return open, sent +} + +// reportsHolding has the anchor report, on what it was last sent, holding what is given. +func reportsHolding(t *testing.T, open *stores, held ...link.Held) { + t.Helper() + ctx := t.Context() + body, err := composed(t, open, "anchor").Body() + if err != nil { + t.Fatal(err) + } + record, err := open.inventory.NodeByName(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + t.Fatal(err) + } + if err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{ + Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body), + Firewall: "ufw", Held: held, + Reachable: []link.Reach{ + {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}, + {Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, + ContainerPort: 80}, + {Protocol: "tcp", Address: "0.0.0.0", Port: 5000, By: "predecessor-registry", + Published: true, ContainerPort: 5000}, + {Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker", + Published: true, ContainerPort: 15672}, + }, + }); err != nil { + t.Fatal(err) + } +} + +var ( + heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container", + Target: "hello-web", Since: time.Now()} + heldFile = link.Held{ID: "notes.conf", Module: "notes", Kind: "file", + Target: "/etc/notes.conf", Since: time.Now(), Kept: "/var/lib/mesh-host/kept/abc-notes.conf"} +) + +func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) { + open, _ := anAdoptedAnchor(t) + if _, err := take(t.Context(), open, "anchor", "nftables"); !errors.Is(err, inventory.ErrNotAssigned) { + t.Fatalf("taking an unassigned module gave %v", err) + } + if _, err := take(t.Context(), open, "laptop", "network"); !errors.Is(err, inventory.ErrNotAdopted) { + t.Fatalf("taking on a converged node gave %v", err) + } +} + +func TestConvergingIsRefusedOnAPreviewThatWouldBeStale(t *testing.T) { + open, sent := anAdoptedAnchor(t) + _, err := converge(t.Context(), open, "anchor", false, "") + if err == nil || !strings.Contains(err.Error(), "has not reported") { + t.Fatalf("a node that never reported was previewed: %v", err) + } + if len(*sent) != 0 { + t.Fatal("a refused converge sent something") + } +} + +func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) { + open, sent := anAdoptedAnchor(t) + reportsHolding(t, open, heldContainer, heldFile) + _, err := converge(t.Context(), open, "anchor", true, "") + if err == nil || !strings.Contains(err.Error(), "take anchor hello-web once its data has moved") { + t.Fatalf("the flip was not refused while hello-web holds its found container: %v", err) + } + if n, _ := open.inventory.NodeByName(t.Context(), "anchor"); !n.Adopted || len(*sent) != 0 { + t.Fatal("a refused flip changed something") + } +} + +func TestTakingNamesWhatItReplaces(t *testing.T) { + open, _ := anAdoptedAnchor(t) + reportsHolding(t, open, heldContainer, heldFile) + said, err := take(t.Context(), open, "anchor", "hello-web") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, "container hello-web (hello-web.server)") || + !strings.Contains(said, "push anchor") { + t.Fatalf("taking did not say what it replaces and what to run:\n%s", said) + } +} + +func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) { + open, sent := anAdoptedAnchor(t) + ctx := t.Context() + if _, err := take(ctx, open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + reportsHolding(t, open, heldFile) + + preview, err := converge(ctx, open, "anchor", false, "") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "tcp/8080 hello-web (published, container port 80)", + "declared by hello-web (from anywhere)", + "WILL CLOSE — no module assigned here declares it", + "ssh is never closed", + "notes\n replacing the found file /etc/notes.conf (original kept at", + "assigns nftables", + "the found firewall (ufw) is disabled, never flushed", + } { + if !strings.Contains(preview, want) { + t.Errorf("the preview does not say %q:\n%s", want, preview) + } + } + if strings.Contains(preview, "15672") { + t.Errorf("a loopback listener is in the preview:\n%s", preview) + } + for _, line := range strings.Split(preview, "\n") { + if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") { + t.Errorf("an undeclared published port is not said to close: %s", line) + } + } + if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 { + t.Fatal("the preview changed something") + } + + if _, err := converge(ctx, open, "anchor", true, ""); err != nil { + t.Fatal(err) + } + n, _ := open.inventory.NodeByName(ctx, "anchor") + if n.Adopted { + t.Fatal("converge --yes left the node adopted") + } + taken, _ := open.inventory.Taken(ctx, "anchor") + if !reflect.DeepEqual(taken, []string{"hello-web", overlay.Name, "nftables", "notes"}) { + t.Fatalf("the flip took %v", taken) + } + if !reflect.DeepEqual(*sent, []string{"anchor"}) { + t.Fatalf("the flip sent %v", *sent) + } + declared := composed(t, open, "anchor") + if declared.Adoption != nil { + t.Fatal("a converged node is still sent an adoption envelope") + } + if !hasID(declared.Resources, "nftables.filtering") { + t.Fatal("the converged node is not declared the mesh's filter") + } + + if _, err := adopt(ctx, open, "anchor"); err != nil { + t.Fatal(err) + } + if _, err := adopt(ctx, open, "anchor"); err == nil { + t.Fatal("adopting an adopted node was not refused") + } + again, _ := open.inventory.Taken(ctx, "anchor") + if !reflect.DeepEqual(again, taken) { + t.Fatalf("returning to adopted lost what was taken: %v", again) + } + declared = composed(t, open, "anchor") + if declared.Adoption == nil || len(declared.Adoption.Untaken) != 0 { + t.Fatalf("returned to adopted, the envelope is %+v", declared.Adoption) + } + if hasID(declared.Resources, "nftables.filtering") || hasID(declared.Resources, "nftables.load") { + t.Fatal("returned to adopted, the mesh's filter is still declared") + } +} + +func hasID(resources []map[string]any, id string) bool { + for _, r := range resources { + if r["id"] == id { + return true + } + } + return false +} + +// The command API refuses a flip exactly as the command line does, in the same words. +func TestTheApiRefusesTheFlipInTheCommandLinesWords(t *testing.T) { + open, _ := anAdoptedAnchor(t) + reportsHolding(t, open, heldContainer) + _, direct := converge(t.Context(), open, "anchor", true, "") + if direct == nil { + t.Fatal("the flip was not refused") + } + got := asking(t, letIn{}, "POST", "/converge", `{"node":"anchor","yes":true}`) + if got.Code != http.StatusConflict { + t.Fatalf("got %d: %s", got.Code, got.Body.String()) + } + var said map[string]any + if err := json.Unmarshal(got.Body.Bytes(), &said); err != nil { + t.Fatal(err) + } + if said["refused"] != direct.Error() { + t.Fatalf("the API said %q and the command line %q", said["refused"], direct.Error()) + } + if got := asking(t, letIn{}, "POST", "/take", `{"node":"anchor"}`); got.Code != http.StatusBadRequest { + t.Fatalf("a take naming no module got %d", got.Code) + } +} diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index eec29d4..bd03494 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -2,10 +2,15 @@ package main import ( "context" + "errors" + "flag" "fmt" + "slices" + "sort" "strings" "time" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" ) @@ -73,3 +78,340 @@ func adoptedNodes(nodes []inventory.Node) []string { } return out } + +// The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the +// command API alike, so a refusal is the same refusal in the same words at both (ADR 0035). + +// sendNodes sends the named machines what they should be now. A variable so a test can see what +// an act would send without a broker. +var sendNodes = sendTo + +// DefaultFilter is the module converging a node assigns to load the mesh's derived filter. +const DefaultFilter = "nftables" + +// take is a module's cutover on an adopted node: the operator's act, done when that module's data +// has moved. From the next push its resources converge there like any other, replacing what the +// node found and holds for it. +func take(ctx context.Context, open *stores, node, module string) (string, error) { + inv := open.inventory + assigned, err := inv.Assigned(ctx, node) + if err != nil { + return "", err + } + if !slices.Contains(assigned, module) { + if plan, _, err := planFor(ctx, open, node); err == nil { + if why, runs := plan.Because[module]; runs { + return "", fmt.Errorf("%w: %s runs on %s because %s — assign it to %s to take it", + inventory.ErrNotAssigned, module, node, why, node) + } + } + } + if err := inv.Take(ctx, node, module); err != nil { + return "", err + } + said := fmt.Sprintf("%s is taken on %s", module, node) + reported, err := inv.AdoptionOf(ctx, node) + if err != nil { + return "", err + } + var replaces []string + for _, h := range reported.Held { + if h.Module == module { + replaces = append(replaces, fmt.Sprintf(" %s %s (%s)", h.Kind, h.Target, h.ID)) + } + } + if len(replaces) > 0 { + said += "; the next push replaces what the node found and holds for it:\n" + + strings.Join(replaces, "\n") + } + return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil +} + +// converge previews, and with yes makes, the flip of an adopted node to converged: every module it +// runs is taken, the filter module is assigned to load the mesh's derived filter in place of the +// guard, and the found firewall is retired — disabled, never flushed — by the host. +// +// Refused while an assigned module still holds a found container: each service is taken on its +// own, when its data has moved, never by the flip. And refused on a preview that would be stale: +// what is reachable is the node's last account, so that account must be of what it was last sent. +func converge(ctx context.Context, open *stores, node string, yes bool, filter string) (string, error) { + inv := open.inventory + if filter == "" { + filter = DefaultFilter + } + record, err := inv.NodeByName(ctx, node) + if err != nil { + return "", err + } + if !record.Adopted { + return "", fmt.Errorf("%s is converged already; there is nothing to flip", node) + } + reports, err := inv.LastReports(ctx) + if err != nil { + return "", err + } + current := false + for _, r := range reports { + if r.Node == node { + current = r.Current + } + } + reported, err := inv.AdoptionOf(ctx, node) + if err != nil { + return "", err + } + if !current || reported.At.IsZero() { + return "", fmt.Errorf("%s has not reported on the declaration it was last sent, so what it "+ + "says is reachable may not be the machine as it is: run `push %s --wait 2m` and "+ + "converge once it has applied", node, node) + } + + plan, settings, err := planFor(ctx, open, node) + if err != nil { + return "", err + } + runs := map[string]bool{} + for _, m := range plan.Modules { + runs[m.Module] = true + } + var holding []string + for _, h := range reported.Held { + if h.Kind == "container" && runs[h.Module] { + holding = append(holding, fmt.Sprintf(" %s holds the found container %s — take %s %s "+ + "once its data has moved", h.Module, h.Target, node, h.Module)) + } + } + if len(holding) > 0 { + sort.Strings(holding) + return "", fmt.Errorf("%s still holds what it found, and a service is taken on its own, "+ + "never by the flip:\n%s", node, strings.Join(holding, "\n")) + } + + shelf, err := inv.Catalogue(ctx) + if err != nil { + return "", err + } + filterModule, known := shelf[filter] + if !known { + return "", fmt.Errorf("%w: %s — converging assigns it to load the mesh's filter; "+ + "name another with --filter", inventory.ErrNoSuchModule, filter) + } + if filterModule.Filtering == nil { + return "", fmt.Errorf("%s loads no filter of the mesh's; name a module that does with --filter", + filter) + } + + gens, err := generators(ctx, open) + if err != nil { + return "", err + } + with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading) + if err != nil { + return "", err + } + rules, err := plan.Rules(with) + if err != nil { + return "", err + } + taken, err := inv.Taken(ctx, node) + if err != nil { + return "", err + } + preview := previewOf(node, reported, rules, with.Foundation, plan, taken, filter, runs[filter]) + if !yes { + return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes` to do it.", node), nil + } + + // The flip. The filter first, and only kept if the node still resolves with it: a node that + // cannot be worked out would be sent nothing, and would sit with its guard and no filter. + assigned, err := inv.Assigned(ctx, node) + if err != nil { + return "", err + } + if !slices.Contains(assigned, filter) { + if err := inv.Assign(ctx, node, filter); err != nil { + return "", err + } + if _, _, err := planFor(ctx, open, node); err != nil { + _ = inv.Unassign(ctx, node, filter) + return "", fmt.Errorf("%s cannot run %s, so it was not converged: %w", node, filter, err) + } + } + took, err := inv.Converge(ctx, node) + if err != nil { + return "", err + } + said := preview + fmt.Sprintf("\n\n%s is converged", node) + if len(took) > 0 { + said += "; took " + strings.Join(took, ", ") + } + if err := sendNodes(ctx, open, []string{node}); err != nil { + return said + "\n and it could not be sent: run `push " + node + "`", err + } + return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil +} + +// previewOf is what converging a node will change, before it changes it. +func previewOf(node string, reported inventory.Adoption, rules []catalogue.Rule, foundation []int, + plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) string { + var b strings.Builder + fmt.Fprintf(&b, "converging %s\n", node) + fmt.Fprintf(&b, "\n reachable on the machine now, as it reported at %s:\n", + reported.At.Local().Format(time.DateTime)) + for _, r := range reported.Reachable { + if loopback(r.Address) { + continue + } + what := fmt.Sprintf("%s/%d", r.Protocol, r.Port) + if r.By != "" { + what += " " + r.By + } + if r.Published { + what += fmt.Sprintf(" (published, container port %d)", r.ContainerPort) + } + fmt.Fprintf(&b, " %-44s %s\n", what, fate(r, rules, foundation)) + } + if len(reported.Reachable) == 0 { + b.WriteString(" nothing reported\n") + } + + isTaken := map[string]bool{} + for _, m := range taken { + isTaken[m] = true + } + var takes []string + for _, m := range plan.Modules { + if !isTaken[m.Module] { + takes = append(takes, m.Module) + } + } + if !filterAssigned && !isTaken[filter] { + takes = append(takes, filter) + } + sort.Strings(takes) + b.WriteString("\n the flip takes:\n") + if len(takes) == 0 { + b.WriteString(" nothing — every module is taken already\n") + } + for _, m := range takes { + fmt.Fprintf(&b, " %s\n", m) + for _, h := range reported.Held { + if h.Module == m && h.Kind == "file" { + fmt.Fprintf(&b, " replacing the found file %s", h.Target) + if h.Kept != "" { + fmt.Fprintf(&b, " (original kept at %s)", h.Kept) + } + b.WriteString("\n") + } + } + } + if !filterAssigned { + fmt.Fprintf(&b, "\n and assigns %s, which loads the mesh's filter in place of its guard\n", filter) + } + fw := reported.Firewall + if fw == "" || fw == "none" { + b.WriteString(" no firewall was found on the machine; the mesh's filter is its first\n") + } else { + fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw) + } + return strings.TrimRight(b.String(), "\n") +} + +// fate is what the derived filter does to one reachable thing: which module declares it and from +// where, or that it will close. +func fate(r inventory.Reach, rules []catalogue.Rule, foundation []int) string { + if r.Protocol == "tcp" && r.Port == catalogue.SSHPort { + return "stays open — ssh is never closed" + } + for _, port := range foundation { + if r.Protocol == "tcp" && r.Port == port { + return "stays open — the mesh's own, from anywhere" + } + } + for _, rule := range rules { + if rule.Port != r.Port || rule.Protocol != r.Protocol { + continue + } + if rule.From == catalogue.FromMachine { + return fmt.Sprintf("WILL CLOSE to the network — declared by %s for this machine only", + strings.Join(rule.Because, ", ")) + } + return fmt.Sprintf("declared by %s (from %s)", strings.Join(rule.Because, ", "), rule.From) + } + return "WILL CLOSE — no module assigned here declares it" +} + +// loopback is an address nothing off the machine reaches. +func loopback(address string) bool { + a := strings.Trim(address, "[]") + return strings.HasPrefix(a, "127.") || a == "::1" || a == "localhost" +} + +// adopt returns a converged node to adopted: the mesh's filter is unloaded, the guard restored, +// the found firewall enabled again and the openings converged through it once more. What was +// taken stays taken. +func adopt(ctx context.Context, open *stores, node string) (string, error) { + inv := open.inventory + record, err := inv.NodeByName(ctx, node) + if err != nil { + return "", err + } + if record.Adopted { + return "", fmt.Errorf("%s is adopted already", node) + } + if err := inv.SetAdopted(ctx, node, true); err != nil { + return "", err + } + said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node) + if err := sendNodes(ctx, open, []string{node}); err != nil { + return said + "\n and it could not be sent: run `push " + node + "`", err + } + return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil +} + +// takeCommand, convergeCommand and adoptCommand are the command line's adapters to the acts above. +func takeCommand(ctx context.Context, args []string) error { + if len(args) != 2 { + return errors.New("take ") + } + return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, args[0], args[1]) }) +} + +func convergeCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("converge", flag.ContinueOnError) + yes := set.Bool("yes", false, "do it; without it, only the preview") + filter := set.String("filter", DefaultFilter, "the module that loads the mesh's filter") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 1 { + return errors.New("converge [--yes] [--filter nftables]") + } + return runAct(ctx, func(open *stores) (string, error) { + return converge(ctx, open, positionals[0], *yes, *filter) + }) +} + +func adoptCommand(ctx context.Context, args []string) error { + if len(args) != 1 { + return errors.New("adopt ") + } + return runAct(ctx, func(open *stores) (string, error) { return adopt(ctx, open, args[0]) }) +} + +func runAct(ctx context.Context, act func(*stores) (string, error)) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + said, err := act(open) + if said != "" { + fmt.Println(said) + } + if err != nil && said != "" { + fmt.Println() + } + return err +} diff --git a/cmd/mesh-controller/api.go b/cmd/mesh-controller/api.go index 1d97030..182c7c9 100644 --- a/cmd/mesh-controller/api.go +++ b/cmd/mesh-controller/api.go @@ -94,19 +94,29 @@ func (n notYet) Who(*http.Request) (string, error) { func commands(who Authenticator) http.Handler { mux := http.NewServeMux() - mux.HandleFunc("POST /assign", acting(who, func(ctx context.Context, open *stores, in request) (string, error) { + mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) { return assign(ctx, open, in.Node, in.Module) })) - mux.HandleFunc("POST /unassign", acting(who, func(ctx context.Context, open *stores, in request) (string, error) { + mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) { return unassign(ctx, open, in.Node, in.Module) })) + // Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`. + mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) { + return take(ctx, open, in.Node, in.Module) + })) + mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) { + return converge(ctx, open, in.Node, in.Yes, in.Filter) + })) + mux.HandleFunc("POST /adopt", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) { + return adopt(ctx, open, in.Node) + })) // Anything else is said plainly, because a command surface answering 404 to a verb somebody // expected is indistinguishable from one that is down. mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { refuse(w, http.StatusNotFound, fmt.Errorf( - "%s %s is not something this mesh can be asked; it accepts POST /assign and "+ - "POST /unassign", r.Method, r.URL.Path)) + "%s %s is not something this mesh can be asked; it accepts POST /assign, "+ + "POST /unassign, POST /take, POST /converge and POST /adopt", r.Method, r.URL.Path)) }) return mux } @@ -114,11 +124,16 @@ func commands(who Authenticator) http.Handler { type request struct { Node string `json:"node"` Module string `json:"module"` + // Yes and Filter are converge's: do it rather than preview it, and which module loads the + // mesh's filter. + Yes bool `json:"yes,omitempty"` + Filter string `json:"filter,omitempty"` } // acting is the shape every route shares: authenticate, read, act, answer. func acting( who Authenticator, + needsModule bool, do func(context.Context, *stores, request) (string, error), ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -131,8 +146,12 @@ func acting( refuse(w, http.StatusBadRequest, fmt.Errorf("this is not a request this understands: %w", err)) return } - if in.Node == "" || in.Module == "" { - refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`)) + if in.Node == "" || (needsModule && in.Module == "") { + if needsModule { + refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`)) + } else { + refuse(w, http.StatusBadRequest, errors.New(`"node" is needed`)) + } return } diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 00cc99b..0fc5b2c 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -98,6 +98,12 @@ func run() error { return moduleCommand(ctx, args[1:]) case "assign", "unassign": return assignCommand(ctx, args[0], args[1:]) + case "take": + return takeCommand(ctx, args[1:]) + case "converge": + return convergeCommand(ctx, args[1:]) + case "adopt": + return adoptCommand(ctx, args[1:]) case "settings": return settingsCommand(ctx, args[1:]) case "secret": @@ -155,6 +161,9 @@ func usage() { api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here assign put a module on a node unassign take it off + take cut a module over on an adopted node, once its data has moved + converge [--yes] [--filter nftables] preview, then make, an adopted node converged + adopt return a converged node to adopted; what was taken stays taken settings set what a module's config should say, for the whole mesh settings set --node ...or for one machine settings clear [--node ] take a layer away diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 3781267..919817f 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -318,10 +318,30 @@ const ( func declarationWith(ctx context.Context, open *stores, node string, plan catalogue.Resolution, settings catalogue.SettingsBy, gens map[string]catalogue.Generator, choosing Choosing) (sendable, error) { + with, record, err := renderingFor(ctx, open, node, plan, settings, gens, choosing) + if err != nil { + return sendable{}, err + } + composed, err := plan.Compose(with) + if err != nil { + return sendable{}, err + } + // And what was taken on it, said in every declaration it is sent from this one place. + adoption, err := adoptionOf(ctx, open.inventory, record, plan, composed) + if err != nil { + return sendable{}, err + } + return sendable{Resources: composed.Resources, Adoption: adoption}, nil +} + +// renderingFor is everything a node's declaration is composed with, and the node's record. +func renderingFor(ctx context.Context, open *stores, node string, + plan catalogue.Resolution, settings catalogue.SettingsBy, + gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) { inv := open.inventory grants, err := grantsFor(ctx, open, node) if err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } // Where this machine puts what each module needs reachable (novox/hq ADR 0038). // @@ -334,7 +354,7 @@ func declarationWith(ctx context.Context, open *stores, node string, if choosing == Reading { held, err := inv.PortsFor(ctx, node) if err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } for _, a := range held { if already[a.Module] == nil { @@ -358,7 +378,7 @@ func declarationWith(ctx context.Context, open *stores, node string, case mayAssign && choosing == Allocating: at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed) if err != nil { - return sendable{}, fmt.Errorf( + return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf( "%s needs %d reachable on %s and it could not be assigned: %w", m.Module, l.Port, node, err) } @@ -399,7 +419,7 @@ func declarationWith(ctx context.Context, open *stores, node string, } } if err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } if needed[m.Module] == nil { needed[m.Module] = map[string]string{} @@ -417,7 +437,7 @@ func declarationWith(ctx context.Context, open *stores, node string, } issued, meshCA, err := certificateFor(ctx, open, node) if err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } certificate, authority = issued, meshCA break @@ -430,11 +450,11 @@ func declarationWith(ctx context.Context, open *stores, node string, // One reading of the catalogue for the three questions below that resolve the whole mesh. shelf, err := inv.Catalogue(ctx) if err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } private, err := onThePrivateNetwork(ctx, inv, shelf) if err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } // And every machine's name, so a container can reach one. The same set that writes the @@ -442,7 +462,7 @@ func declarationWith(ctx context.Context, open *stores, node string, // about where another machine is. names, err := namesInTheMesh(ctx, inv, shelf) if err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } // And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the @@ -451,7 +471,7 @@ func declarationWith(ctx context.Context, open *stores, node string, // to serve and knows nothing about what they mean. routes, err := routeNamesInTheMesh(ctx, open) if err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } for name, at := range routes { names[name] = at @@ -480,7 +500,7 @@ func declarationWith(ctx context.Context, open *stores, node string, continue } if kept, err = inv.OperatorExport(ctx); err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } break } @@ -489,21 +509,13 @@ func declarationWith(ctx context.Context, open *stores, node string, // the declaration carries openings and the mesh's guard in place of a filter. record, err := inv.NodeByName(ctx, node) if err != nil { - return sendable{}, err + return catalogue.Rendering{}, inventory.Node{}, err } - composed, err := plan.Compose(catalogue.Rendering{ + return catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, - Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted}) - if err != nil { - return sendable{}, err - } - // And what was taken on it, said in every declaration it is sent from this one place. - adoption, err := adoptionOf(ctx, inv, record, plan, composed) - if err != nil { - return sendable{}, err - } - return sendable{Resources: composed.Resources, Adoption: adoption}, nil + Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted, + }, record, nil } // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index d4f5ffa..59faa42 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -244,17 +244,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri // Once, from every module's listens -- not per module. A module receiving only its own ports // would write a rule set that closed every other module on the machine. Each module's per-node // exposure settings override its listens' source first (novox/hq ADR 0046). - exposure := map[string]map[int]string{} - for _, m := range r.Modules { - e, err := Exposure(m, with.Settings[m.Module]) - if err != nil { - return nil, err - } - if e != nil { - exposure[m.Module] = e - } - } - rules, err := r.Filtering(with.Generators, with.Ports, exposure) + rules, err := r.Rules(with) if err != nil { return nil, err } @@ -627,6 +617,23 @@ func (r Resolution) guarded(out []map[string]any, owner map[string]string, with return ports } +// Rules is the rule set this node's filter is derived from: every module's listens, what was +// computed for this machine, and each module's per-node exposure. The same answer whether the node +// is adopted or converged — the one loads it as a filter, the other declares it as openings. +func (r Resolution) Rules(with Rendering) ([]Rule, error) { + exposure := map[string]map[int]string{} + for _, m := range r.Modules { + e, err := Exposure(m, with.Settings[m.Module]) + if err != nil { + return nil, err + } + if e != nil { + exposure[m.Module] = e + } + } + return r.Filtering(with.Generators, with.Ports, exposure) +} + // Contribution is one module telling the answer to a requirement what it needs from it. type Contribution struct { // From is the module that said it, so the provider and a person reading the file can tell From dbf62b5212096643c5528e78f9a01d7a78556f6e Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:31:07 +0200 Subject: [PATCH 06/33] Read the foundation's ports from each node's settings, wherever a port is used (hq ADR 0100) --- cmd/mesh-controller/plan.go | 37 +++++++++-- cmd/mesh-controller/sendable_test.go | 57 +++++++++++++++++ internal/catalogue/adoption_test.go | 60 ++++++++++++++++++ internal/catalogue/declaration.go | 37 ++++++++++- internal/catalogue/filtering.go | 92 ++++++++++++++++++++++++++++ internal/catalogue/settings.go | 10 +++ internal/inventory/catalogue.go | 2 +- internal/inventory/ports.go | 42 +++++++++++++ internal/inventory/ports_test.go | 21 +++++++ 9 files changed, 352 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 919817f..48ff6d9 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -233,12 +233,19 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory, if err != nil { return catalogue.World{}, err } + layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module) + if err != nil { + return catalogue.World{}, err + } + // A port that node was given is where its consumers reach it (novox/hq ADR + // 0100). Unreadable given ports are that node's refusal to report, not this one's. + if given, err := catalogue.GivenPorts(m, layers); err == nil { + for wanted, at := range given { + assigned[wanted] = at + } + } serves := catalogue.ServedOn(m, name, assigned) if len(serves) > 0 { - layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module) - if err != nil { - return catalogue.World{}, err - } serves, err = catalogue.Settle(serves, layers) if err != nil { return catalogue.World{}, err @@ -364,9 +371,30 @@ func renderingFor(ctx context.Context, open *stores, node string, } } + // The machine ports this node was given for its modules (novox/hq ADR 0100): the foundation's + // ports, as genesis chose them. A given port wins over anything assigned and over a manifest's + // own long-form mapping. + given := map[string]map[int]int{} + for _, m := range plan.Modules { + g, err := catalogue.GivenPorts(m, settings[m.Module]) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + if g != nil { + given[m.Module] = g + } + } + ports := map[string]map[int]int{} for _, m := range plan.Modules { for _, l := range m.Listens { + if at, isGiven := given[m.Module][l.Port]; isGiven { + if ports[m.Module] == nil { + ports[m.Module] = map[int]int{} + } + ports[m.Module][l.Port] = at + continue + } // **Only a port the module actually publishes is the mesh's to move.** A container's // mapping is the thing that translates; without one the software binds what it binds, // and an assignment would not move the service — it would open the wrong number in the @@ -515,6 +543,7 @@ func renderingFor(ctx context.Context, open *stores, node string, Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted, + Given: given, }, record, nil } diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go index bf052b9..cb82110 100644 --- a/cmd/mesh-controller/sendable_test.go +++ b/cmd/mesh-controller/sendable_test.go @@ -168,3 +168,60 @@ func stdoutOf(t *testing.T, run func() error) string { } return out } + +// novox/hq ADR 0100: a port a node was given for the store is where its consumers on other +// machines are told to reach it, and a port given for the whole mesh is refused. +func TestConsumersAreToldTheGivenPort(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "store", Version: "1", + Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Guards: []int{5432}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, + "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) + register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}}) + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "app"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "store", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil { + t.Fatal(err) + } + + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + var told any + for _, n := range plan.Needs { + if n.Name == "database" { + told = n.Serves["port"] + } + } + if told != 5433 { + t.Fatalf("the consumer is told the store is on %v", told) + } + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) { + t.Fatalf("the store publishes %v", r["ports"]) + } + } + + if err := open.inventory.SetSettings(ctx, "", "store", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err != nil { + t.Fatal(err) + } + plan, settings, err := planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if _, err := declarationFor(ctx, open, "anchor", plan, settings); err == nil || + !strings.Contains(err.Error(), "per node") { + t.Fatalf("a port given for the whole mesh was not refused: %v", err) + } +} diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index 20569bd..c5446b8 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -221,3 +221,63 @@ func TestAGuardedPortMustBeAPort(t *testing.T) { func keys[V any](m map[string]V) []string { return sortedKeys(m) } + +// novox/hq ADR 0100: the foundation's ports are the node's. Given 5433 for the store, every place +// that uses the port reads it from there: the container, the filter, the openings, the guard. +func TestAGivenPortIsUsedEverywhereThePortIs(t *testing.T) { + given := map[string]map[int]int{"postgres": {5432: 5433}, "lavinmq": {15672: 15673}} + for _, adopted := range []bool{true, false} { + with := anchorRendering(adopted) + with.Given = given + with.Ports["postgres"] = map[int]int{5432: 5433} + composed, err := anAdoptedAnchor().Compose(with) + if err != nil { + t.Fatal(err) + } + got := byID(composed.Resources) + if ports := got["postgres.server"]["ports"]; !reflect.DeepEqual(ports, []any{"5433:5432"}) { + t.Fatalf("the store's container publishes %v", ports) + } + if ports := got["lavinmq.server"]["ports"]; !reflect.DeepEqual(ports, + []any{"5671:5671", "5672:5672", "127.0.0.1:15673:15672"}) { + t.Fatalf("the broker's container publishes %v", ports) + } + if !adopted { + filter, _ := got["nftables.filtering"]["content"].(string) + if !strings.Contains(filter, "tcp dport 5433 accept") || strings.Contains(filter, "5432") { + t.Fatalf("the filter does not use the given port:\n%s", filter) + } + continue + } + if o := got["adoption.opening-tcp-5433-forwarded"]; o == nil || o["to"] != 5432 { + t.Fatalf("no opening for the given port: %v", keys(got)) + } + if guard := got[GuardID()]["content"]; guard != AsGuard([]int{5433, 15673}) { + t.Fatalf("the guard does not guard the given ports:\n%s", guard) + } + } +} + +func TestAGivenPortIsTheNodesAndReachesSomething(t *testing.T) { + store := anAdoptedAnchor().Modules[1] + node := func(v any) []Layer { + return []Layer{{From: "anchor", Values: map[string]any{PortsSetting: v}}} + } + if got, err := GivenPorts(store, node(map[string]any{"5432": float64(5433)})); err != nil || + got[5432] != 5433 { + t.Fatalf("a node's given port was not read: %v %v", got, err) + } + if _, err := GivenPorts(store, []Layer{{From: MeshWideLayer, + Values: map[string]any{PortsSetting: map[string]any{"5432": float64(5433)}}}}); err == nil { + t.Fatal("a port given for the whole mesh was accepted") + } + if _, err := GivenPorts(store, node(map[string]any{"6000": float64(6001)})); err == nil { + t.Fatal("a port the module neither listens on, publishes nor guards was given") + } + if _, err := GivenPorts(store, node(map[string]any{"5432": float64(70000)})); err == nil { + t.Fatal("a machine port that is not a port was given") + } + if stray := UnusedSettings(store, node(map[string]any{"5432": float64(5433)})); len(stray) != 0 { + t.Fatalf("a given port is called stray: %v", stray) + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 59faa42..2e07ddf 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -133,12 +133,20 @@ type Rendering struct { // force, so no module that loads a filter is declared there, and what the mesh needs // reachable is declared as openings, with its own ports guarded by a table that only refuses. Adopted bool + + // Given is the machine ports this node was given for its modules' ports, by module and by the + // port the software uses (novox/hq ADR 0100) — the foundation's ports, as genesis chose them. + // They win over anything the mesh would assign and over a manifest's own long-form mapping. + Given map[string]map[int]int } // machinePort is where a module's port lives on this machine, or the port itself when the mesh has // not been asked. Unassigned is not an error here: a module with no `listens` never needed one, // and a caller composing a declaration without a store still gets something coherent. func (r Rendering) machinePort(module string, wanted int) int { + if at, given := r.Given[module][wanted]; given { + return at + } if at, known := r.Ports[module][wanted]; known { return at } @@ -1188,7 +1196,11 @@ func publishedOn(resource map[string]any, module string, with Rendering) { for _, entry := range listed { written := fmt.Sprint(entry) if strings.Contains(written, ":") { - out = append(out, written) + // Written the long way, and left alone — unless this node was given a machine port for + // it (novox/hq ADR 0100): the foundation's ports are the node's, and a manifest's + // number is only the default. The outer port only; an address and the software's + // port stay as written. + out = append(out, givenOuter(written, with.Given[module])) continue } wanted, err := strconv.Atoi(strings.TrimSpace(written)) @@ -1203,6 +1215,29 @@ func publishedOn(resource map[string]any, module string, with Rendering) { resource["ports"] = out } +// givenOuter rewrites the machine side of a long-form mapping to the port this node was given for +// its software side, when it was given one. +func givenOuter(written string, given map[int]int) string { + if len(given) == 0 { + return written + } + mapping, protocol := written, "" + if cut := strings.LastIndex(written, "/"); cut >= 0 { + mapping, protocol = written[:cut], written[cut:] + } + parts := strings.Split(mapping, ":") + inner, err := strconv.Atoi(strings.TrimSpace(parts[len(parts)-1])) + if err != nil { + return written + } + at, ok := given[inner] + if !ok { + return written + } + parts[len(parts)-2] = strconv.Itoa(at) + return strings.Join(parts, ":") + protocol +} + // ServedOn is what a provider tells a consumer, with the port that machine actually uses. // // **The module writes the port once, in `listens`** (novox/hq ADR 0038). It used to write it three diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index 3726e87..ae1354f 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -457,3 +457,95 @@ func byFamily(addresses []string) (four []string, six []string) { } return four, six } + +// PortsSetting is the settings key that gives a module's port a machine port on one node (novox/hq +// ADR 0100): +// +// {"ports": {"5432": 5433}} +// +// puts what the software calls 5432 on the machine's 5433. The foundation's ports are the node's: +// every one is an input to genesis, checked free there, and becomes that node's setting for the +// foundation's modules — the catalogue's numbers are only their defaults. Keyed by the port the +// software uses, like expose; the value is where the machine puts it. +const PortsSetting = "ports" + +// MeshWideLayer is what a layer set for the whole mesh is called, rather than for one node. +const MeshWideLayer = "the mesh" + +// GivenPorts reads a module's given machine ports from its settings: software port → machine port. +// +// Refused from a mesh-wide layer — a port is a fact about one machine, and one number for every +// machine is the collision this exists to avoid — and for a port the module neither listens on, +// publishes from a container, nor guards: a given port that reaches nothing is a setting somebody +// believes changed something. +func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) { + known := map[int]bool{} + for _, l := range m.Listens { + known[l.Port] = true + } + for _, p := range m.Guards { + known[p] = true + } + for _, p := range containerPorts(m) { + known[p] = true + } + out := map[int]int{} + for _, layer := range layers { + raw, ok := layer.Values[PortsSetting] + if !ok { + continue + } + if layer.From == MeshWideLayer { + return nil, fmt.Errorf("%s: %s is given per node — a port is a fact about one "+ + "machine; set it with --node", m.Module, PortsSetting) + } + entries, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: %s is a { port: machine-port } map, and %q set it to "+ + "something else", m.Module, PortsSetting, layer.From) + } + for portText, value := range entries { + port, err := strconv.Atoi(portText) + if err != nil { + return nil, fmt.Errorf("%s gives %q a port, which is not a port", m.Module, portText) + } + if !known[port] { + return nil, fmt.Errorf("%s gives port %d a machine port, and it neither listens "+ + "on, publishes nor guards %d — the setting reaches nothing", m.Module, port, port) + } + at, ok := asPort(value) + if !ok || at < 1 || at > 65535 { + return nil, fmt.Errorf("%s gives port %d the machine port %v, which is not a port", + m.Module, port, value) + } + out[port] = at + } + } + if len(out) == 0 { + return nil, nil + } + return out, nil +} + +// containerPorts are the software ports a module's containers publish, whichever form they are +// written in. +func containerPorts(m Manifest) []int { + var out []int + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + listed, _ := r["ports"].([]any) + for _, entry := range listed { + written := strings.TrimSpace(fmt.Sprint(entry)) + if cut := strings.LastIndex(written, "/"); cut >= 0 { + written = written[:cut] + } + parts := strings.Split(written, ":") + if n, err := strconv.Atoi(parts[len(parts)-1]); err == nil { + out = append(out, n) + } + } + } + return out +} diff --git a/internal/catalogue/settings.go b/internal/catalogue/settings.go index 5fa4865..854a9cf 100644 --- a/internal/catalogue/settings.go +++ b/internal/catalogue/settings.go @@ -101,6 +101,11 @@ func settle(base map[string]any, layers []Layer, protected map[string]bool, what merged := deepCopy(base) for _, layer := range layers { for key, value := range layer.Values { + if key == PortsSetting { + // Where the machine puts a port is the mesh's to apply, not a value for a file or + // for what a consumer is told (novox/hq ADR 0100); it reaches both as the port. + continue + } if protected[key] { // The module said it must own this one. Refused rather than ignored: a setting // that is quietly dropped is somebody believing they changed something. @@ -176,6 +181,11 @@ func UnusedSettings(m Manifest, layers []Layer) []string { if key == ExposeSetting && len(m.Listens) > 0 { continue } + // `ports` gives a module's port a machine port on one node (novox/hq ADR 0100), + // validated in GivenPorts, so it is not stray here either. + if key == PortsSetting { + continue + } unused = append(unused, fmt.Sprintf( "%s sets %q, and %s has no file or contribution to merge it into", layer.From, key, m.Module)) diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 71527d3..322de46 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -654,7 +654,7 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([ } from := nodeName if meshWide { - from = "the mesh" + from = catalogue.MeshWideLayer } layers = append(layers, catalogue.Layer{From: from, Values: values}) } diff --git a/internal/inventory/ports.go b/internal/inventory/ports.go index 3e031c4..20cb4ed 100644 --- a/internal/inventory/ports.go +++ b/internal/inventory/ports.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "encoding/json" "errors" "fmt" @@ -132,6 +133,17 @@ func (i *Inventory) assignPort( taken[port] = "something this machine already runs" } } + // And every port this machine was given for a module (novox/hq ADR 0100): the foundation's + // ports, as genesis chose them, are the node's settings and never the mesh's to hand out. + given, err := i.givenOn(ctx, nodeID) + if err != nil { + return Assigned{}, err + } + for port, by := range given { + if _, mine := taken[port]; !mine { + taken[port] = by + } + } machine := wanted if !fixed { @@ -258,3 +270,33 @@ func (i *Inventory) ReleasePorts(ctx context.Context, node, module string) error `delete from port_assignment where node = $1 and module = $2`, record.ID, module) return err } + +// givenOn is every machine port a module was given on this node by its `ports` setting, and which +// module it was given to. +func (i *Inventory) givenOn(ctx context.Context, nodeID any) (map[int]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select module, values->'ports' from settings + where node = $1 and jsonb_typeof(values->'ports') = 'object'`, nodeID) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[int]string{} + for rows.Next() { + var module string + var raw []byte + if err := rows.Scan(&module, &raw); err != nil { + return nil, err + } + var given map[string]any + if err := json.Unmarshal(raw, &given); err != nil { + return nil, err + } + for _, v := range given { + if at, ok := v.(float64); ok { + out[int(at)] = module + } + } + } + return out, rows.Err() +} diff --git a/internal/inventory/ports_test.go b/internal/inventory/ports_test.go index ef2c8fa..b62f310 100644 --- a/internal/inventory/ports_test.go +++ b/internal/inventory/ports_test.go @@ -236,3 +236,24 @@ func TestUnassigningReleasesTheModulesPorts(t *testing.T) { t.Fatalf("port 25 is still held in the name of a module that was unassigned: %v", err) } } + +// novox/hq ADR 0100: a port a node was given for a module is the node's, and the mesh never hands +// it to another. +func TestAGivenPortIsNeverAssigned(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "web") + ctx := t.Context() + if err := inv.SetSettings(ctx, node, "postgres", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 20000}}); err != nil { + t.Fatal(err) + } + got, err := inv.PortFor(ctx, node, "web", 8080, false) + if err != nil { + t.Fatal(err) + } + if got.Machine == 20000 { + t.Fatal("a port given to postgres was assigned to web") + } + if _, err := inv.PortFor(ctx, node, "web", 20000, true); !errors.Is(err, ErrPortTaken) { + t.Fatalf("a fixed port given to another module was handed over: %v", err) + } +} From c93128d82fc3fc1a5f17c0c7a598feb9b12a5e06 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:33:09 +0200 Subject: [PATCH 07/33] Hold the catalogue's store, broker and filter manifests to what adoption needs of them (hq ADR 0100) --- internal/catalogue/adoption.go | 2 +- .../catalogue/foundation_manifests_test.go | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 internal/catalogue/foundation_manifests_test.go diff --git a/internal/catalogue/adoption.go b/internal/catalogue/adoption.go index 5e31ef0..c165944 100644 --- a/internal/catalogue/adoption.go +++ b/internal/catalogue/adoption.go @@ -186,7 +186,7 @@ func AsGuard(ports []int) string { func GuardUnitText() string { return "[Unit]\n" + "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + - "After=network-pre.target\n" + + "Before=network-pre.target\n" + "Wants=network-pre.target\n" + "\n" + "[Service]\n" + diff --git a/internal/catalogue/foundation_manifests_test.go b/internal/catalogue/foundation_manifests_test.go new file mode 100644 index 0000000..08886c4 --- /dev/null +++ b/internal/catalogue/foundation_manifests_test.go @@ -0,0 +1,66 @@ +package catalogue + +import ( + "os" + "reflect" + "strings" + "testing" +) + +// The catalogue's foundation modules as they are, parsed by the real parser (novox/hq ADR 0100): +// the store and the broker say which of their ports the mesh guards on an adopted node, and the +// filter module loads its table through a unit of its own whose stop deletes only that table. +func catalogueManifest(t *testing.T, module string) Manifest { + t.Helper() + raw, err := os.ReadFile("../../../mesh-catalog/modules/" + module + "/module.json") + if err != nil { + t.Skipf("the catalogue is not beside this checkout: %v", err) + } + m, err := ParseManifest(raw) + if err != nil { + t.Fatalf("%s does not parse:\n%v", module, err) + } + return m +} + +func TestTheStoreAndTheBrokerSayWhatTheMeshGuards(t *testing.T) { + if got := catalogueManifest(t, "postgres").Guards; !reflect.DeepEqual(got, []int{5432}) { + t.Errorf("postgres guards %v; the store's port must be refused from outside", got) + } + if got := catalogueManifest(t, "lavinmq").Guards; !reflect.DeepEqual(got, []int{15672}) { + t.Errorf("lavinmq guards %v; the management port must be refused from outside", got) + } +} + +func TestTheFilterModuleNeverFlushesTheRuleset(t *testing.T) { + m := catalogueManifest(t, "nftables") + var unit map[string]any + var load map[string]any + for _, r := range m.Resources { + switch r["id"] { + case "unit": + unit = r + case "load": + load = r + } + } + if load == nil || load["unit"] != "mesh-filter.service" { + t.Fatalf("the filter is not loaded by its own unit: %v", load) + } + content, _ := unit["content"].(string) + if unit == nil || unit["path"] != "/etc/systemd/system/mesh-filter.service" { + t.Fatalf("the filter's unit is not written: %v", unit) + } + if strings.Contains(content, "flush") { + t.Fatalf("stopping the filter flushes the whole ruleset — the runtime's and the found "+ + "firewall's with it:\n%s", content) + } + if !strings.Contains(content, "ExecStop=nft delete table inet mesh\n") || + !strings.Contains(content, "ExecStart=nft -f "+m.Filtering.Into+"\n") { + t.Fatalf("the unit does not load the computed rule set and delete only its own table:\n%s", + content) + } + if !reflect.DeepEqual(load["restart-on"], []any{"filtering", "unit"}) { + t.Fatalf("the filter is not reloaded when its rules or its unit change: %v", load["restart-on"]) + } +} From 28b7fb81ba903277fb33da29d9bd98d0e9b1bac1 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:51:38 +0200 Subject: [PATCH 08/33] Write the registry's trust into the runtime's file and reload the runtime instead of restarting it; prefix reload-on like restart-on (hq ADR 0102) --- internal/catalogue/computed_test.go | 45 +++++++++++++++++++++++++++++ internal/catalogue/declaration.go | 5 ++++ internal/overlay/generator.go | 18 +++++++----- internal/overlay/generator_test.go | 14 +++++++-- 4 files changed, 71 insertions(+), 11 deletions(-) diff --git a/internal/catalogue/computed_test.go b/internal/catalogue/computed_test.go index 55167db..3e74716 100644 --- a/internal/catalogue/computed_test.go +++ b/internal/catalogue/computed_test.go @@ -1,6 +1,7 @@ package catalogue import ( + "fmt" "strings" "testing" ) @@ -157,3 +158,47 @@ func TestTwoWaysToBeOnAPrivateNetworkRefuseAndNameBoth(t *testing.T) { } } } + +// reloading answers the runtime's trust as the networking module does (novox/hq ADR 0102): a file +// written into, and the runtime reloaded on it. +type reloading struct{} + +func (reloading) Resources(node string) ([]map[string]any, bool, error) { + return []map[string]any{ + {"id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json", + "merge": MergeJSON, "into": "json", "content": `{"insecure-registries":["r:5000"]}`}, + {"id": "registry-trust-reload", "type": "service", "unit": "docker.service", + "state": "running", "reload-on": []string{"registry-trust"}}, + }, true, nil +} + +func TestWhatAServiceIsReloadedOnIsNamedAsTheHostWillSeeIt(t *testing.T) { + // Ids are prefixed with their module on the way out. An unprefixed reload-on would name a + // resource the host never sees, and the runtime would never be reloaded for its trust. + got, err := Resolve(computedShelf(), []string{"mesh-network"}, workstation(), World{}) + if err != nil { + t.Fatal(err) + } + out, err := got.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": reloading{}}}) + if err != nil { + t.Fatal(err) + } + var file, service map[string]any + for _, r := range out { + switch r["type"] { + case "file": + file = r + case "service": + service = r + } + } + if file == nil || service == nil { + t.Fatalf("got %v", out) + } + if file["into"] != "json" { + t.Errorf("into did not reach the host: %v", file) + } + if want := "[" + file["id"].(string) + "]"; fmt.Sprint(service["reload-on"]) != want { + t.Errorf("reload-on names %v, the file is %v", service["reload-on"], file["id"]) + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 2e07ddf..59281f1 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -559,6 +559,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil { copied["restart-on"] = renamed } + // And what it is reloaded on, by the same rule (novox/hq ADR 0102): an id left + // unprefixed matches nothing, and the service is never reloaded. + if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil { + copied["reload-on"] = renamed + } owner[fmt.Sprint(copied["id"])] = m.Module out = append(out, copied) } diff --git a/internal/overlay/generator.go b/internal/overlay/generator.go index 54995ed..c30eb67 100644 --- a/internal/overlay/generator.go +++ b/internal/overlay/generator.go @@ -140,18 +140,20 @@ func (g *Generator) Resources(node string) ([]map[string]any, bool, error) { } resources = append(resources, map[string]any{ - // Merged, not owned: the runtime's daemon file is the machine's, and this states - // one fact into it. The registry speaks plain HTTP because every path to it is - // already inside the overlay's encryption (ADR 0082) — this line is the runtime - // being told what the mesh already means. + // Written into, not over (novox/hq ADR 0102): the runtime's daemon file is the + // machine's — its data directory, its logging, whatever a predecessor set — and + // this states one fact in it. The host sets this key and keeps every other. + // ("merge" is the operator's settings merged into this content; "into" is the + // content written into the machine's file.) The registry speaks plain HTTP + // because every path to it is already inside the overlay's encryption (ADR 0082). "id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json", - "content": string(trust) + "\n", "mode": "0644", "merge": "json", + "content": string(trust) + "\n", "mode": "0644", "merge": "json", "into": "json", }, map[string]any{ - // The runtime reloads nothing for this setting, so it is restarted when the fact - // changes — once, at joining, before the machine runs anything that would mind. + // Reloaded, not restarted: the runtime re-reads its trusted registries on a reload, + // and a restart stops every container on the machine (measured; ADR 0102). "id": "registry-trust-reload", "type": "service", "unit": "docker.service", - "state": "running", "restart-on": []string{"registry-trust"}, + "state": "running", "reload-on": []string{"registry-trust"}, }) } return resources, true, nil diff --git a/internal/overlay/generator_test.go b/internal/overlay/generator_test.go index 0cc935a..2a54332 100644 --- a/internal/overlay/generator_test.go +++ b/internal/overlay/generator_test.go @@ -1,6 +1,7 @@ package overlay import ( + "fmt" "strings" "testing" ) @@ -44,13 +45,20 @@ func TestTheNetworkCarriesRegistryTrust(t *testing.T) { if file == nil || service == nil { t.Fatalf("the trust file or its reload is missing: %v", trusted) } - if file["path"] != "/etc/docker/daemon.json" || file["merge"] != "json" { - t.Fatalf("the trust is not a merged daemon.json: %v", file) + if file["path"] != "/etc/docker/daemon.json" || file["merge"] != "json" || file["into"] != "json" { + t.Fatalf("the trust is not written into daemon.json (ADR 0102): %v", file) } if content, _ := file["content"].(string); !strings.Contains(content, `"anchor.internal:5000"`) { t.Fatalf("the trust does not name the store: %v", file["content"]) } if service["unit"] != "docker.service" { - t.Fatalf("the reload does not restart the runtime: %v", service) + t.Fatalf("the reload is not the runtime's: %v", service) + } + // Reloaded, never restarted: a restart stops every container on the machine (ADR 0102). + if _, restarts := service["restart-on"]; restarts { + t.Fatalf("the runtime is restarted for its trust: %v", service) + } + if fmt.Sprint(service["reload-on"]) != "[registry-trust]" { + t.Fatalf("the runtime is not reloaded for its trust: %v", service) } } From 8db66e9532804c0e1b2d66e63e541ee83821e9af Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:58:37 +0200 Subject: [PATCH 09/33] Derive the guard from taken modules only: their published private-network ports and their manifests' guards (hq ADR 0103) --- cmd/mesh-controller/plan.go | 15 ++++- cmd/mesh-controller/sendable_test.go | 35 ++++++++++ internal/catalogue/adoption_test.go | 61 +++++++++++++++++- internal/catalogue/declaration.go | 95 ++++++++++++++++++++++------ 4 files changed, 182 insertions(+), 24 deletions(-) diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 48ff6d9..0f540c7 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -539,11 +539,24 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } + // And, on an adopted node, which modules were taken there: the guard is derived from those + // only (novox/hq ADR 0103). + var taken map[string]bool + if record.Adopted { + list, err := inv.Taken(ctx, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + taken = map[string]bool{} + for _, m := range list { + taken[m] = true + } + } return catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted, - Given: given, + Given: given, Taken: taken, }, record, nil } diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go index cb82110..72d9ffb 100644 --- a/cmd/mesh-controller/sendable_test.go +++ b/cmd/mesh-controller/sendable_test.go @@ -225,3 +225,38 @@ func TestConsumersAreToldTheGivenPort(t *testing.T) { t.Fatalf("a port given for the whole mesh was not refused: %v", err) } } + +// novox/hq ADR 0103: the guard an adopted node is sent follows what was taken there. A store +// assigned but not taken is not guarded — its port may still be the predecessor's — and taking it +// guards it from the next declaration. +func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "store", Version: "1", + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Guards: []int{5432}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, + "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) + if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + if hasID(composed(t, open, "anchor").Resources, catalogue.GuardID()) { + t.Fatal("an untaken store is guarded") + } + if err := open.inventory.Take(ctx, "anchor", "store"); err != nil { + t.Fatal(err) + } + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == catalogue.GuardID() { + if r["content"] != catalogue.AsGuard([]int{5432}) { + t.Fatalf("the taken store's guard is:\n%s", r["content"]) + } + return + } + } + t.Fatal("a taken store is not guarded") +} diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index c5446b8..01aa720 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -59,6 +59,8 @@ func anchorRendering(adopted bool) Rendering { Mesh: []string{"10.42.0.1"}, Foundation: []int{5671}, Adopted: adopted, + // Genesis takes the foundation's modules. + Taken: map[string]bool{"postgres": true, "lavinmq": true}, } } @@ -151,8 +153,9 @@ func TestAnAdoptedNodeLoadsNoFilterOfTheMeshs(t *testing.T) { if guard == nil || got[GuardUnitID()] == nil || got[GuardRunningID()] == nil { t.Fatalf("no guard: %v", keys(got)) } - if guard["content"] != AsGuard([]int{5432, 15672}) { - t.Fatalf("the guard does not guard the store and the management port:\n%s", guard["content"]) + if guard["content"] != AsGuard([]int{5432, 5672, 15672}) { + t.Fatalf("the guard does not guard the store, the broker and its management port:\n%s", + guard["content"]) } if !reflect.DeepEqual(got[GuardRunningID()]["restart-on"], []any{GuardID(), GuardUnitID()}) { t.Fatalf("the guard is not reloaded when its table changes: %v", got[GuardRunningID()]) @@ -252,7 +255,7 @@ func TestAGivenPortIsUsedEverywhereThePortIs(t *testing.T) { if o := got["adoption.opening-tcp-5433-forwarded"]; o == nil || o["to"] != 5432 { t.Fatalf("no opening for the given port: %v", keys(got)) } - if guard := got[GuardID()]["content"]; guard != AsGuard([]int{5433, 15673}) { + if guard := got[GuardID()]["content"]; guard != AsGuard([]int{5433, 5672, 15673}) { t.Fatalf("the guard does not guard the given ports:\n%s", guard) } } @@ -281,3 +284,55 @@ func TestAGivenPortIsTheNodesAndReachesSomething(t *testing.T) { t.Fatalf("a given port is called stray: %v", stray) } } + +// novox/hq ADR 0103: the guard is derived, and from taken modules only — every machine port a taken +// module publishes that the filter admits from the private network only, and the ports its +// manifest guards. A module assigned but not taken is not guarded: its port may still be the +// predecessor's. +func TestTheGuardIsDerivedFromTakenModulesOnly(t *testing.T) { + guardOf := func(with Rendering) string { + t.Helper() + composed, err := anAdoptedAnchor().Compose(with) + if err != nil { + t.Fatal(err) + } + content, _ := byID(composed.Resources)[GuardID()]["content"].(string) + return content + } + + // The broker taken, the store not: the broker's plain port follows from its listens (from + // the mesh, published), its management port from its manifest; the store is not guarded, and + // neither is the bus, which the mesh needs from everywhere. + with := anchorRendering(true) + with.Taken = map[string]bool{"lavinmq": true} + if got := guardOf(with); got != AsGuard([]int{5672, 15672}) { + t.Fatalf("the guard is not the taken broker's ports:\n%s", got) + } + + // A taken module publishing a port admitted from everywhere is not guarded; one admitted from + // the mesh is. The registry is exposed everywhere on this node, and hello-web listens from + // everywhere. + with.Taken = map[string]bool{"distribution": true, "hello-web": true} + if got := guardOf(with); got != "" { + t.Fatalf("a port admitted from everywhere is guarded:\n%s", got) + } + with.Settings = nil + if got := guardOf(with); got != AsGuard([]int{5000}) { + t.Fatalf("the registry, from the mesh only, is not guarded:\n%s", got) + } + + // Nothing taken, nothing guarded — and no guard at all rather than an empty set. + with = anchorRendering(true) + with.Taken = nil + if got := guardOf(with); got != "" { + t.Fatalf("an untaken store is guarded:\n%s", got) + } + + // A given port is followed: where the machine put it is what is refused. + with = anchorRendering(true) + with.Given = map[string]map[int]int{"lavinmq": {5672: 5682, 15672: 15673}} + with.Ports["lavinmq"] = map[int]int{5671: 5671, 5672: 5682} + if got := guardOf(with); got != AsGuard([]int{5432, 5682, 15673}) { + t.Fatalf("the guard does not follow the given ports:\n%s", got) + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 59281f1..0cc318f 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -138,6 +138,11 @@ type Rendering struct { // port the software uses (novox/hq ADR 0100) — the foundation's ports, as genesis chose them. // They win over anything the mesh would assign and over a manifest's own long-form mapping. Given map[string]map[int]int + + // Taken is the modules taken on this adopted node (novox/hq ADR 0100). The guard is derived + // from these only (ADR 0103): a port of a module assigned but not taken may still be the + // predecessor's. + Taken map[string]bool } // machinePort is where a module's port lives on this machine, or the port itself when the mesh has @@ -586,50 +591,100 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri // First, before anything a module declares: what the mesh needs reachable, then its guard. // The order a machine applies is the order written here. ours := Openings(rules, with.Foundation, Published(out)) - ours = append(ours, GuardResources(r.guarded(out, owner, with))...) + ours = append(ours, GuardResources(r.guarded(out, owner, rules, with))...) out = append(ours, out...) } return out, nil } -// guarded is the machine ports of every guarded port of the modules here: where each module's -// container publishes it, as composed — or where the machine put it when no container does. -func (r Resolution) guarded(out []map[string]any, owner map[string]string, with Rendering) []int { +// guarded is what the mesh's guard refuses on an adopted node (novox/hq ADR 0103): derived, and +// for taken modules only. +// +// For each taken module, every machine port its containers publish that the filter would admit +// from the private network only — a published port is forwarded, not received, so a found +// firewall filtering only what it receives never sees it — together with the ports the module's +// manifest guards explicitly (the store's port, the broker's management port), wherever the +// machine put them. A port of a module assigned but not taken is not guarded: it may still be the +// predecessor's, serving the predecessor's other machines. The foundation's ports are admitted +// from everywhere and are never guarded. +func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules []Rule, + with Rendering) []int { + meshOnly := map[int]bool{} + for _, rule := range rules { + if rule.Protocol == "tcp" && rule.From == FromMesh { + meshOnly[rule.Port] = true + } + } + for _, port := range with.Foundation { + delete(meshOnly, port) + } seen := map[int]bool{} var ports []int + guard := func(at int) { + if !seen[at] { + seen[at] = true + ports = append(ports, at) + } + } for _, m := range r.Modules { + if !with.Taken[m.Module] { + continue + } + var mine []map[string]any + for _, resource := range out { + if owner[fmt.Sprint(resource["id"])] == m.Module { + mine = append(mine, resource) + } + } + for outer := range Published(mine)["tcp"] { + if meshOnly[outer] { + guard(outer) + } + } for _, want := range m.Guards { at := with.machinePort(m.Module, want) - for _, resource := range out { - if owner[fmt.Sprint(resource["id"])] != m.Module || - fmt.Sprint(resource["type"]) != "container" { + for _, resource := range mine { + if fmt.Sprint(resource["type"]) != "container" { continue } listed, _ := resource["ports"].([]any) for _, entry := range listed { - parts := strings.Split(strings.TrimSpace(fmt.Sprint(entry)), ":") - if len(parts) < 2 { - continue - } - inner, err := strconv.Atoi(strings.SplitN(parts[len(parts)-1], "/", 2)[0]) - if err != nil || inner != want { - continue - } - if outer, err := strconv.Atoi(parts[len(parts)-2]); err == nil { + outer, inner, _, ok := mapping(fmt.Sprint(entry)) + if ok && inner == want { at = outer } } } - if !seen[at] { - seen[at] = true - ports = append(ports, at) - } + guard(at) } } sort.Ints(ports) return ports } +// mapping reads a container's port mapping — `[address:]outer:inner[/protocol]`, the address +// possibly an IPv6 one in brackets — indexing from the end, so an address's own colons never +// shift the ports. Not ok for a short form or anything that is not a mapping. +func mapping(written string) (outer, inner int, address string, ok bool) { + written = strings.TrimSpace(written) + if cut := strings.LastIndex(written, "/"); cut >= 0 { + written = written[:cut] + } + parts := strings.Split(written, ":") + if len(parts) < 2 { + return 0, 0, "", false + } + inner, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + return 0, 0, "", false + } + outer, err = strconv.Atoi(parts[len(parts)-2]) + if err != nil { + return 0, 0, "", false + } + return outer, inner, strings.Join(parts[:len(parts)-2], ":"), true +} + // Rules is the rule set this node's filter is derived from: every module's listens, what was // computed for this machine, and each module's per-node exposure. The same answer whether the node // is adopted or converged — the one loads it as a filter, the other declares it as openings. From a82bfb41f2fa8122da0aeea203f441318f61de3a Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:58:50 +0200 Subject: [PATCH 10/33] Leave an IPv6 loopback mapping out of what a container publishes, reading ports from the end (hq ADR 0100) --- internal/catalogue/adoption.go | 18 +++++------------- internal/catalogue/adoption_test.go | 12 ++++++++++++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/internal/catalogue/adoption.go b/internal/catalogue/adoption.go index c165944..7b8c827 100644 --- a/internal/catalogue/adoption.go +++ b/internal/catalogue/adoption.go @@ -123,22 +123,14 @@ func Published(resources []map[string]any) map[string]map[int]int { protocol := "tcp" if cut := strings.LastIndex(written, "/"); cut >= 0 { protocol = written[cut+1:] - written = written[:cut] } - parts := strings.Split(written, ":") - if len(parts) < 2 { + // Indexed from the end, so an IPv6 address's own colons never shift the ports. + outer, inner, address, ok := mapping(written) + if !ok { continue } - if len(parts) == 3 && (parts[0] == "127.0.0.1" || parts[0] == "localhost" || - parts[0] == "[::1]") { - continue - } - outer, err := strconv.Atoi(parts[len(parts)-2]) - if err != nil { - continue - } - inner, err := strconv.Atoi(parts[len(parts)-1]) - if err != nil { + switch strings.Trim(address, "[]") { + case "127.0.0.1", "localhost", "::1": continue } if out[protocol] == nil { diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index 01aa720..da2dc78 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -336,3 +336,15 @@ func TestTheGuardIsDerivedFromTakenModulesOnly(t *testing.T) { t.Fatalf("the guard does not follow the given ports:\n%s", got) } } + +// A mapping bound to loopback is not published to anything off the machine — in either address +// family — and an address's own colons never shift the ports. +func TestPublishedLeavesOutLoopbackInBothFamilies(t *testing.T) { + got := Published([]map[string]any{{"type": "container", "ports": []any{ + "127.0.0.1:15672:15672", "[::1]:8080:80", "localhost:9090:90", + "[::]:8443:443", "0.0.0.0:5000:5000", "5353:53/udp"}}}) + want := map[string]map[int]int{"tcp": {8443: 443, 5000: 5000}, "udp": {5353: 53}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("published is %v, want %v", got, want) + } +} From a2dfaaf4d1725338d80a4518c53f02bb57e35ae5 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:59:09 +0200 Subject: [PATCH 11/33] Guard only packets addressed to this machine, and order the guard's unit before the network and against shutdown (hq ADR 0103) --- internal/catalogue/adoption.go | 18 +++++++++++++----- internal/catalogue/adoption_test.go | 22 +++++++++++++++++++++- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/internal/catalogue/adoption.go b/internal/catalogue/adoption.go index 7b8c827..04dc85d 100644 --- a/internal/catalogue/adoption.go +++ b/internal/catalogue/adoption.go @@ -146,8 +146,9 @@ func Published(resources []map[string]any) map[string]map[int]int { // // It passes everything by default and holds nothing but a refusal, so it cannot close anything // the machine serves; and it is the mesh's own table, so the found firewall reloading does not -// touch it. It refuses the ports except from the machine itself — its loopback and the container -// runtime's own networks — and from the private network, known by the interface a packet arrives +// touch it. It refuses only packets addressed to this machine, and the ports except from the +// machine itself — its loopback and the container runtime's own networks — and from the private +// network, known by the interface a packet arrives // on and never by its source address. At prerouting, ahead of the runtime's destination // translation, so it matches the port the packet was sent to; in the inet family, so both address // families. @@ -166,8 +167,11 @@ func AsGuard(ports []int) string { b.WriteString("table inet mesh_guard {\n") b.WriteString("\tchain prerouting {\n") b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n") - fmt.Fprintf(&b, "\t\tiifname != \"lo\" iifname != \"docker0\" iifname != \"br-*\" "+ - "iifname != \"mesh0\" tcp dport { %s } drop\n", strings.Join(listed, ", ")) + // Only packets addressed to this machine: traffic it routes for others — a predecessor's hub, + // say — is never the guard's business (novox/hq ADR 0103). + fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" "+ + "iifname != \"br-*\" iifname != \"mesh0\" tcp dport { %s } drop\n", + strings.Join(listed, ", ")) b.WriteString("\t}\n") b.WriteString("}\n") return b.String() @@ -178,8 +182,12 @@ func AsGuard(ports []int) string { func GuardUnitText() string { return "[Unit]\n" + "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + - "Before=network-pre.target\n" + + // Early, before the network is up, and without the default dependencies that would + // order it after the network; stopped at shutdown like any unit. + "DefaultDependencies=no\n" + "Wants=network-pre.target\n" + + "Before=network-pre.target shutdown.target\n" + + "Conflicts=shutdown.target\n" + "\n" + "[Service]\n" + "Type=oneshot\n" + diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index da2dc78..89ccc37 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -200,13 +200,33 @@ delete table inet mesh_guard table inet mesh_guard { chain prerouting { type filter hook prerouting priority raw; policy accept; - iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop + fib daddr type local iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop } } ` if got := AsGuard([]int{15672, 5432}); got != golden { t.Fatalf("the guard changed:\n%s", got) } + const unit = `[Unit] +Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100) +DefaultDependencies=no +Wants=network-pre.target +Before=network-pre.target shutdown.target +Conflicts=shutdown.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=nft -f /etc/mesh/guard.nft +ExecReload=nft -f /etc/mesh/guard.nft +ExecStop=nft delete table inet mesh_guard + +[Install] +WantedBy=multi-user.target +` + if got := GuardUnitText(); got != unit { + t.Fatalf("the guard's unit changed:\n%s", got) + } if GuardResources(nil) != nil { t.Fatal("a guard with nothing to guard is an empty set nft refuses to load") } From ade6b2bfb6f559df3eaa82386a27930ee2c28ec3 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 17:59:32 +0200 Subject: [PATCH 12/33] Declare nftables before the guard's table, so a node joining adopted without nft can load it (hq ADR 0103) --- internal/catalogue/adoption.go | 21 ++++++++++++++------- internal/catalogue/adoption_test.go | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/internal/catalogue/adoption.go b/internal/catalogue/adoption.go index 04dc85d..f38c7ae 100644 --- a/internal/catalogue/adoption.go +++ b/internal/catalogue/adoption.go @@ -45,6 +45,13 @@ func GuardID() string { return AdoptionPrefix + "guard" } func GuardUnitID() string { return AdoptionPrefix + "guard-unit" } func GuardRunningID() string { return AdoptionPrefix + "guard-running" } +// GuardPackageID is the tool that loads the guard, declared first: a node joining adopted has +// no filter module and may have no nft at all, and a table nothing can load guards nothing. +func GuardPackageID() string { return AdoptionPrefix + "guard-package" } + +// GuardPackage is the package that carries nft. +const GuardPackage = "nftables" + // OpeningID is an opening's resource identity: its protocol, port and path say what it is. func OpeningID(protocol string, port int, path string) string { return fmt.Sprintf("%sopening-%s-%d-%s", AdoptionPrefix, protocol, port, path) @@ -148,10 +155,9 @@ func Published(resources []map[string]any) map[string]map[int]int { // the machine serves; and it is the mesh's own table, so the found firewall reloading does not // touch it. It refuses only packets addressed to this machine, and the ports except from the // machine itself — its loopback and the container runtime's own networks — and from the private -// network, known by the interface a packet arrives -// on and never by its source address. At prerouting, ahead of the runtime's destination -// translation, so it matches the port the packet was sent to; in the inet family, so both address -// families. +// network, known by the interface a packet arrives on and never by its source address. At +// prerouting, ahead of the runtime's destination translation, so it matches the port the packet +// was sent to; in the inet family, so both address families. // // The same text the installer raises on an adopted genesis; a test holds both to it. func AsGuard(ports []int) string { @@ -200,14 +206,15 @@ func GuardUnitText() string { "WantedBy=multi-user.target\n" } -// GuardResources are the guard as three resources of the existing kinds: the table, the unit, and -// the unit running, restarted when the table changes. Nothing when there is nothing to guard: an -// empty set is not a table nft loads. +// GuardResources are the guard as four resources of the existing kinds: the tool that loads it, +// the table, the unit, and the unit running, restarted when the table changes. Nothing when there +// is nothing to guard: an empty set is not a table nft loads. func GuardResources(ports []int) []map[string]any { if len(ports) == 0 { return nil } return []map[string]any{ + {"id": GuardPackageID(), "type": "package", "package": GuardPackage}, {"id": GuardID(), "type": "file", "path": GuardPath, "content": AsGuard(ports), "mode": "0644"}, {"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(), diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index 89ccc37..0d839e1 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -157,6 +157,23 @@ func TestAnAdoptedNodeLoadsNoFilterOfTheMeshs(t *testing.T) { t.Fatalf("the guard does not guard the store, the broker and its management port:\n%s", guard["content"]) } + // The tool that loads it comes first, and the table after it: a node joining adopted has no + // filter module and may have no nft. + pkg, table := -1, -1 + for i, r := range composed.Resources { + switch r["id"] { + case GuardPackageID(): + pkg = i + if r["type"] != "package" || r["package"] != "nftables" { + t.Fatalf("the guard's package is %v", r) + } + case GuardID(): + table = i + } + } + if pkg < 0 || pkg > table { + t.Fatalf("nftables is not declared before the guard's table (%d, %d)", pkg, table) + } if !reflect.DeepEqual(got[GuardRunningID()]["restart-on"], []any{GuardID(), GuardUnitID()}) { t.Fatalf("the guard is not reloaded when its table changes: %v", got[GuardRunningID()]) } From 3dc7d0e386412069818f2f39694b740f65a4d58b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:00:53 +0200 Subject: [PATCH 13/33] Preview ssh as the derived filter admits it, and say a narrowing to the private network closes (hq ADR 0100) --- cmd/mesh-controller/adopting_test.go | 73 +++++++++++++++++++++++----- cmd/mesh-controller/adoption.go | 56 ++++++++++++++++----- 2 files changed, 106 insertions(+), 23 deletions(-) diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index 1936b9a..e52a6f4 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -60,6 +60,21 @@ func anAdoptedAnchor(t *testing.T) (*stores, *[]string) { // reportsHolding has the anchor report, on what it was last sent, holding what is given. func reportsHolding(t *testing.T, open *stores, held ...link.Held) { + t.Helper() + reportsReaching(t, open, []link.Reach{ + {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}, + {Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, + ContainerPort: 80}, + {Protocol: "tcp", Address: "0.0.0.0", Port: 5000, By: "predecessor-registry", + Published: true, ContainerPort: 5000}, + {Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker", + Published: true, ContainerPort: 15672}, + }, held...) +} + +// reportsReaching has the anchor report, on what it was last sent, what is reachable on it and +// holding what is given. +func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ...link.Held) { t.Helper() ctx := t.Context() body, err := composed(t, open, "anchor").Body() @@ -75,16 +90,7 @@ func reportsHolding(t *testing.T, open *stores, held ...link.Held) { } if err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{ Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body), - Firewall: "ufw", Held: held, - Reachable: []link.Reach{ - {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}, - {Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, - ContainerPort: 80}, - {Protocol: "tcp", Address: "0.0.0.0", Port: 5000, By: "predecessor-registry", - Published: true, ContainerPort: 5000}, - {Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker", - Published: true, ContainerPort: 15672}, - }, + Firewall: "ufw", Held: held, Reachable: reachable, }); err != nil { t.Fatal(err) } @@ -159,7 +165,9 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) "tcp/8080 hello-web (published, container port 80)", "declared by hello-web (from anywhere)", "WILL CLOSE — no module assigned here declares it", - "ssh is never closed", + // The anchor faces inward and is on the private network: the derived filter admits ssh + // from the mesh only. + "WILL CLOSE to everything outside the private network — ssh stays open from the mesh", "notes\n replacing the found file /etc/notes.conf (original kept at", "assigns nftables", "the found firewall (ufw) is disabled, never flushed", @@ -253,3 +261,46 @@ func TestTheApiRefusesTheFlipInTheCommandLinesWords(t *testing.T) { t.Fatalf("a take naming no module got %d", got.Code) } } + +// novox/hq ADR 0100: the preview says what the derived filter does, rendered as it is rendered. On +// a machine that faces inward, ssh is admitted from the private network only, and a port a module +// admits from the mesh only closes to everything outside it: both are said to close. +func TestThePreviewSaysWhatNarrowsToTheMeshCloses(t *testing.T) { + open, _ := anAdoptedAnchor(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "store", Version: "1", + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}}) + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + reportsReaching(t, open, []link.Reach{ + {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}, + {Protocol: "tcp", Address: "0.0.0.0", Port: 5432, By: "postgres"}, + {Protocol: "tcp", Address: "10.77.0.1", Port: 5432, By: "postgres"}, + {Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, + ContainerPort: 80}, + }) + preview, err := converge(ctx, open, "anchor", false, "") + if err != nil { + t.Fatal(err) + } + lines := map[string]string{} + for _, line := range strings.Split(preview, "\n") { + fields := strings.Fields(line) + if len(fields) > 1 && strings.HasPrefix(fields[0], "tcp/") { + lines[fields[0]+" "+fields[1]] += line + "\n" + } + } + if got := lines["tcp/22 sshd"]; !strings.Contains(got, "WILL CLOSE to everything outside "+ + "the private network") { + t.Errorf("ssh on an inward machine is not said to close outside the mesh:\n%s", preview) + } + store := lines["tcp/5432 postgres"] + if strings.Count(store, "WILL CLOSE to everything outside the private network") != 1 || + !strings.Contains(store, "declared by store (from mesh)") { + t.Errorf("the store's narrowing is not said to close, or its mesh address is:\n%s", preview) + } + if got := lines["tcp/8080 hello-web"]; !strings.Contains(got, "declared by hello-web (from anywhere)") { + t.Errorf("a port open to everywhere is not said to stay:\n%s", preview) + } +} diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index bd03494..152998c 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -217,7 +217,9 @@ func converge(ctx context.Context, open *stores, node string, yes bool, filter s if err != nil { return "", err } - preview := previewOf(node, reported, rules, with.Foundation, plan, taken, filter, runs[filter]) + derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh, + outward: plan.PublicDomain != ""} + preview := previewOf(node, reported, derived, plan, taken, filter, runs[filter]) if !yes { return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes` to do it.", node), nil } @@ -252,7 +254,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, filter s } // previewOf is what converging a node will change, before it changes it. -func previewOf(node string, reported inventory.Adoption, rules []catalogue.Rule, foundation []int, +func previewOf(node string, reported inventory.Adoption, derived derivedFilter, plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) string { var b strings.Builder fmt.Fprintf(&b, "converging %s\n", node) @@ -269,7 +271,7 @@ func previewOf(node string, reported inventory.Adoption, rules []catalogue.Rule, if r.Published { what += fmt.Sprintf(" (published, container port %d)", r.ContainerPort) } - fmt.Fprintf(&b, " %-44s %s\n", what, fate(r, rules, foundation)) + fmt.Fprintf(&b, " %-44s %s\n", what, derived.fate(r)) } if len(reported.Reachable) == 0 { b.WriteString(" nothing reported\n") @@ -317,26 +319,56 @@ func previewOf(node string, reported inventory.Adoption, rules []catalogue.Rule, return strings.TrimRight(b.String(), "\n") } +// derivedFilter is what the filter the flip loads is rendered from, as AsNftables renders it. +type derivedFilter struct { + rules []catalogue.Rule + foundation []int + // mesh is every address on the private network; outward says the machine faces outside. + mesh []string + outward bool +} + +// closesOutside is what a narrowing from everywhere to the private network is called: it closes. +const closesOutside = "WILL CLOSE to everything outside the private network" + // fate is what the derived filter does to one reachable thing: which module declares it and from -// where, or that it will close. -func fate(r inventory.Reach, rules []catalogue.Rule, foundation []int) string { +// where, or that it will close — wholly, or to everything outside the private network. Rendered +// exactly as AsNftables admits it, ssh included. +func (d derivedFilter) fate(r inventory.Reach) string { + // Bound to an address on the private network, it was never reachable from outside it, so + // admitting it from the mesh narrows nothing. + onMesh := slices.Contains(d.mesh, strings.Trim(r.Address, "[]")) if r.Protocol == "tcp" && r.Port == catalogue.SSHPort { - return "stays open — ssh is never closed" + // From everywhere only when the machine faces outward or the mesh has no addresses to + // narrow it to; otherwise from the private network only. + if d.outward || len(d.mesh) == 0 || onMesh { + return "stays open — ssh is never closed" + } + return closesOutside + " — ssh stays open from the mesh, never closed there" } - for _, port := range foundation { + for _, port := range d.foundation { if r.Protocol == "tcp" && r.Port == port { return "stays open — the mesh's own, from anywhere" } } - for _, rule := range rules { + for _, rule := range d.rules { if rule.Port != r.Port || rule.Protocol != r.Protocol { continue } - if rule.From == catalogue.FromMachine { - return fmt.Sprintf("WILL CLOSE to the network — declared by %s for this machine only", - strings.Join(rule.Because, ", ")) + by := strings.Join(rule.Because, ", ") + switch rule.From { + case catalogue.FromMachine: + return fmt.Sprintf("WILL CLOSE to the network — declared by %s for this machine only", by) + case catalogue.FromMesh: + if len(d.mesh) == 0 { + return fmt.Sprintf("WILL CLOSE — declared by %s from the mesh, and this node "+ + "knows no mesh addresses", by) + } + if !onMesh { + return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by) + } } - return fmt.Sprintf("declared by %s (from %s)", strings.Join(rule.Because, ", "), rule.From) + return fmt.Sprintf("declared by %s (from %s)", by, rule.From) } return "WILL CLOSE — no module assigned here declares it" } From ba0f44a36dde0bd7ca20219f33aa14e72070164f Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:01:04 +0200 Subject: [PATCH 14/33] Say in the converge preview that routed traffic is not previewed and is dropped unless declared (hq ADR 0100) --- cmd/mesh-controller/adopting_test.go | 3 +++ cmd/mesh-controller/adoption.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index e52a6f4..bc39707 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -171,6 +171,9 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) "notes\n replacing the found file /etc/notes.conf (original kept at", "assigns nftables", "the found firewall (ufw) is disabled, never flushed", + // What it routes is not a listener: said not to be previewed, and to be dropped. + "not previewed: traffic the machine routes that is not a published port", + "the derived filter drops it unless a module declares it", } { if !strings.Contains(preview, want) { t.Errorf("the preview does not say %q:\n%s", want, preview) diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index 152998c..b63616b 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -276,6 +276,11 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter, if len(reported.Reachable) == 0 { b.WriteString(" nothing reported\n") } + // What the machine routes for others is not a listener and not a published port, so nothing + // above can show it; the derived filter's forward chain drops it all the same. + b.WriteString(" not previewed: traffic the machine routes that is not a published port " + + "(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " + + "declares it\n") isTaken := map[string]bool{} for _, m := range taken { From c91fe1a6ebf6ac4448720dcb6c621b8d1989b797 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:02:30 +0200 Subject: [PATCH 15/33] Converge only on the preview the operator saw, named by its digest, and never on an account older than 15 minutes (hq ADR 0100) --- cmd/mesh-controller/adopting_test.go | 90 ++++++++++++++++++++++++++-- cmd/mesh-controller/adoption.go | 60 +++++++++++++++---- cmd/mesh-controller/api.go | 7 ++- cmd/mesh-controller/main.go | 2 +- 4 files changed, 139 insertions(+), 20 deletions(-) diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index bc39707..5de0064 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -115,7 +115,7 @@ func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) { func TestConvergingIsRefusedOnAPreviewThatWouldBeStale(t *testing.T) { open, sent := anAdoptedAnchor(t) - _, err := converge(t.Context(), open, "anchor", false, "") + _, err := converge(t.Context(), open, "anchor", false, "", "") if err == nil || !strings.Contains(err.Error(), "has not reported") { t.Fatalf("a node that never reported was previewed: %v", err) } @@ -127,7 +127,7 @@ func TestConvergingIsRefusedOnAPreviewThatWouldBeStale(t *testing.T) { func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) { open, sent := anAdoptedAnchor(t) reportsHolding(t, open, heldContainer, heldFile) - _, err := converge(t.Context(), open, "anchor", true, "") + _, err := converge(t.Context(), open, "anchor", true, "", "") if err == nil || !strings.Contains(err.Error(), "take anchor hello-web once its data has moved") { t.Fatalf("the flip was not refused while hello-web holds its found container: %v", err) } @@ -157,7 +157,7 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) } reportsHolding(t, open, heldFile) - preview, err := converge(ctx, open, "anchor", false, "") + preview, err := converge(ctx, open, "anchor", false, "", "") if err != nil { t.Fatal(err) } @@ -191,7 +191,7 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) t.Fatal("the preview changed something") } - if _, err := converge(ctx, open, "anchor", true, ""); err != nil { + if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil { t.Fatal(err) } n, _ := open.inventory.NodeByName(ctx, "anchor") @@ -245,7 +245,7 @@ func hasID(resources []map[string]any, id string) bool { func TestTheApiRefusesTheFlipInTheCommandLinesWords(t *testing.T) { open, _ := anAdoptedAnchor(t) reportsHolding(t, open, heldContainer) - _, direct := converge(t.Context(), open, "anchor", true, "") + _, direct := converge(t.Context(), open, "anchor", true, "", "") if direct == nil { t.Fatal("the flip was not refused") } @@ -283,7 +283,7 @@ func TestThePreviewSaysWhatNarrowsToTheMeshCloses(t *testing.T) { {Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80}, }) - preview, err := converge(ctx, open, "anchor", false, "") + preview, err := converge(ctx, open, "anchor", false, "", "") if err != nil { t.Fatal(err) } @@ -307,3 +307,81 @@ func TestThePreviewSaysWhatNarrowsToTheMeshCloses(t *testing.T) { t.Errorf("a port open to everywhere is not said to stay:\n%s", preview) } } + +// digestIn is the digest a converge preview printed. +func digestIn(t *testing.T, preview string) string { + t.Helper() + for _, line := range strings.Split(preview, "\n") { + if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" { + return fields[1] + } + } + t.Fatalf("the preview printed no digest:\n%s", preview) + return "" +} + +// The flip acts on the preview the operator saw: it names that preview's digest, and it is refused +// when the digest is missing, when anything the preview says has changed since, or when the node's +// account of itself is too old to be the machine as it is. +func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) { + open, sent := anAdoptedAnchor(t) + ctx := t.Context() + if _, err := take(ctx, open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + reportsHolding(t, open, heldFile) + preview, err := converge(ctx, open, "anchor", false, "", "") + if err != nil { + t.Fatal(err) + } + saw := digestIn(t, preview) + if !strings.Contains(preview, "converge anchor --yes "+saw) { + t.Fatalf("the preview does not say how to act on it:\n%s", preview) + } + unchanged := func() { + t.Helper() + if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 { + t.Fatal("a refused flip changed something") + } + } + + if _, err := converge(ctx, open, "anchor", true, "", ""); err == nil || + !strings.Contains(err.Error(), "--yes "+saw) { + t.Fatalf("a flip naming no preview was not refused: %v", err) + } + unchanged() + + // Something new is reachable: the preview the operator saw is not what would happen. + reportsReaching(t, open, []link.Reach{ + {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}, + {Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, + ContainerPort: 80}, + {Protocol: "tcp", Address: "0.0.0.0", Port: 6000, By: "something-new"}, + }, heldFile) + _, err = converge(ctx, open, "anchor", true, saw, "") + if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) { + t.Fatalf("a flip on a changed preview was not refused: %v", err) + } + unchanged() + + // An account older than the flip trusts is refused, whatever digest is named. + again, err := converge(ctx, open, "anchor", false, "", "") + if err != nil { + t.Fatal(err) + } + saved := reportFreshFor + reportFreshFor = time.Nanosecond + _, err = converge(ctx, open, "anchor", true, digestIn(t, again), "") + reportFreshFor = saved + if err == nil || !strings.Contains(err.Error(), "wait for its next report") { + t.Fatalf("a flip on an old account was not refused: %v", err) + } + unchanged() + + if _, err := converge(ctx, open, "anchor", true, digestIn(t, again), ""); err != nil { + t.Fatalf("the flip on the preview just seen was refused: %v", err) + } + if n, _ := open.inventory.NodeByName(ctx, "anchor"); n.Adopted { + t.Fatal("the flip did not converge the node") + } +} diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index b63616b..a3a92b0 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -2,6 +2,8 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "flag" "fmt" @@ -127,6 +129,10 @@ func take(ctx context.Context, open *stores, node, module string) (string, error return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil } +// reportFreshFor is how old a node's account of itself may be for the flip to act on it. A +// variable so a test can age a report without waiting. +var reportFreshFor = 15 * time.Minute + // converge previews, and with yes makes, the flip of an adopted node to converged: every module it // runs is taken, the filter module is assigned to load the mesh's derived filter in place of the // guard, and the found firewall is retired — disabled, never flushed — by the host. @@ -134,7 +140,14 @@ func take(ctx context.Context, open *stores, node, module string) (string, error // Refused while an assigned module still holds a found container: each service is taken on its // own, when its data has moved, never by the flip. And refused on a preview that would be stale: // what is reachable is the node's last account, so that account must be of what it was last sent. -func converge(ctx context.Context, open *stores, node string, yes bool, filter string) (string, error) { +// +// **The flip acts on the preview the operator saw** and on nothing else. The preview ends with a +// short digest of what it said — every reachable thing and its fate, the modules the flip takes and +// the filter — and yes must name that digest: if anything the preview would say has changed since, +// the flip is refused rather than done on a preview nobody read. And it is refused on an account +// older than reportFreshFor: what was reachable then is not evidence of what is reachable now. +func converge(ctx context.Context, open *stores, node string, yes bool, digest string, + filter string) (string, error) { inv := open.inventory if filter == "" { filter = DefaultFilter @@ -219,9 +232,25 @@ func converge(ctx context.Context, open *stores, node string, yes bool, filter s } derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh, outward: plan.PublicDomain != ""} - preview := previewOf(node, reported, derived, plan, taken, filter, runs[filter]) + preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter]) + preview += "\n\n preview " + saw if !yes { - return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes` to do it.", node), nil + return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+ + "it.", node, saw), nil + } + if age := time.Since(reported.At); age > reportFreshFor { + return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+ + "only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+ + "then preview again", node, age.Round(time.Second), reportFreshFor, node) + } + if digest == "" { + return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+ + "`converge %s --yes %s`, once you have read it", node, node, saw) + } + if digest != saw { + return preview, fmt.Errorf("what converging %s would do has changed since preview %s "+ + "(it is now %s): read the preview above, and run `converge %s --yes %s` if it is "+ + "what you want", node, digest, saw, node, saw) } // The flip. The filter first, and only kept if the node still resolves with it: a node that @@ -253,9 +282,12 @@ func converge(ctx context.Context, open *stores, node string, yes bool, filter s return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil } -// previewOf is what converging a node will change, before it changes it. +// previewOf is what converging a node will change, before it changes it, and a short digest of +// what it said: every reachable thing and its fate, the modules the flip takes and the filter. The +// digest is what the flip is asked to act on, so it changes whenever any of those would. func previewOf(node string, reported inventory.Adoption, derived derivedFilter, - plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) string { + plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) (string, string) { + var said []string var b strings.Builder fmt.Fprintf(&b, "converging %s\n", node) fmt.Fprintf(&b, "\n reachable on the machine now, as it reported at %s:\n", @@ -271,7 +303,9 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter, if r.Published { what += fmt.Sprintf(" (published, container port %d)", r.ContainerPort) } - fmt.Fprintf(&b, " %-44s %s\n", what, derived.fate(r)) + fate := derived.fate(r) + fmt.Fprintf(&b, " %-44s %s\n", what, fate) + said = append(said, fmt.Sprintf("reach %s %s %s", r.Address, what, fate)) } if len(reported.Reachable) == 0 { b.WriteString(" nothing reported\n") @@ -302,6 +336,7 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter, } for _, m := range takes { fmt.Fprintf(&b, " %s\n", m) + said = append(said, "take "+m) for _, h := range reported.Held { if h.Module == m && h.Kind == "file" { fmt.Fprintf(&b, " replacing the found file %s", h.Target) @@ -321,7 +356,11 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter, } else { fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw) } - return strings.TrimRight(b.String(), "\n") + said = append(said, fmt.Sprintf("filter %s assigned=%t firewall=%s", filter, filterAssigned, fw)) + // Sorted: the same account, reported in another order, is the same preview. + sort.Strings(said) + sum := sha256.Sum256([]byte(strings.Join(said, "\n"))) + return strings.TrimRight(b.String(), "\n"), hex.EncodeToString(sum[:])[:12] } // derivedFilter is what the filter the flip loads is rendered from, as AsNftables renders it. @@ -416,17 +455,18 @@ func takeCommand(ctx context.Context, args []string) error { func convergeCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("converge", flag.ContinueOnError) - yes := set.Bool("yes", false, "do it; without it, only the preview") + yes := set.String("yes", "", "do it, naming the digest the preview printed; without it, only "+ + "the preview") filter := set.String("filter", DefaultFilter, "the module that loads the mesh's filter") positionals, err := parseAround(set, args) if err != nil { return err } if len(positionals) != 1 { - return errors.New("converge [--yes] [--filter nftables]") + return errors.New("converge [--yes ] [--filter nftables]") } return runAct(ctx, func(open *stores) (string, error) { - return converge(ctx, open, positionals[0], *yes, *filter) + return converge(ctx, open, positionals[0], *yes != "", *yes, *filter) }) } diff --git a/cmd/mesh-controller/api.go b/cmd/mesh-controller/api.go index 182c7c9..b1b6a25 100644 --- a/cmd/mesh-controller/api.go +++ b/cmd/mesh-controller/api.go @@ -105,7 +105,7 @@ func commands(who Authenticator) http.Handler { return take(ctx, open, in.Node, in.Module) })) mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) { - return converge(ctx, open, in.Node, in.Yes, in.Filter) + return converge(ctx, open, in.Node, in.Yes, in.Digest, in.Filter) })) mux.HandleFunc("POST /adopt", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) { return adopt(ctx, open, in.Node) @@ -124,9 +124,10 @@ func commands(who Authenticator) http.Handler { type request struct { Node string `json:"node"` Module string `json:"module"` - // Yes and Filter are converge's: do it rather than preview it, and which module loads the - // mesh's filter. + // Yes, Digest and Filter are converge's: do it rather than preview it, the digest of the + // preview it acts on, and which module loads the mesh's filter. Yes bool `json:"yes,omitempty"` + Digest string `json:"digest,omitempty"` Filter string `json:"filter,omitempty"` } diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 0fc5b2c..8442f42 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -162,7 +162,7 @@ func usage() { assign put a module on a node unassign take it off take cut a module over on an adopted node, once its data has moved - converge [--yes] [--filter nftables] preview, then make, an adopted node converged + converge [--yes ] [--filter nftables] preview, then make, an adopted node converged adopt return a converged node to adopted; what was taken stays taken settings set what a module's config should say, for the whole mesh settings set --node ...or for one machine From 41c300eae0eb6a1adb266181066fffa2dd1c9322 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:03:03 +0200 Subject: [PATCH 16/33] Name every kind of an untaken module's resources in the adoption envelope, not only files and containers (hq ADR 0103) --- cmd/mesh-controller/sendable.go | 16 ++++++++-------- cmd/mesh-controller/sendable_test.go | 6 ++++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index fcf182c..8c282ac 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "sort" + "strings" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" @@ -23,8 +24,8 @@ type sendable struct { } // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on -// it that it runs; Untaken is, for every module it runs that is not taken, the ids of that module's -// files and containers — the resources the host keeps as found until the module is taken. Ids +// it that it runs; Untaken is, for every module it runs that is not taken, the ids of every one of +// that module's resources — what the host keeps as found until the module is taken (ADR 0103). Ids // rather than a rule to split them by, because a module's name may contain a dot. type adoptionEnvelope struct { Taken []string `json:"taken"` @@ -75,12 +76,11 @@ func adoptionFor(plan catalogue.Resolution, taken []string, for _, r := range composed.Resources { id, _ := r["id"].(string) module, owned := composed.Owner[id] - if !owned || !runs[module] || isTaken[module] { - continue - } - switch r["type"] { - case "file", "container": - default: + // Every kind, not only files and containers (novox/hq ADR 0103): a directory, a service, a + // container mounting what was found and an action run in a held container all reach what + // the machine already has. What the mesh declares of its own is never held. + if !owned || !runs[module] || isTaken[module] || + strings.HasPrefix(id, catalogue.AdoptionPrefix) { continue } if out.Untaken == nil { diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go index 72d9ffb..cecfbe0 100644 --- a/cmd/mesh-controller/sendable_test.go +++ b/cmd/mesh-controller/sendable_test.go @@ -84,8 +84,10 @@ func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) { t.Fatal("an adopted node's declaration does not say it is adopted") } untaken := declared.Adoption.Untaken["hello-web"] - if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server"}) { - t.Fatalf("hello-web's files and containers are not named untaken: %v", declared.Adoption) + // Every kind — its directory too (novox/hq ADR 0103). + if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server", + "hello-web.served"}) { + t.Fatalf("hello-web's resources are not all named untaken: %v", declared.Adoption) } if len(declared.Adoption.Taken) != 0 { t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken) From 9280513afab3f74a6061eb0c5c6ceba97dfc5143 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:03:45 +0200 Subject: [PATCH 17/33] Refuse token issue --adopted for a converged node and point to adopt, rather than flip it quietly (hq ADR 0100) --- cmd/mesh-controller/nodes.go | 17 ++++++++++++----- cmd/mesh-controller/nodes_test.go | 15 ++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index 1ca2257..d7ce9dd 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -279,13 +279,20 @@ func issueFor(ctx context.Context, inv *inventory.Inventory, existing, fresh str } name = node.Name } - // Saying adopted makes the node adopted. Not saying it leaves the node as it is: re-issuing a - // token for an adopted node does not converge it — converging is its own act, previewed - // (novox/hq ADR 0100). - if adopted { - if err := inv.SetAdopted(ctx, name, true); err != nil { + // Not saying adopted leaves the node as it is: re-issuing a token for an adopted node does not + // converge it — converging is its own act, previewed (novox/hq ADR 0100). And saying it for a + // node already converged is refused rather than done quietly: returning a node to adopted is + // its own act too, which unloads the mesh's filter and enables the found firewall again. + if adopted && fresh == "" { + node, err := inv.NodeByName(ctx, name) + if err != nil { return inventory.Issued{}, err } + if !node.Adopted { + return inventory.Issued{}, fmt.Errorf("%s is converged, and a token does not change "+ + "that: run `adopt %s` to return it to adopted, then issue the token without "+ + "--adopted", name, name) + } } return inv.IssueToken(ctx, name, validFor) diff --git a/cmd/mesh-controller/nodes_test.go b/cmd/mesh-controller/nodes_test.go index c9f55a7..86b9558 100644 --- a/cmd/mesh-controller/nodes_test.go +++ b/cmd/mesh-controller/nodes_test.go @@ -139,11 +139,16 @@ func TestATokenIssuedAdoptedSaysSoAndReissuingDoesNotConverge(t *testing.T) { if !again.Node.Adopted { t.Fatal("re-issuing without --adopted converged the node; converging is its own act") } - existing, err := issueFor(ctx, open.inventory, "laptop", "", true, time.Hour) - if err != nil { + // --adopted for a node already converged is refused, and points at the act that does it. + if _, err := issueFor(ctx, open.inventory, "laptop", "", true, time.Hour); err == nil || + !strings.Contains(err.Error(), "adopt laptop") { + t.Fatalf("--adopted on a converged node was not refused: %v", err) + } + if n, _ := open.inventory.NodeByName(ctx, "laptop"); n.Adopted { + t.Fatal("a refused token flipped the node to adopted") + } + // And said for a node that is adopted already, it is the ordinary re-issue. + if _, err := issueFor(ctx, open.inventory, "joiner", "", true, time.Hour); err != nil { t.Fatal(err) } - if !existing.Node.Adopted { - t.Fatal("--adopted on an existing record did not make it adopted") - } } From 4bb19c9e4067d652293022019961e18e7fc1d9ba Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:05:31 +0200 Subject: [PATCH 18/33] Give a machine port one holder: refuse ssh's, another module's and a doubled one, and release the assignment a given port replaces (hq ADR 0100) --- cmd/mesh-controller/plan.go | 14 +++ internal/catalogue/adoption_test.go | 8 ++ internal/catalogue/filtering.go | 13 +++ internal/inventory/catalogue.go | 152 +++++++++++++++++++++++++++- internal/inventory/ports_test.go | 61 +++++++++++ 5 files changed, 246 insertions(+), 2 deletions(-) diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 0f540c7..4b5cbbf 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -384,6 +384,20 @@ func renderingFor(ctx context.Context, open *stores, node string, given[m.Module] = g } } + // And one holder per machine port across the node's modules: settings written before this was + // refused where they are set are refused here rather than composed into two containers on + // one port. + holders := map[int]string{} + for _, m := range plan.Modules { + for _, at := range given[m.Module] { + if other, twice := holders[at]; twice && other != m.Module { + return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf("%w: %s and %s are "+ + "both given machine port %d on %s", inventory.ErrPortTaken, other, m.Module, + at, node) + } + holders[at] = m.Module + } + } ports := map[string]map[int]int{} for _, m := range plan.Modules { diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index 0d839e1..b53e956 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -317,6 +317,14 @@ func TestAGivenPortIsTheNodesAndReachesSomething(t *testing.T) { if _, err := GivenPorts(store, node(map[string]any{"5432": float64(70000)})); err == nil { t.Fatal("a machine port that is not a port was given") } + if _, err := GivenPorts(store, node(map[string]any{"5432": float64(22)})); err == nil { + t.Fatal("ssh's port was given") + } + broker := anAdoptedAnchor().Modules[2] + if _, err := GivenPorts(broker, node(map[string]any{"5671": float64(5700), + "5672": float64(5700)})); err == nil { + t.Fatal("one machine port was given for two of the module's ports") + } if stray := UnusedSettings(store, node(map[string]any{"5432": float64(5433)})); len(stray) != 0 { t.Fatalf("a given port is called stray: %v", stray) } diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index ae1354f..453f651 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -518,9 +518,22 @@ func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) { return nil, fmt.Errorf("%s gives port %d the machine port %v, which is not a port", m.Module, port, value) } + if at == SSHPort { + return nil, fmt.Errorf("%s gives port %d the machine port %d, which is ssh's — the "+ + "one port a machine may never lose", m.Module, port, at) + } out[port] = at } } + // One holder per machine port, within the module too. + holder := map[int]int{} + for port, at := range out { + if other, twice := holder[at]; twice { + return nil, fmt.Errorf("%s gives machine port %d to both its %d and its %d", m.Module, + at, min(port, other), max(port, other)) + } + holder[at] = port + } if len(out) == 0 { return nil, nil } diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 322de46..65a6e5d 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "sort" + "strconv" "strings" "github.com/jackc/pgx/v5" @@ -593,11 +595,157 @@ func (i *Inventory) SetSettings(ctx context.Context, nodeName, module string, va if err != nil { return err } - _, err = i.store.Pool().Exec(ctx, + given, err := givenIn(module, raw) + if err != nil { + return err + } + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + if len(given) > 0 { + if err := refuseGivenCollisions(ctx, tx, node.ID, nodeName, module, given); err != nil { + return err + } + } + _, err = tx.Exec(ctx, `insert into settings (node, module, values) values ($1, $2, $3) on conflict (node, module) where node is not null do update set values = excluded.values, set_at = now()`, node.ID, module, raw) - return wrapModule(err, module) + if err != nil { + return wrapModule(err, module) + } + // A given port replaces what the mesh assigned for that port: the assignment is given back, + // so the number is free for the next module rather than held for ever in the name of a port + // that now lives elsewhere. + for wanted := range given { + if _, err := tx.Exec(ctx, + `delete from port_assignment where node = $1 and module = $2 and wanted = $3`, + node.ID, module, wanted); err != nil { + return err + } + } + return tx.Commit(ctx) +} + +// givenIn is the machine ports a node-level settings layer gives a module, software port → +// machine port (novox/hq ADR 0100). Nothing when the layer gives none; what is not a port is left +// for composition to refuse in its own words. +func givenIn(module string, raw []byte) (map[int]int, error) { + var layer map[string]any + if err := json.Unmarshal(raw, &layer); err != nil { + return nil, err + } + entries, ok := layer[catalogue.PortsSetting].(map[string]any) + if !ok { + return nil, nil + } + out := map[int]int{} + by := map[int]int{} + for text, value := range entries { + wanted, err := strconv.Atoi(text) + if err != nil { + continue + } + at, ok := value.(float64) + if !ok { + continue + } + machine := int(at) + if machine == catalogue.SSHPort { + return nil, fmt.Errorf("%s cannot be given port %d for its %d: that is ssh's, the one "+ + "port a machine may never lose", module, machine, wanted) + } + if other, twice := by[machine]; twice { + return nil, fmt.Errorf("%s gives machine port %d to both its %d and its %d; a machine "+ + "port has one holder", module, machine, min(other, wanted), max(other, wanted)) + } + by[machine] = wanted + out[wanted] = machine + } + return out, nil +} + +// refuseGivenCollisions refuses a given machine port another module on the node already has — +// assigned by the mesh or given by its own setting — or that this module has for another of its +// ports. A port it was assigned for the same software port is not a collision: the given one +// replaces it. +func refuseGivenCollisions(ctx context.Context, tx pgx.Tx, nodeID any, node, module string, + given map[int]int) error { + type holder struct { + module string + wanted int + } + held := map[int]holder{} + rows, err := tx.Query(ctx, + `select machine, module, wanted from port_assignment where node = $1`, nodeID) + if err != nil { + return err + } + for rows.Next() { + var h holder + var machine int + if err := rows.Scan(&machine, &h.module, &h.wanted); err != nil { + rows.Close() + return err + } + held[machine] = h + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + rows, err = tx.Query(ctx, + `select module, values->'ports' from settings + where node = $1 and module <> $2 and jsonb_typeof(values->'ports') = 'object'`, + nodeID, module) + if err != nil { + return err + } + for rows.Next() { + var other string + var raw []byte + if err := rows.Scan(&other, &raw); err != nil { + rows.Close() + return err + } + var theirs map[string]any + if err := json.Unmarshal(raw, &theirs); err != nil { + rows.Close() + return err + } + for text, v := range theirs { + wanted, _ := strconv.Atoi(text) + if at, ok := v.(float64); ok { + held[int(at)] = holder{module: other, wanted: wanted} + } + } + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + + wanted := make([]int, 0, len(given)) + for w := range given { + wanted = append(wanted, w) + } + sort.Ints(wanted) + for _, w := range wanted { + machine := given[w] + h, taken := held[machine] + if !taken || (h.module == module && h.wanted == w) { + continue + } + if _, moving := given[h.wanted]; h.module == module && moving { + // Its own port for another of its software ports, which this same layer moves away. + continue + } + return fmt.Errorf("%w: %s cannot be given %d on %s for its %d — %s already has it for "+ + "its %d", ErrPortTaken, module, machine, node, w, h.module, h.wanted) + } + return nil } func wrapModule(err error, module string) error { diff --git a/internal/inventory/ports_test.go b/internal/inventory/ports_test.go index b62f310..21a3761 100644 --- a/internal/inventory/ports_test.go +++ b/internal/inventory/ports_test.go @@ -257,3 +257,64 @@ func TestAGivenPortIsNeverAssigned(t *testing.T) { t.Fatalf("a fixed port given to another module was handed over: %v", err) } } + +// novox/hq ADR 0100: a given machine port has one holder. It is refused when another module has it, +// assigned or given, when the same layer gives it twice, and when it is ssh's; and when it replaces +// what the mesh assigned for that port, the assignment is given back. +func TestAGivenPortHasOneHolderAndReplacesTheAssignment(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "web", "cache") + ctx := t.Context() + give := func(module string, ports map[string]any) error { + return inv.SetSettings(ctx, node, module, map[string]any{catalogue.PortsSetting: ports}) + } + + web, err := inv.PortFor(ctx, node, "web", 8080, false) + if err != nil { + t.Fatal(err) + } + if err := give("postgres", map[string]any{"5432": web.Machine}); !errors.Is(err, ErrPortTaken) { + t.Fatalf("a port the mesh assigned to web was given to postgres: %v", err) + } + if err := give("postgres", map[string]any{"5432": 22}); err == nil { + t.Fatal("ssh's port was given") + } + if err := give("postgres", map[string]any{"5432": 5433, "5433": 5433}); err == nil { + t.Fatal("one machine port was given for two ports") + } + if err := give("cache", map[string]any{"6379": 6380}); err != nil { + t.Fatal(err) + } + if err := give("postgres", map[string]any{"5432": 6380}); !errors.Is(err, ErrPortTaken) { + t.Fatalf("a port given to cache was given to postgres: %v", err) + } + + // Postgres was assigned a port for 5432; given one, the assignment is released and the number + // is free again. + assigned, err := inv.PortFor(ctx, node, "postgres", 5432, false) + if err != nil { + t.Fatal(err) + } + if err := give("postgres", map[string]any{"5432": 5433}); err != nil { + t.Fatal(err) + } + // Given again, the same: its own given port is not a collision with itself. + if err := give("postgres", map[string]any{"5432": 5433}); err != nil { + t.Fatal(err) + } + held, err := inv.PortsFor(ctx, node) + if err != nil { + t.Fatal(err) + } + for _, a := range held { + if a.Module == "postgres" { + t.Fatalf("the assignment a given port replaced is still held: %+v", a) + } + } + other, err := inv.PortFor(ctx, node, "cache", 11211, false) + if err != nil { + t.Fatal(err) + } + if other.Machine != assigned.Machine { + t.Fatalf("the released port %d was not free again (got %d)", assigned.Machine, other.Machine) + } +} From 1eff586a40461fd0bdb54dfcfc6ba060096abe8b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:06:15 +0200 Subject: [PATCH 19/33] Hold the filter module to replacing the stock unit's flushing stop (hq ADR 0100) --- .../catalogue/foundation_manifests_test.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/catalogue/foundation_manifests_test.go b/internal/catalogue/foundation_manifests_test.go index 08886c4..f8f524e 100644 --- a/internal/catalogue/foundation_manifests_test.go +++ b/internal/catalogue/foundation_manifests_test.go @@ -1,6 +1,7 @@ package catalogue import ( + "fmt" "os" "reflect" "strings" @@ -34,12 +35,13 @@ func TestTheStoreAndTheBrokerSayWhatTheMeshGuards(t *testing.T) { func TestTheFilterModuleNeverFlushesTheRuleset(t *testing.T) { m := catalogueManifest(t, "nftables") - var unit map[string]any - var load map[string]any + var unit, stock, load map[string]any for _, r := range m.Resources { switch r["id"] { case "unit": unit = r + case "stock-unit-stop": + stock = r case "load": load = r } @@ -60,7 +62,16 @@ func TestTheFilterModuleNeverFlushesTheRuleset(t *testing.T) { t.Fatalf("the unit does not load the computed rule set and delete only its own table:\n%s", content) } - if !reflect.DeepEqual(load["restart-on"], []any{"filtering", "unit"}) { - t.Fatalf("the filter is not reloaded when its rules or its unit change: %v", load["restart-on"]) + // A node converged before the filter had its own unit still has the stock nftables.service + // enabled, whose stop flushes the whole ruleset: a drop-in makes it delete only the mesh's + // table, and the load is restarted on it so the host reloads units and the drop-in is read. + if stock == nil || stock["path"] != "/etc/systemd/system/nftables.service.d/mesh.conf" || + !strings.HasSuffix(fmt.Sprint(stock["content"]), + "[Service]\nExecStop=\nExecStop=nft delete table inet mesh\n") { + t.Fatalf("the stock unit's stop is not replaced with deleting the mesh's table: %v", stock) + } + if !reflect.DeepEqual(load["restart-on"], []any{"filtering", "unit", "stock-unit-stop"}) { + t.Fatalf("the filter is not reloaded when its rules, its unit or the stock unit's drop-in "+ + "change: %v", load["restart-on"]) } } From 689c2c6d331248a060dda4bcb256e172db05876a Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:10:04 +0200 Subject: [PATCH 20/33] Hold a node from composing to sending, so a push composed before converge or adopt is never sent after it (hq ADR 0100) --- cmd/mesh-controller/adopting_test.go | 46 +++++++++++++++++++++++++++ cmd/mesh-controller/adoption.go | 16 ++++++++++ cmd/mesh-controller/hold.go | 37 ++++++++++++++++++++++ cmd/mesh-controller/push.go | 34 +++++++++++++++++--- internal/inventory/hold.go | 47 ++++++++++++++++++++++++++++ internal/inventory/hold_test.go | 44 ++++++++++++++++++++++++++ 6 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 cmd/mesh-controller/hold.go create mode 100644 internal/inventory/hold.go create mode 100644 internal/inventory/hold_test.go diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index 5de0064..3157e46 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -385,3 +385,49 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) { t.Fatal("the flip did not converge the node") } } + +// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is +// composed after it — never sent after it with the node still adopted. And the send inside the +// flip is not made to wait on the flip's own hold. +func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) { + open, _ := anAdoptedAnchor(t) + ctx := t.Context() + if _, err := take(ctx, open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + reportsHolding(t, open, heldFile) + preview, err := converge(ctx, open, "anchor", false, "", "") + if err != nil { + t.Fatal(err) + } + var heldElsewhere, heldHere error + sendNodes = func(inner context.Context, open *stores, names []string) error { + // Another caller cannot hold the node while the flip sends it. + waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond) + defer cancel() + if release, err := open.inventory.HoldNodes(waiting, names); err == nil { + release() + heldElsewhere = errors.New("another caller held the node while the flip sent it") + } + // The flip's own send holds it without waiting on itself. + _, release, err := holdNodes(inner, open, names) + if err != nil { + heldHere = err + return err + } + release() + return nil + } + if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil { + t.Fatal(err) + } + if heldElsewhere != nil || heldHere != nil { + t.Fatalf("%v %v", heldElsewhere, heldHere) + } + // And given back once it is done. + release, err := open.inventory.HoldNodes(ctx, []string{"anchor"}) + if err != nil { + t.Fatal(err) + } + release() +} diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index a3a92b0..ed89c3d 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -152,6 +152,16 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s if filter == "" { filter = DefaultFilter } + if yes { + // Held from the checks to the send, so no push composed before the flip is sent after it + // and returns the node to adopted. + held, release, err := holdNodes(ctx, open, []string{node}) + if err != nil { + return "", err + } + defer release() + ctx = held + } record, err := inv.NodeByName(ctx, node) if err != nil { return "", err @@ -435,6 +445,12 @@ func adopt(ctx context.Context, open *stores, node string) (string, error) { if record.Adopted { return "", fmt.Errorf("%s is adopted already", node) } + held, release, err := holdNodes(ctx, open, []string{node}) + if err != nil { + return "", err + } + defer release() + ctx = held if err := inv.SetAdopted(ctx, node, true); err != nil { return "", err } diff --git a/cmd/mesh-controller/hold.go b/cmd/mesh-controller/hold.go new file mode 100644 index 0000000..6c7d4d8 --- /dev/null +++ b/cmd/mesh-controller/hold.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" +) + +// heldKey carries the nodes this call already holds, so an act that holds a node and then sends +// through sendTo does not wait on itself. +type heldKey struct{} + +// holdNodes holds the named nodes for composing and sending their declarations (novox/hq ADR +// 0100), skipping any the context already holds, and returns a context that says it holds them. +// Release gives back only what this call took. +func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) { + already, _ := ctx.Value(heldKey{}).(map[string]bool) + var take []string + for _, n := range names { + if !already[n] { + take = append(take, n) + } + } + if len(take) == 0 { + return ctx, func() {}, nil + } + release, err := open.inventory.HoldNodes(ctx, take) + if err != nil { + return ctx, nil, err + } + held := map[string]bool{} + for n := range already { + held[n] = true + } + for _, n := range take { + held[n] = true + } + return context.WithValue(ctx, heldKey{}, held), release, nil +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 86ac3d9..759cb34 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -282,8 +282,14 @@ func pushCommand(ctx context.Context, args []string) error { asked = append(asked, n.Name) } + // Held from composing to sending, so a converge on one of them cannot send between the two + // and be overtaken by what was composed before it (novox/hq ADR 0100). + held, release, err := holdNodes(ctx, open, asked) + if err != nil { + return err + } sending, refusals := composeEach(asked, func(node string) (sendable, error) { - plan, settings, err := planFor(ctx, open, node) + plan, settings, err := planFor(held, open, node) if err != nil { return sendable{}, err } @@ -294,10 +300,11 @@ func pushCommand(ctx context.Context, args []string) error { // The private network is in here with everything else. It used to be composed separately // and prepended, which meant every machine with an address was on it and no machine could // be kept off. It is a module now, so it arrives the way a module does. - return declarationWith(ctx, open, node, plan, settings, gens, Allocating) + return declarationWith(held, open, node, plan, settings, gens, Allocating) }) sentDigest := map[string]string{} + defer release() for _, s := range sending { body, err := s.declared.Body() if err != nil { @@ -319,6 +326,7 @@ func pushCommand(ctx context.Context, args []string) error { sentDigest[s.node] = digest fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources)) } + release() fmt.Printf("\n%d node(s) told\n", len(sending)) // **A named push leaves the mesh consistent, not just the machine it named** (novox/hq @@ -373,13 +381,19 @@ func pushCommand(ctx context.Context, args []string) error { // (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is // all-or-nothing — so one swept machine's compose error failed the operator's named // push and skipped its --wait, the very intolerance the main path exists to avoid. + // Held for this round only, and after the last round's were given back, so two pushes + // cascading into each other's machines never each wait on the other. + held, release, err := holdNodes(ctx, open, also) + if err != nil { + return err + } sending, refused := composeEach(also, func(node string) (sendable, error) { - plan, settings, err := planFor(ctx, open, node) + plan, settings, err := planFor(held, open, node) if err != nil { return sendable{}, err } reportUnhostable(node, plan) - return declarationWith(ctx, open, node, plan, settings, gens, Allocating) + return declarationWith(held, open, node, plan, settings, gens, Allocating) }) refusals = append(refusals, refused...) for _, s := range sending { @@ -388,17 +402,21 @@ func pushCommand(ctx context.Context, args []string) error { return err } if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { + release() return err } record, err := inv.NodeByName(ctx, s.node) if err != nil { + release() return err } if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + release() return err } fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources)) } + release() // Every candidate this round is marked handled — the sent ones so they are not // re-listed, and the refused ones so a machine that cannot be composed does not make // the loop spin on it for ever. Its refusal is already in the report. @@ -532,6 +550,14 @@ func sendTo(ctx context.Context, open *stores, names []string) error { return err } + // Held from composing to sending (novox/hq ADR 0100); a caller that holds them already — + // converge, which flips the node and then sends it — is not made to wait on itself. + ctx, release, err := holdNodes(ctx, open, names) + if err != nil { + return err + } + defer release() + var sending []readyNode var refusals []string for _, name := range names { diff --git a/internal/inventory/hold.go b/internal/inventory/hold.go new file mode 100644 index 0000000..c9d2dd3 --- /dev/null +++ b/internal/inventory/hold.go @@ -0,0 +1,47 @@ +package inventory + +import ( + "context" + "slices" + "sync" +) + +// HoldNodes serialises composing and sending a declaration per node (novox/hq ADR 0100): while +// one caller holds a node, another asking for it waits. Without it a push that composed a node as +// adopted could send that declaration after `converge --yes` sent the converged one, and the node +// would return to adopted with nobody having asked. +// +// Session-level advisory locks on one connection, taken in name order so two callers holding +// overlapping sets cannot each wait on the other. Release gives every one back, and may be called +// more than once; a caller whose context ends while waiting holds nothing. +func (i *Inventory) HoldNodes(ctx context.Context, names []string) (func(), error) { + sorted := slices.Clone(names) + slices.Sort(sorted) + sorted = slices.Compact(sorted) + conn, err := i.store.Pool().Acquire(ctx) + if err != nil { + return nil, err + } + var once sync.Once + release := func() { + once.Do(func() { + // Unlocking all of this session's advisory locks, then handing the connection back: a + // connection returned still holding one would hold it for whoever borrows it next. + _, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`) + if err != nil { + // The session's locks die with the session: close it rather than return it. + _ = conn.Conn().Close(context.WithoutCancel(ctx)) + } + conn.Release() + }) + } + for _, name := range sorted { + if _, err := conn.Exec(ctx, + `select pg_advisory_lock(hashtext('mesh-node-declaration:' || $1)::bigint)`, + name); err != nil { + release() + return nil, err + } + } + return release, nil +} diff --git a/internal/inventory/hold_test.go b/internal/inventory/hold_test.go new file mode 100644 index 0000000..70c110f --- /dev/null +++ b/internal/inventory/hold_test.go @@ -0,0 +1,44 @@ +package inventory + +import ( + "testing" + "time" +) + +// Composing and sending one node's declaration is serialised: a second holder waits for the first. +func TestHoldingANodeMakesTheNextHolderWait(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + release, err := inv.HoldNodes(ctx, []string{"anchor", "laptop"}) + if err != nil { + t.Fatal(err) + } + got := make(chan func(), 1) + go func() { + second, err := inv.HoldNodes(ctx, []string{"laptop"}) + if err != nil { + t.Error(err) + got <- func() {} + return + } + got <- second + }() + select { + case <-got: + t.Fatal("a node held by one caller was held by another at the same time") + case <-time.After(300 * time.Millisecond): + } + // Another node is not held up. + other, err := inv.HoldNodes(ctx, []string{"joiner"}) + if err != nil { + t.Fatal(err) + } + other() + release() + select { + case second := <-got: + second() + case <-time.After(5 * time.Second): + t.Fatal("releasing the node did not let the next holder in") + } +} From bfe4991dd76b8c2d19157e1861ac6d160ef08284 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:27:19 +0200 Subject: [PATCH 21/33] Name every held kind of each module the flip takes in the converge preview and its digest (hq ADR 0103) --- cmd/mesh-controller/adopting_test.go | 44 +++++++++++++++++++++++++++- cmd/mesh-controller/adoption.go | 23 ++++++++++----- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index 3157e46..b10b202 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -168,7 +168,7 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) // The anchor faces inward and is on the private network: the derived filter admits ssh // from the mesh only. "WILL CLOSE to everything outside the private network — ssh stays open from the mesh", - "notes\n replacing the found file /etc/notes.conf (original kept at", + "notes\n replacing the found file /etc/notes.conf (notes.conf), original kept at", "assigns nftables", "the found firewall (ufw) is disabled, never flushed", // What it routes is not a listener: said not to be previewed, and to be dropped. @@ -431,3 +431,45 @@ func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) { } release() } + +// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found, +// not only its files, and the digest changes when any of them does. +func TestThePreviewNamesEveryHeldKind(t *testing.T) { + open, _ := anAdoptedAnchor(t) + ctx := t.Context() + if _, err := take(ctx, open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + since := time.Now() + held := []link.Held{heldFile, + {ID: "notes.data", Module: "notes", Kind: "directory", Target: "/var/lib/notes", Since: since}, + {ID: "notes.daemon", Module: "notes", Kind: "service", Target: "notes.service", Since: since}, + {ID: "notes.seed", Module: "notes", Kind: "archive", Target: "/srv/notes", Since: since}, + {ID: "notes.worker", Module: "notes", Kind: "process", Target: "notes-worker", Since: since}, + {ID: "notes.account", Module: "notes", Kind: "user", Target: "notes", Since: since}, + } + reportsHolding(t, open, held...) + preview, err := converge(ctx, open, "anchor", false, "", "") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "replacing the found directory /var/lib/notes (notes.data)", + "replacing the found service notes.service (notes.daemon)", + "replacing the found archive /srv/notes (notes.seed)", + "replacing the found process notes-worker (notes.worker)", + "replacing the found user notes (notes.account)", + } { + if !strings.Contains(preview, want) { + t.Errorf("the preview does not say %q:\n%s", want, preview) + } + } + reportsHolding(t, open, held[:len(held)-1]...) + fewer, err := converge(ctx, open, "anchor", false, "", "") + if err != nil { + t.Fatal(err) + } + if digestIn(t, fewer) == digestIn(t, preview) { + t.Fatal("the digest does not change with what is held") + } +} diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index ed89c3d..de93a6c 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -119,7 +119,7 @@ func take(ctx context.Context, open *stores, node, module string) (string, error var replaces []string for _, h := range reported.Held { if h.Module == module { - replaces = append(replaces, fmt.Sprintf(" %s %s (%s)", h.Kind, h.Target, h.ID)) + replaces = append(replaces, " "+heldLine(h)) } } if len(replaces) > 0 { @@ -347,14 +347,18 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter, for _, m := range takes { fmt.Fprintf(&b, " %s\n", m) said = append(said, "take "+m) + // Every kind it holds — a directory, a service, an archive, a process, a user as well as a + // file (novox/hq ADR 0103) — each said, and each part of what the flip is asked to act on. for _, h := range reported.Held { - if h.Module == m && h.Kind == "file" { - fmt.Fprintf(&b, " replacing the found file %s", h.Target) - if h.Kept != "" { - fmt.Fprintf(&b, " (original kept at %s)", h.Kept) - } - b.WriteString("\n") + if h.Module != m { + continue } + fmt.Fprintf(&b, " replacing the found %s", heldLine(h)) + if h.Kept != "" { + fmt.Fprintf(&b, ", original kept at %s", h.Kept) + } + b.WriteString("\n") + said = append(said, "replace "+m+" "+heldLine(h)+" "+h.Kept) } } if !filterAssigned { @@ -427,6 +431,11 @@ func (d derivedFilter) fate(r inventory.Reach) string { return "WILL CLOSE — no module assigned here declares it" } +// heldLine is one thing a node holds as found, as take and the converge preview both say it. +func heldLine(h inventory.Held) string { + return fmt.Sprintf("%s %s (%s)", h.Kind, h.Target, h.ID) +} + // loopback is an address nothing off the machine reaches. func loopback(address string) bool { a := strings.Trim(address, "[]") From cc473308845841902adfde370b1758f23ed97bce Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:27:27 +0200 Subject: [PATCH 22/33] Reload the guard on its table rather than restart it, so a change leaves no port unguarded (hq ADR 0103) --- internal/catalogue/adoption.go | 8 +++++--- internal/catalogue/adoption_test.go | 8 ++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/internal/catalogue/adoption.go b/internal/catalogue/adoption.go index f38c7ae..c7957c0 100644 --- a/internal/catalogue/adoption.go +++ b/internal/catalogue/adoption.go @@ -207,8 +207,10 @@ func GuardUnitText() string { } // GuardResources are the guard as four resources of the existing kinds: the tool that loads it, -// the table, the unit, and the unit running, restarted when the table changes. Nothing when there -// is nothing to guard: an empty set is not a table nft loads. +// the table, the unit, and the unit running — reloaded when the table changes, so the new table +// replaces the old in one `nft -f` through the unit's ExecReload with no moment unguarded, and +// restarted only when the unit itself changes. Nothing when there is nothing to guard: an empty +// set is not a table nft loads. func GuardResources(ports []int) []map[string]any { if len(ports) == 0 { return nil @@ -220,6 +222,6 @@ func GuardResources(ports []int) []map[string]any { {"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(), "mode": "0644"}, {"id": GuardRunningID(), "type": "service", "unit": GuardUnit, "state": "running", - "boot": "enabled", "restart-on": []any{GuardID(), GuardUnitID()}}, + "boot": "enabled", "restart-on": []any{GuardUnitID()}, "reload-on": []any{GuardID()}}, } } diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index b53e956..949288a 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -174,8 +174,12 @@ func TestAnAdoptedNodeLoadsNoFilterOfTheMeshs(t *testing.T) { if pkg < 0 || pkg > table { t.Fatalf("nftables is not declared before the guard's table (%d, %d)", pkg, table) } - if !reflect.DeepEqual(got[GuardRunningID()]["restart-on"], []any{GuardID(), GuardUnitID()}) { - t.Fatalf("the guard is not reloaded when its table changes: %v", got[GuardRunningID()]) + // A changed table is reloaded — one `nft -f`, atomic — never restarted, which would delete the + // table and leave the ports unguarded until it is loaded again. Only a changed unit restarts. + if !reflect.DeepEqual(got[GuardRunningID()]["reload-on"], []any{GuardID()}) || + !reflect.DeepEqual(got[GuardRunningID()]["restart-on"], []any{GuardUnitID()}) { + t.Fatalf("the guard is not reloaded on its table and restarted on its unit: %v", + got[GuardRunningID()]) } // Nothing of the mesh's own is anybody's to hold. for id, module := range composed.Owner { From 2e6b9d30bdfb2e3bfcd9d5eb22287a7ed77124f4 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:31:00 +0200 Subject: [PATCH 23/33] Give a cascade round's hold back on every return, a body that cannot be marshalled included (hq ADR 0100) --- cmd/mesh-controller/hold_test.go | 45 +++++++++++++++++ cmd/mesh-controller/push.go | 83 ++++++++++++++++++++------------ 2 files changed, 97 insertions(+), 31 deletions(-) create mode 100644 cmd/mesh-controller/hold_test.go diff --git a/cmd/mesh-controller/hold_test.go b/cmd/mesh-controller/hold_test.go new file mode 100644 index 0000000..9c2b480 --- /dev/null +++ b/cmd/mesh-controller/hold_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "context" + "errors" + "testing" + "time" +) + +// A cascade round gives its hold back on every way out: a declaration that cannot be marshalled +// and a send that fails leave nobody waiting for the node. +func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + unmarshallable := func(context.Context, string) (sendable, error) { + return sendable{Resources: []map[string]any{{"id": "x", "bad": make(chan int)}}}, nil + } + plain := func(context.Context, string) (sendable, error) { + return sendable{Resources: []map[string]any{{"id": "x"}}}, nil + } + failing := func(readyNode, []byte) error { return errors.New("the broker went away") } + fine := func(readyNode, []byte) error { return nil } + + for name, round := range map[string]func() error{ + "a body that cannot be marshalled": func() error { + _, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine) + return err + }, + "a send that fails": func() error { + _, err := sendRound(ctx, open, []string{"anchor"}, plain, failing) + return err + }, + } { + if err := round(); err == nil { + t.Fatalf("%s was not an error", name) + } + waiting, cancel := context.WithTimeout(ctx, 2*time.Second) + release, err := open.inventory.HoldNodes(waiting, []string{"anchor"}) + cancel() + if err != nil { + t.Fatalf("after %s the node is still held: %v", name, err) + } + release() + } +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 759cb34..2319213 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -383,40 +383,34 @@ func pushCommand(ctx context.Context, args []string) error { // push and skipped its --wait, the very intolerance the main path exists to avoid. // Held for this round only, and after the last round's were given back, so two pushes // cascading into each other's machines never each wait on the other. - held, release, err := holdNodes(ctx, open, also) + refused, err := sendRound(ctx, open, also, + func(held context.Context, node string) (sendable, error) { + plan, settings, err := planFor(held, open, node) + if err != nil { + return sendable{}, err + } + reportUnhostable(node, plan) + return declarationWith(held, open, node, plan, settings, gens, Allocating) + }, + func(s readyNode, body []byte) error { + if err := link.Declare(ctx, server.Channel(), ident, s.node, body, + 15*time.Second); err != nil { + return err + } + record, err := inv.NodeByName(ctx, s.node) + if err != nil { + return err + } + if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + return err + } + fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources)) + return nil + }) + refusals = append(refusals, refused...) if err != nil { return err } - sending, refused := composeEach(also, func(node string) (sendable, error) { - plan, settings, err := planFor(held, open, node) - if err != nil { - return sendable{}, err - } - reportUnhostable(node, plan) - return declarationWith(held, open, node, plan, settings, gens, Allocating) - }) - refusals = append(refusals, refused...) - for _, s := range sending { - body, err := s.declared.Body() - if err != nil { - return err - } - if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { - release() - return err - } - record, err := inv.NodeByName(ctx, s.node) - if err != nil { - release() - return err - } - if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { - release() - return err - } - fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources)) - } - release() // Every candidate this round is marked handled — the sent ones so they are not // re-listed, and the refused ones so a machine that cannot be composed does not make // the loop spin on it for ever. Its refusal is already in the report. @@ -516,6 +510,33 @@ func composeEach(names []string, return sending, refusals } +// sendRound holds the named nodes, composes each and sends each that composed, and gives the hold +// back on every way out — a body that cannot be marshalled and a send that fails included +// (novox/hq ADR 0100). A node that cannot be composed is a refusal, not an error: the others are +// still sent. +func sendRound(ctx context.Context, open *stores, names []string, + compose func(held context.Context, node string) (sendable, error), + send func(s readyNode, body []byte) error) ([]string, error) { + held, release, err := holdNodes(ctx, open, names) + if err != nil { + return nil, err + } + defer release() + sending, refused := composeEach(names, func(node string) (sendable, error) { + return compose(held, node) + }) + for _, s := range sending { + body, err := s.declared.Body() + if err != nil { + return refused, err + } + if err := send(s, body); err != nil { + return refused, err + } + } + return refused, nil +} + // couldNotBeResolved is what a push ends with when some machines could not be worked out. // // **After the rest have been sent, never instead of sending them.** It is still an error, because From 65187d4de0e63bf64519fef7b034831d33c86d34 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 18:31:10 +0200 Subject: [PATCH 24/33] Wait for a held node without pinning a pool connection, and give up after a bounded wait naming it (hq ADR 0100) --- internal/inventory/hold.go | 60 ++++++++++++++++++++++++++++----- internal/inventory/hold_test.go | 44 ++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/internal/inventory/hold.go b/internal/inventory/hold.go index c9d2dd3..4ddf382 100644 --- a/internal/inventory/hold.go +++ b/internal/inventory/hold.go @@ -2,8 +2,22 @@ package inventory import ( "context" + "errors" + "fmt" "slices" + "strings" "sync" + "time" +) + +// ErrNodeBusy is a node another act kept holding for longer than a caller waits. +var ErrNodeBusy = errors.New("another act is composing or sending that node's declaration") + +// HoldWaitFor is how long HoldNodes waits for a node another act holds, and HoldPoll how often it +// looks again. Variables so a test need not wait minutes. +var ( + HoldWaitFor = 2 * time.Minute + HoldPoll = 250 * time.Millisecond ) // HoldNodes serialises composing and sending a declaration per node (novox/hq ADR 0100): while @@ -11,16 +25,39 @@ import ( // adopted could send that declaration after `converge --yes` sent the converged one, and the node // would return to adopted with nobody having asked. // -// Session-level advisory locks on one connection, taken in name order so two callers holding -// overlapping sets cannot each wait on the other. Release gives every one back, and may be called -// more than once; a caller whose context ends while waiting holds nothing. +// Session-level advisory locks on one connection, all or none: a set not wholly free is given back +// at once, so two callers holding overlapping sets never each wait on the other. **A waiter pins +// no connection.** It looks again every HoldPoll with a connection borrowed for the look, and gives +// up after HoldWaitFor with ErrNodeBusy naming the node — so a stuck holder costs the pool one +// connection, never one per caller queued behind it. Release gives every one back, and may be +// called more than once. func (i *Inventory) HoldNodes(ctx context.Context, names []string) (func(), error) { sorted := slices.Clone(names) slices.Sort(sorted) sorted = slices.Compact(sorted) + deadline := time.Now().Add(HoldWaitFor) + for { + release, busy, err := i.tryHold(ctx, sorted) + if err != nil || busy == "" { + return release, err + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("%w: %s has been held for over %s — try again once it is done", + ErrNodeBusy, busy, HoldWaitFor) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(HoldPoll): + } + } +} + +// tryHold takes every named node's lock or none, and says which node was busy when it took none. +func (i *Inventory) tryHold(ctx context.Context, sorted []string) (func(), string, error) { conn, err := i.store.Pool().Acquire(ctx) if err != nil { - return nil, err + return nil, "", err } var once sync.Once release := func() { @@ -36,12 +73,17 @@ func (i *Inventory) HoldNodes(ctx context.Context, names []string) (func(), erro }) } for _, name := range sorted { - if _, err := conn.Exec(ctx, - `select pg_advisory_lock(hashtext('mesh-node-declaration:' || $1)::bigint)`, - name); err != nil { + var took bool + if err := conn.QueryRow(ctx, + `select pg_try_advisory_lock(hashtext('mesh-node-declaration:' || $1)::bigint)`, + name).Scan(&took); err != nil { release() - return nil, err + return nil, "", err + } + if !took { + release() + return nil, strings.TrimSpace(name), nil } } - return release, nil + return release, "", nil } diff --git a/internal/inventory/hold_test.go b/internal/inventory/hold_test.go index 70c110f..7de1d61 100644 --- a/internal/inventory/hold_test.go +++ b/internal/inventory/hold_test.go @@ -1,6 +1,8 @@ package inventory import ( + "errors" + "strings" "testing" "time" ) @@ -42,3 +44,45 @@ func TestHoldingANodeMakesTheNextHolderWait(t *testing.T) { t.Fatal("releasing the node did not let the next holder in") } } + +// A waiter pins no pool connection while it waits, and gives up after a bounded wait saying which +// node is busy. +func TestAWaiterPinsNoConnectionAndGivesUp(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + release, err := inv.HoldNodes(ctx, []string{"anchor"}) + if err != nil { + t.Fatal(err) + } + defer release() + savedWait, savedPoll := HoldWaitFor, HoldPoll + HoldWaitFor, HoldPoll = 1500*time.Millisecond, 50*time.Millisecond + defer func() { HoldWaitFor, HoldPoll = savedWait, savedPoll }() + + pool := inv.store.Pool() + base := pool.Stat().AcquiredConns() + const waiters = 3 + done := make(chan error, waiters) + for range waiters { + go func() { + _, err := inv.HoldNodes(ctx, []string{"anchor"}) + done <- err + }() + } + // While they wait, the pool lends nothing to them for longer than a look. + pinned := 0 + for range 10 { + time.Sleep(60 * time.Millisecond) + if n := int(pool.Stat().AcquiredConns() - base); n > pinned { + pinned = n + } + } + if pinned >= waiters { + t.Fatalf("%d connections were held by %d waiters", pinned, waiters) + } + for range waiters { + if err := <-done; !errors.Is(err, ErrNodeBusy) || !strings.Contains(err.Error(), "anchor") { + t.Fatalf("a waiter did not give up naming the busy node: %v", err) + } + } +} From dd6aad4a2ff53dc0036634422dd39849d5eef861 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:44:35 +0200 Subject: [PATCH 25/33] Give a machine port only to a port a module's container publishes, which is the only one the mesh can move (hq ADR 0038) --- internal/catalogue/adoption_test.go | 13 +++++++++++++ internal/catalogue/filtering.go | 22 +++++++++++----------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index 949288a..ee6497b 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -397,3 +397,16 @@ func TestPublishedLeavesOutLoopbackInBothFamilies(t *testing.T) { t.Fatalf("published is %v, want %v", got, want) } } + +// novox/hq ADR 0038: only a published port is the mesh's to move. A module that binds the machine +// itself listens where its software was told to, so giving it a machine port is refused. +func TestAGivenPortIsRefusedForAPortNoContainerPublishes(t *testing.T) { + onTheMachine := Manifest{Module: "daemon", + Listens: []Listening{{Port: 9000, From: FromMesh}}, Guards: []int{9000}} + layers := []Layer{{From: "node anchor", + Values: map[string]any{PortsSetting: map[string]any{"9000": float64(9100)}}}} + _, err := GivenPorts(onTheMachine, layers) + if err == nil || !strings.Contains(err.Error(), "does not publish") { + t.Fatalf("a port no container publishes was given: %v", err) + } +} diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index 453f651..674ebe1 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -475,17 +475,16 @@ const MeshWideLayer = "the mesh" // GivenPorts reads a module's given machine ports from its settings: software port → machine port. // // Refused from a mesh-wide layer — a port is a fact about one machine, and one number for every -// machine is the collision this exists to avoid — and for a port the module neither listens on, -// publishes from a container, nor guards: a given port that reaches nothing is a setting somebody -// believes changed something. +// machine is the collision this exists to avoid — and for a port the module's containers do not +// publish. +// +// **Only a published port is the mesh's to move** (novox/hq ADR 0038). A container's mapping is +// what translates; a module binding the machine's network directly binds the number its software +// was configured with, and moving that number would put it in the filter, in the openings and in +// what consumers are told while the software still listens on the old one — a port that reads as +// moved and is not. func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) { known := map[int]bool{} - for _, l := range m.Listens { - known[l.Port] = true - } - for _, p := range m.Guards { - known[p] = true - } for _, p := range containerPorts(m) { known[p] = true } @@ -510,8 +509,9 @@ func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) { return nil, fmt.Errorf("%s gives %q a port, which is not a port", m.Module, portText) } if !known[port] { - return nil, fmt.Errorf("%s gives port %d a machine port, and it neither listens "+ - "on, publishes nor guards %d — the setting reaches nothing", m.Module, port, port) + return nil, fmt.Errorf("%s gives port %d a machine port, and no container of its "+ + "publishes %d — the mesh cannot move a port the module does not publish; the "+ + "software would go on listening where it was told to", m.Module, port, port) } at, ok := asPort(value) if !ok || at < 1 || at > 65535 { From 0f3eedd1637220eaf3c810fb6323427905e63638 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:44:35 +0200 Subject: [PATCH 26/33] Do not guard a port this node is told to open to everyone (hq ADR 0103) --- internal/catalogue/adoption_test.go | 19 +++++++++++++++++++ internal/catalogue/declaration.go | 17 +++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index ee6497b..6de58f1 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -410,3 +410,22 @@ func TestAGivenPortIsRefusedForAPortNoContainerPublishes(t *testing.T) { t.Fatalf("a port no container publishes was given: %v", err) } } + +// novox/hq ADR 0103: a guarded port this node is told to open to everyone is opened, not guarded. +// An opening from everywhere beside a guard dropping it is one statement refusing the other. +func TestAGuardedPortOpenedToEveryoneIsNotGuarded(t *testing.T) { + with := anchorRendering(true) + with.Settings["postgres"] = []Layer{{From: "node anchor", + Values: map[string]any{ExposeSetting: map[string]any{"5432": FromEverywhere}}}} + composed, err := anAdoptedAnchor().Compose(with) + if err != nil { + t.Fatal(err) + } + got := byID(composed.Resources) + if o := got["adoption.opening-tcp-5432-forwarded"]; o == nil || o["from"] != OpeningFromEverywhere { + t.Fatalf("the store's port is not opened to everyone: %v", o) + } + if guard, _ := got[GuardID()]["content"].(string); guard != AsGuard([]int{5672, 15672}) { + t.Fatalf("a port opened to everyone is still guarded:\n%s", guard) + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 0cc318f..b9cfdf7 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -610,13 +610,21 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules []Rule, with Rendering) []int { meshOnly := map[int]bool{} + fromEverywhere := map[int]bool{} for _, rule := range rules { - if rule.Protocol == "tcp" && rule.From == FromMesh { + if rule.Protocol != "tcp" { + continue + } + switch rule.From { + case FromMesh: meshOnly[rule.Port] = true + case FromEverywhere: + fromEverywhere[rule.Port] = true } } for _, port := range with.Foundation { delete(meshOnly, port) + fromEverywhere[port] = true } seen := map[int]bool{} var ports []int @@ -655,7 +663,12 @@ func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules } } } - guard(at) + // Not what this node is told to open to everyone: a per-node exposure setting that + // widens a guarded port is the operator saying so, and declaring an opening for it + // and a guard dropping it would be one statement refusing the other. + if !fromEverywhere[at] { + guard(at) + } } } sort.Ints(ports) From 87ecc9326edb001529f15790518a87d5f3922e0b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:45:35 +0200 Subject: [PATCH 27/33] Refuse a port given for the whole mesh where it is set, not at every node's composition (hq ADR 0100) --- cmd/mesh-controller/sendable_test.go | 12 +++++++----- internal/inventory/catalogue.go | 11 +++++++++++ internal/inventory/ports_test.go | 26 ++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go index cecfbe0..c434265 100644 --- a/cmd/mesh-controller/sendable_test.go +++ b/cmd/mesh-controller/sendable_test.go @@ -214,17 +214,19 @@ func TestConsumersAreToldTheGivenPort(t *testing.T) { } } + // A port for the whole mesh is refused where it is set, not stored to refuse every node's + // declaration afterwards. if err := open.inventory.SetSettings(ctx, "", "store", - map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err != nil { - t.Fatal(err) + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil || + !strings.Contains(err.Error(), "per node") { + t.Fatalf("a port given for the whole mesh was not refused: %v", err) } plan, settings, err := planFor(ctx, open, "anchor") if err != nil { t.Fatal(err) } - if _, err := declarationFor(ctx, open, "anchor", plan, settings); err == nil || - !strings.Contains(err.Error(), "per node") { - t.Fatalf("a port given for the whole mesh was not refused: %v", err) + if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil { + t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err) } } diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 65a6e5d..5ca6d12 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -585,6 +585,17 @@ func (i *Inventory) SetSettings(ctx context.Context, nodeName, module string, va return err } if nodeName == "" { + // A port is a fact about one machine (novox/hq ADR 0100). Refused here, in composition's + // words: stored, it refuses every node running the module at composition, and the mesh + // cannot be pushed at all until somebody finds the layer that did it. + given, err := givenIn(module, raw) + if err != nil { + return err + } + if len(given) > 0 { + return fmt.Errorf("%s: %s is given per node — a port is a fact about one machine; "+ + "set it with --node", module, catalogue.PortsSetting) + } _, err = i.store.Pool().Exec(ctx, `insert into settings (node, module, values) values (null, $1, $2) on conflict (module) where node is null diff --git a/internal/inventory/ports_test.go b/internal/inventory/ports_test.go index 21a3761..2ac93c8 100644 --- a/internal/inventory/ports_test.go +++ b/internal/inventory/ports_test.go @@ -2,6 +2,7 @@ package inventory import ( "errors" + "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" @@ -318,3 +319,28 @@ func TestAGivenPortHasOneHolderAndReplacesTheAssignment(t *testing.T) { t.Fatalf("the released port %d was not free again (got %d)", assigned.Machine, other.Machine) } } + +// novox/hq ADR 0100: a port is a fact about one machine, so a layer for the whole mesh cannot give +// one. Refused where it is set — stored, it refuses every node running the module at composition, +// and the mesh cannot be pushed until somebody finds the layer that did it. +func TestAPortGivenForTheWholeMeshIsRefusedWhereItIsSet(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres") + ctx := t.Context() + err := inv.SetSettings(ctx, "", "postgres", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}) + if err == nil || !strings.Contains(err.Error(), "per node") { + t.Fatalf("a port given for the whole mesh was accepted: %v", err) + } + layers, err := inv.SettingsFor(ctx, node, "postgres") + if err != nil { + t.Fatal(err) + } + if len(layers) != 0 { + t.Fatalf("the refused layer was stored: %v", layers) + } + // The same values for one machine are the ordinary setting. + if err := inv.SetSettings(ctx, node, "postgres", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil { + t.Fatal(err) + } +} From 2cf8739a8483fbd2a89045dba4e7f2dc4355caa7 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:45:35 +0200 Subject: [PATCH 28/33] Clear the whole account an adopted node gave when it converges (hq ADR 0100) --- internal/inventory/adoption.go | 2 +- internal/inventory/adoption_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/internal/inventory/adoption.go b/internal/inventory/adoption.go index feae77c..1edd697 100644 --- a/internal/inventory/adoption.go +++ b/internal/inventory/adoption.go @@ -118,7 +118,7 @@ func (i *Inventory) Converge(ctx context.Context, nodeName string) ([]string, er } if _, err := tx.Exec(ctx, `update node set adopted = false, adopted_since = null, converged_at = now(), - held = null, reachable = null + held = null, reachable = null, firewall = null, adoption_reported = null where id = $1`, node.ID); err != nil { return nil, err diff --git a/internal/inventory/adoption_test.go b/internal/inventory/adoption_test.go index 9e505a7..a0443f5 100644 --- a/internal/inventory/adoption_test.go +++ b/internal/inventory/adoption_test.go @@ -132,3 +132,30 @@ func TestATakenModuleOutlivesItsAssignmentAndReturningToAdopted(t *testing.T) { t.Fatalf("returning to adopted lost what was taken: %v", taken) } } + +// Converging clears the whole of a node's account of itself: what it held, what was reachable, the +// firewall it found and when it said so. Keeping any of it would have `node show` report an +// adopted machine's account of a converged one. +func TestConvergingClearsTheAccountTheNodeGave(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + made, err := inv.AddNodeAs(ctx, "anchor", true) + if err != nil { + t.Fatal(err) + } + if err := inv.RecordAdoption(ctx, made.ID, + []Held{{ID: "notes.conf", Module: "notes", Kind: "file", Target: "/etc/notes.conf"}}, + "ufw", []Reach{{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}); err != nil { + t.Fatal(err) + } + if _, err := inv.Converge(ctx, "anchor"); err != nil { + t.Fatal(err) + } + said, err := inv.AdoptionOf(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + if len(said.Held) != 0 || len(said.Reachable) != 0 || said.Firewall != "" || !said.At.IsZero() { + t.Fatalf("converging kept the adopted machine's account: %+v", said) + } +} From 01814854b8e79f6bb6a339edf8102dcc8bb23d62 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:47:23 +0200 Subject: [PATCH 29/33] Refuse the flip on an account naming nothing reachable, and mark such an account partial in the preview (hq ADR 0100) --- cmd/mesh-controller/adopting_test.go | 29 +++++++++++++++++++ cmd/mesh-controller/adoption.go | 42 ++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index b10b202..ef81c2a 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -473,3 +473,32 @@ func TestThePreviewNamesEveryHeldKind(t *testing.T) { t.Fatal("the digest does not change with what is held") } } + +// novox/hq ADR 0100: the flip acts on what the node said is reachable, so an account naming nothing +// is refused. Every machine that is up answers on ssh; nothing reported means the host's collectors +// did not, and flipping would close ports the preview never named. +func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) { + open, sent := anAdoptedAnchor(t) + ctx := t.Context() + if _, err := take(ctx, open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + // Only a loopback listener: nothing off the machine, which is the same silence. + reportsReaching(t, open, []link.Reach{ + {Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"}, + }, heldFile) + preview, err := converge(ctx, open, "anchor", false, "", "") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(preview, "this account looks partial") { + t.Errorf("the preview does not mark a partial account:\n%s", preview) + } + _, err = converge(ctx, open, "anchor", true, digestIn(t, preview), "") + if err == nil || !strings.Contains(err.Error(), "says nothing is reachable on it") { + t.Fatalf("the flip was not refused on an account naming nothing: %v", err) + } + if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 { + t.Fatal("a refused flip changed something") + } +} diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index de93a6c..0f39abf 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -189,6 +189,10 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s "converge once it has applied", node, node) } + assignedWhenPreviewed, err := inv.Assigned(ctx, node) + if err != nil { + return "", err + } plan, settings, err := planFor(ctx, open, node) if err != nil { return "", err @@ -248,6 +252,16 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+ "it.", node, saw), nil } + // An account naming nothing reachable is not an account of a machine: every machine answers + // on ssh, and the host's collectors failing — `ss` refusing, or the container runtime not + // answering, which drops every published port at once — leaves exactly this. Flipping on it + // would close ports the preview never named. + if yes && countReachable(reported) == 0 { + return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+ + "up ever is: its account looks partial — whatever reads what is listening, or what "+ + "the container runtime publishes, did not answer. Fix that on the machine and run "+ + "`push %s --wait 2m`, then preview again", node, node) + } if age := time.Since(reported.At); age > reportFreshFor { return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+ "only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+ @@ -269,6 +283,13 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s if err != nil { return "", err } + // And nothing assigned since the preview was composed: the flip takes every module the node + // runs, and one assigned in between would be taken without ever having been previewed. + if !slices.Equal(assigned, assignedWhenPreviewed) { + return preview, fmt.Errorf("what %s runs changed while this was converging (it is now %s): "+ + "the flip takes every module on the node, so read the preview again", node, + strings.Join(assigned, ", ")) + } if !slices.Contains(assigned, filter) { if err := inv.Assign(ctx, node, filter); err != nil { return "", err @@ -317,8 +338,12 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter, fmt.Fprintf(&b, " %-44s %s\n", what, fate) said = append(said, fmt.Sprintf("reach %s %s %s", r.Address, what, fate)) } - if len(reported.Reachable) == 0 { - b.WriteString(" nothing reported\n") + if countReachable(reported) == 0 { + // Said as what it is: no machine that is up is reachable on nothing, so this is an + // account that did not come back, not a machine with nothing on it. + b.WriteString(" nothing reported — this account looks partial, and the flip is " + + "refused on it\n") + said = append(said, "reach nothing reported") } // What the machine routes for others is not a listener and not a published port, so nothing // above can show it; the derived filter's forward chain drops it all the same. @@ -431,6 +456,19 @@ func (d derivedFilter) fate(r inventory.Reach) string { return "WILL CLOSE — no module assigned here declares it" } +// countReachable is how much of a node's account of itself names something off the machine. +// Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it, +// so a report of loopback alone says nothing about what the filter would close. +func countReachable(reported inventory.Adoption) int { + n := 0 + for _, r := range reported.Reachable { + if !loopback(r.Address) { + n++ + } + } + return n +} + // heldLine is one thing a node holds as found, as take and the converge preview both say it. func heldLine(h inventory.Held) string { return fmt.Sprintf("%s %s (%s)", h.Kind, h.Target, h.ID) From d05a5e87af8d76347b703eac04faaa0e55b6ce56 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:47:23 +0200 Subject: [PATCH 30/33] Hold the node while an assignment is recorded, so none lands between a preview and the flip (hq ADR 0100) --- cmd/mesh-controller/acts.go | 12 +++++++++++ cmd/mesh-controller/adopting_test.go | 30 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/cmd/mesh-controller/acts.go b/cmd/mesh-controller/acts.go index 4a0abe8..c5bae72 100644 --- a/cmd/mesh-controller/acts.go +++ b/cmd/mesh-controller/acts.go @@ -39,6 +39,13 @@ import ( // It costs a resolution per machine. Assignment is a person typing a command, and being told which // machines this just blocked is worth more than the milliseconds. func assign(ctx context.Context, open *stores, node, module string) (string, error) { + // Held while it is recorded, so it cannot land between a converge's preview and its flip and + // be taken without ever having been previewed (novox/hq ADR 0100). + ctx, release, err := holdNodes(ctx, open, []string{node}) + if err != nil { + return "", err + } + defer release() if err := open.inventory.Assign(ctx, node, module); err != nil { return "", err } @@ -71,6 +78,11 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err // module off one machine is the ordinary way to stop providing something to another, and nothing // about the command's own output would ever have said so. func unassign(ctx context.Context, open *stores, node, module string) (string, error) { + ctx, release, err := holdNodes(ctx, open, []string{node}) + if err != nil { + return "", err + } + defer release() if err := open.inventory.Unassign(ctx, node, module); err != nil { return "", err } diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index ef81c2a..3eddda5 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -502,3 +502,33 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) { t.Fatal("a refused flip changed something") } } + +// An assignment cannot land between a preview and the flip that takes every module: assigning +// holds the node, so it waits for whatever is converging it. +func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) { + open, _ := anAdoptedAnchor(t) + ctx := t.Context() + if _, err := take(ctx, open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + reportsHolding(t, open, heldFile) + preview, err := converge(ctx, open, "anchor", false, "", "") + if err != nil { + t.Fatal(err) + } + saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll + inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond + defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }() + + var whileFlipping error + sendNodes = func(context.Context, *stores, []string) error { + _, whileFlipping = assign(ctx, open, "anchor", "notes") + return nil + } + if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil { + t.Fatal(err) + } + if !errors.Is(whileFlipping, inventory.ErrNodeBusy) { + t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping) + } +} From 379f45949871496299c02aa5ca02c1be78789ae1 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:47:43 +0200 Subject: [PATCH 31/33] Hold the filter module to reloading its rules and restarting only on its units (hq ADR 0102) --- internal/catalogue/foundation_manifests_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/catalogue/foundation_manifests_test.go b/internal/catalogue/foundation_manifests_test.go index f8f524e..44f95d8 100644 --- a/internal/catalogue/foundation_manifests_test.go +++ b/internal/catalogue/foundation_manifests_test.go @@ -70,8 +70,15 @@ func TestTheFilterModuleNeverFlushesTheRuleset(t *testing.T) { "[Service]\nExecStop=\nExecStop=nft delete table inet mesh\n") { t.Fatalf("the stock unit's stop is not replaced with deleting the mesh's table: %v", stock) } - if !reflect.DeepEqual(load["restart-on"], []any{"filtering", "unit", "stock-unit-stop"}) { - t.Fatalf("the filter is not reloaded when its rules, its unit or the stock unit's drop-in "+ - "change: %v", load["restart-on"]) + // A changed rule set is RELOADED — ExecReload replaces the table in one `nft -f`, so the node + // is never unfiltered — and only the units themselves restart it, which is the one change a + // reload cannot carry. + if !reflect.DeepEqual(load["reload-on"], []any{"filtering"}) { + t.Fatalf("the filter is restarted rather than reloaded when its rules change, leaving the "+ + "node unfiltered in between: %v", load) + } + if !reflect.DeepEqual(load["restart-on"], []any{"unit", "stock-unit-stop"}) { + t.Fatalf("the filter is not restarted when its unit or the stock unit's drop-in changes: %v", + load["restart-on"]) } } From 1096299e06eaf93c0a8c3ec81711a967c5cddbdc Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:51:10 +0200 Subject: [PATCH 32/33] Hold a converged declaration to the bytes main sends, captured from it, rather than to re-marshalling itself (hq ADR 0100) --- cmd/mesh-controller/sendable_test.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go index c434265..f41f618 100644 --- a/cmd/mesh-controller/sendable_test.go +++ b/cmd/mesh-controller/sendable_test.go @@ -11,6 +11,7 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/overlay" ) // novox/hq ADR 0100: every declaration an adopted node is sent says it is adopted and which modules @@ -41,13 +42,24 @@ func composed(t *testing.T, open *stores, node string) sendable { return declared } +// convergedBefore is the envelope a converged node was sent before adoption existed, captured by +// running this same composition at the commit this branch left main (0a39b7d). The mesh here is +// aMesh's laptop with the private network taken off it, so nothing in the declaration is random: +// what changes this string is a change to what a converged machine is sent, which is the thing an +// older host would refuse. +const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"hosts":["anchor.internal:10.77.0.1"],"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}` + func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) { open := aMesh(t) + ctx := t.Context() register(t, open, helloWeb()) - if _, err := assign(t.Context(), open, "anchor", "hello-web"); err != nil { + if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil { t.Fatal(err) } - declared := composed(t, open, "anchor") + if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil { + t.Fatal(err) + } + declared := composed(t, open, "laptop") if declared.Adoption != nil { t.Fatal("a converged node was given an adoption envelope") } @@ -55,13 +67,9 @@ func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) { if err != nil { t.Fatal(err) } - // The envelope exactly as every send site marshalled it before adoption existed. - before, err := json.Marshal(map[string]any{"declaration": 1, "resources": declared.Resources}) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(body, before) { - t.Fatalf("a converged declaration changed:\n%s\n%s", body, before) + if string(body) != convergedBefore { + t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s", + body, convergedBefore) } if bytes.Contains(body, []byte(`"adoption"`)) { t.Fatal("a converged declaration names adoption; an older host would refuse it") From fad8b30e4300c9fbbb48ca292881ab55d06fa715 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 19:51:10 +0200 Subject: [PATCH 33/33] Say that the guard names the runtime's bridges where the filter names their addresses (hq ADR 0103) --- internal/catalogue/adoption.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/catalogue/adoption.go b/internal/catalogue/adoption.go index c7957c0..6debcb5 100644 --- a/internal/catalogue/adoption.go +++ b/internal/catalogue/adoption.go @@ -159,6 +159,16 @@ func Published(resources []map[string]any) map[string]map[int]int { // prerouting, ahead of the runtime's destination translation, so it matches the port the packet // was sent to; in the inet family, so both address families. // +// **The machine's own interfaces are named, where the derived filter names address ranges.** The +// filter accepts the container runtime's networks by CIDR; this excludes its bridges by name — lo, +// docker0, the br-* a compose network gets, and the mesh's own mesh0. A runtime whose bridge is +// named anything else (a podman or libvirt bridge, or a docker network created with a fixed name) +// would have its containers' traffic to a guarded port refused, which reads as the port being +// down. Names rather than addresses is deliberate: a source address can be claimed by whoever +// sends the packet, and this table exists to refuse what the found firewall never sees. Widening +// it means adding names here and in the installer's copy together, which the golden test holds to +// one text. +// // The same text the installer raises on an adopted genesis; a test holds both to it. func AsGuard(ports []int) string { sorted := append([]int{}, ports...)