Compare commits

..
Author SHA1 Message Date
jschoubben e8aa7ed9e7 The move mints every credential and tells each machine its membership
`rollout mint` gives every principal the new bus will have a credential it does
not yet have and puts each where its owner reads it: a machine's as a membership
— bus address, fingerprint, password, transport — sealed into its declaration
(migration 0041, the `bus-membership` resource the host reads after applying); a
module's as its broker secret, through the same delivery `module issue` uses; the
control plane's own as its `bus` secret. Idempotent, and worked out from where the
bus's module is assigned rather than from this process's environment, because this
process is still on the old bus when it runs and must be.

This is the half of design 28 task 5.2 the first live attempt found missing: a
credential was minted only at enrolment, at `module issue` and for a person, so no
machine already enrolled could ever be moved. `rollout check` was right to refuse;
now there is something to run first.
2026-09-28 00:16:24 +02:00
11 changed files with 301 additions and 13 deletions
-2
View File
@@ -164,8 +164,6 @@ func usage() {
upgrade <name> record ...record that they are behind, and send nothing
status [--json] what is wrong, what is quiet, and what is out of date
seats [--json] every seat this mesh defines, what it delivers, and who holds it
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
board [--listen ADDR] the same three questions, as a page that holds nothing
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
assign <node> <module> put a module on a node
+22 -8
View File
@@ -614,6 +614,15 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
if err != nil {
return err
}
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
}
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
// secret, and the module's consumer created where the bus can be reached. Split from the minting
// so the move can issue every module against a bus whose address it worked out itself
// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
node, busAddress string, known broker.Broker, reachable, user, password string) error {
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
@@ -639,14 +648,19 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
Kind: broker.KindModule, Node: node, Module: m.Module,
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
}); needed {
js, err := broker.Dial(busAddress)
if err != nil {
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
"how %s hears what it consumes: %w", m.Module, err)
}
defer js.Close()
if err := js.EnsureConsumer(consumer); err != nil {
return err
if busAddress == "" {
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
"`rollout mint` again is harmless)\n", m.Module)
} else {
js, err := broker.Dial(busAddress)
if err != nil {
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
"how %s hears what it consumes: %w", m.Module, err)
}
defer js.Close()
if err := js.EnsureConsumer(consumer); err != nil {
return err
}
}
}
+6 -1
View File
@@ -636,8 +636,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
memberships, err := inv.BusMemberships(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
return catalogue.Rendering{
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines,
Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,
+163 -1
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
@@ -12,6 +13,7 @@ import (
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/secrets"
)
// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
@@ -31,12 +33,14 @@ import (
// ability to change things, not the services its modules are serving — measured on 2026-09-27, when
// a seat emptied mid-change and the control plane looped for two hours while every service stayed up.
const rolloutUsage = "rollout check | rollout --confirm"
const rolloutUsage = "rollout check | rollout mint | rollout --confirm"
func rolloutCommand(ctx context.Context, args []string) error {
switch {
case len(args) == 1 && args[0] == "check":
return rolloutCheck(ctx)
case len(args) == 1 && args[0] == "mint":
return rolloutMint(ctx)
case len(args) == 1 && args[0] == "--confirm":
return errors.New(
"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
@@ -190,3 +194,161 @@ func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node stri
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
// printing. The reasoning itself is the broker package's, where it is pure.
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
// sealed into its declaration, a module's as its broker secret, the control plane's own as its
// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
//
// **Before anything moves, and it is what makes moving possible.** A machine moved without a
// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
// the module that provides it is assigned, rather than read from this process's environment: this
// process is still on the old bus when this runs, and must be.
func rolloutMint(ctx context.Context) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
"must carry its fingerprint: %w", err)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
var busNode, controllerNode string
for _, e := range entries {
switch {
case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
busNode = e.On[0]
case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
controllerNode = e.On[0]
}
}
if busNode == "" {
return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
"bus to mint credentials for — register and assign it first")
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return err
}
if onNetwork[busNode] == "" {
return fmt.Errorf("%s runs the new bus and has no address on the private network, so no machine "+
"could be told where it is", busNode)
}
busAddress := onNetwork[busNode] + ":4222"
records, err := inv.BusRecords(ctx)
if err != nil {
return err
}
users, err := broker.Users(records)
if err != nil {
return err
}
kept, err := inv.BusUsers(ctx)
if err != nil {
return err
}
hashes := make(map[string]string, len(kept))
for name, u := range kept {
hashes[name] = u.PasswordHash
}
_, missing := broker.WithPasswords(users, hashes)
wanted := map[string]bool{}
for _, m := range missing {
wanted[m] = true
}
var machines, modules, skipped int
for _, p := range users {
if !wanted[p.Username()] {
continue
}
switch p.Kind {
case broker.KindController:
if controllerNode == "" {
return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
if err != nil {
return err
}
url := "nats://" + p.Username() + ":" + password + "@" + busAddress
if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
}
fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)
case broker.KindNode:
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
if err != nil {
return err
}
membership, _ := json.Marshal(map[string]string{
"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
})
key, err := inv.SealingKeyOf(ctx, p.Node)
if err != nil {
return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
}
sealed, err := secrets.Seal(key, membership)
if err != nil {
return err
}
if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
return err
}
machines++
case broker.KindModule:
m, inShelf := shelf[p.Module]
if !inShelf {
skipped++
continue
}
if _, reads := m.OwnSecrets["broker"]; !reads {
fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
skipped++
continue
}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
if err != nil {
return err
}
if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
return err
}
modules++
default:
skipped++
}
}
fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
machines, modules, skipped, busAddress)
fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;")
fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it")
return nil
}
// providesBus is whether a manifest provides the mesh's bus.
func providesBus(m catalogue.Manifest) bool {
for _, o := range m.Provides {
if o.Name == "mesh-bus" {
return true
}
}
return false
}
+19
View File
@@ -103,6 +103,11 @@ type Rendering struct {
// **Only the users, never the server's own settings**: those are the module's, in its image and
// its mounts (Manifest.BusUsers).
BusUsers string
// BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it
// (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads
// after the declaration has applied, so the bus it names is standing before the machine leaves
// the one it is on.
BusMembership string
// MeshRange is the private network's CIDR (the range node addresses are allocated from), for a
// module that must name the whole mesh rather than one machine — an intrusion filter that must
@@ -238,9 +243,23 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
if err != nil {
return Composed{}, err
}
if with.BusMembership != "" {
// The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a
// secret and placed where the host looks for exactly this (design 28, task 5.2).
resources = append(resources, map[string]any{
"id": BusMembershipID(), "type": "file", "path": BusMembershipPath,
"sealed": with.BusMembership, "mode": "0600",
})
}
return Composed{Resources: resources, Owner: owner}, nil
}
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
// BusMembershipPath is where the host reads it — the same constant on both sides.
func BusMembershipID() string { return "bus-membership" }
const BusMembershipPath = "/var/lib/mesh/membership-next.json"
func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) {
// Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings,
// credentials and contributions land are resolved against this node's directories, so every
+29
View File
@@ -114,3 +114,32 @@ func TestCanHoldJudgesClaimScopeAndWhatTheSeatDelivers(t *testing.T) {
t.Fatalf("with the row saying amqp, an amqp provider was refused: %v", err)
}
}
// A machine being moved is handed its membership for the new bus as a sealed file in its own
// declaration — the machine's, not any module's (design 28, task 5.2).
func TestAMembershipForTheNewBusIsComposedAsASealedFile(t *testing.T) {
r := Resolution{Node: "anchor"}
got, err := r.Compose(Rendering{BusMembership: "sealed-blob"})
if err != nil {
t.Fatal(err)
}
var found map[string]any
for _, res := range got.Resources {
if res["id"] == BusMembershipID() {
found = res
}
}
if found == nil {
t.Fatalf("no membership resource in %v", got.Resources)
}
if found["path"] != BusMembershipPath || found["sealed"] != "sealed-blob" || found["mode"] != "0600" {
t.Fatalf("the membership is not a sealed 0600 file where the host reads it: %v", found)
}
// And a machine not being moved is handed nothing.
got, _ = r.Compose(Rendering{})
for _, res := range got.Resources {
if res["id"] == BusMembershipID() {
t.Fatal("a machine with no membership on record was handed one")
}
}
}
+32
View File
@@ -230,3 +230,35 @@ func (i *Inventory) ForgetPerson(ctx context.Context, name string) error {
}
return i.ForgetBusUser(ctx, "person."+name)
}
// PutBusMembership records a machine's membership for the new bus, sealed to it (design 28, 5.2).
// Replaces any earlier one: a machine has one membership per bus, and re-minting is re-telling.
func (i *Inventory) PutBusMembership(ctx context.Context, nodeName, sealed string) error {
node, err := i.NodeByName(ctx, nodeName)
if err != nil {
return err
}
_, err = i.store.Pool().Exec(ctx,
`insert into bus_membership (node, sealed) values ($1, $2)
on conflict (node) do update set sealed = excluded.sealed, since = now()`, node.ID, sealed)
return err
}
// BusMemberships is every machine's sealed membership for the new bus, by node name.
func (i *Inventory) BusMemberships(ctx context.Context) (map[string]string, error) {
rows, err := i.store.Pool().Query(ctx,
`select n.name, b.sealed from bus_membership b join node n on n.id = b.node`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]string{}
for rows.Next() {
var name, sealed string
if err := rows.Scan(&name, &sealed); err != nil {
return nil, err
}
out[name] = sealed
}
return out, rows.Err()
}
+17
View File
@@ -80,3 +80,20 @@ func TestUnassigningTheHolderTakesTheHoldingWithIt(t *testing.T) {
t.Fatalf("the holding outlived the assignment it pointed at: %+v", held)
}
}
func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T) {
inv, ctx := twoBrokersOnTwoNodes(t)
if err := inv.PutBusMembership(ctx, "anchor", "first"); err != nil {
t.Fatal(err)
}
if err := inv.PutBusMembership(ctx, "anchor", "second"); err != nil {
t.Fatal(err)
}
got, err := inv.BusMemberships(ctx)
if err != nil {
t.Fatal(err)
}
if got["anchor"] != "second" || len(got) != 1 {
t.Fatalf("a re-told membership did not replace the first: %v", got)
}
}
@@ -0,0 +1,11 @@
-- A machine already enrolled is moved to the new bus by being told its membership for it
-- (novox/hq design 28, task 5.2). Until this, a membership — bus address, fingerprint, password,
-- transport — existed only in the enrolment reply, and nothing could hand one to a machine that
-- had already joined. The row is the membership sealed to that machine, composed into its
-- declaration as a file it reads after applying; the plaintext exists once, at minting, and then
-- only on the machine. One per node: the mesh moves to one bus.
create table bus_membership (
node uuid primary key references node(id) on delete cascade,
sealed text not null,
since timestamptz not null default now()
);
BIN
View File
Binary file not shown.
+2 -1
View File
@@ -23,7 +23,8 @@
"licences": "/var/lib/mesh/mesh-controller/licences",
"broker": "/var/lib/mesh/mesh-controller/broker",
"broker-management": "/var/lib/mesh/mesh-controller/broker-management",
"broker-address": "/var/lib/mesh/mesh-controller/broker-address"
"broker-address": "/var/lib/mesh/mesh-controller/broker-address",
"bus": "/var/lib/mesh/mesh-controller/bus"
},
"secrets-owner": "65534:65534",
"resources": [