Compare commits

...
Author SHA1 Message Date
jschoubben e8aa7ed9e7 The move mints every credential and tells each machine its membership
`rollout mint` gives every principal the new bus will have a credential it does
not yet have and puts each where its owner reads it: a machine's as a membership
— bus address, fingerprint, password, transport — sealed into its declaration
(migration 0041, the `bus-membership` resource the host reads after applying); a
module's as its broker secret, through the same delivery `module issue` uses; the
control plane's own as its `bus` secret. Idempotent, and worked out from where the
bus's module is assigned rather than from this process's environment, because this
process is still on the old bus when it runs and must be.

This is the half of design 28 task 5.2 the first live attempt found missing: a
credential was minted only at enrolment, at `module issue` and for a person, so no
machine already enrolled could ever be moved. `rollout check` was right to refuse;
now there is something to run first.
2026-09-28 00:16:24 +02:00
jschoubben 337aaea123 Merge pull request 'The first handover records the standing holder without re-judging it' (#93) from fix/record-the-standing-holder into main 2026-09-27 21:35:08 +00:00
jschoubben 4c41628b20 The first handover records the standing holder without re-judging it
On a mesh that predates the record, every handover has to begin by writing down who
already holds the seat — otherwise the next holder cannot be assigned beside it,
because two eligible claimants with nothing on record are refused. Found on the
live mesh minutes after 0040 moved the bus seat's row: the standing holder no
longer satisfies what the seat delivers, on purpose, and so could not be recorded,
and so nothing could stand beside it.

Recording who already holds is not making a new holder. Derivation never read
what the seat delivers, so the standing holder holds regardless; when nothing is
on record and the named assignment claims the seat at its scope, only that claim
is checked. Every change of holder is still judged in full.
2026-09-27 23:34:40 +02:00
jschoubben ae7fb520d7 Merge pull request 'AMQP is not a provision: the bus seat delivers the bus, and the word is refused' (#92) from feat/amqp-is-not-a-provision into main 2026-09-27 21:30:09 +00:00
10 changed files with 328 additions and 19 deletions
+14
View File
@@ -614,6 +614,15 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
if err != nil {
return err
}
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
}
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
// secret, and the module's consumer created where the bus can be reached. Split from the minting
// so the move can issue every module against a bus whose address it worked out itself
// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
node, busAddress string, known broker.Broker, reachable, user, password string) error {
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
@@ -639,6 +648,10 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
Kind: broker.KindModule, Node: node, Module: m.Module,
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
}); needed {
if busAddress == "" {
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
"`rollout mint` again is harmless)\n", m.Module)
} else {
js, err := broker.Dial(busAddress)
if err != nil {
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
@@ -649,6 +662,7 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
return err
}
}
}
fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module)
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n", node, node)
+5
View File
@@ -636,7 +636,12 @@ func renderingFor(ctx context.Context, open *stores, node string,
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
memberships, err := inv.BusMemberships(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
return catalogue.Rendering{
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines,
+163 -1
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
@@ -12,6 +13,7 @@ import (
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/secrets"
)
// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
@@ -31,12 +33,14 @@ import (
// ability to change things, not the services its modules are serving — measured on 2026-09-27, when
// a seat emptied mid-change and the control plane looped for two hours while every service stayed up.
const rolloutUsage = "rollout check | rollout --confirm"
const rolloutUsage = "rollout check | rollout mint | rollout --confirm"
func rolloutCommand(ctx context.Context, args []string) error {
switch {
case len(args) == 1 && args[0] == "check":
return rolloutCheck(ctx)
case len(args) == 1 && args[0] == "mint":
return rolloutMint(ctx)
case len(args) == 1 && args[0] == "--confirm":
return errors.New(
"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
@@ -190,3 +194,161 @@ func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node stri
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
// printing. The reasoning itself is the broker package's, where it is pure.
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
// sealed into its declaration, a module's as its broker secret, the control plane's own as its
// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
//
// **Before anything moves, and it is what makes moving possible.** A machine moved without a
// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
// the module that provides it is assigned, rather than read from this process's environment: this
// process is still on the old bus when this runs, and must be.
func rolloutMint(ctx context.Context) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
"must carry its fingerprint: %w", err)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
var busNode, controllerNode string
for _, e := range entries {
switch {
case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
busNode = e.On[0]
case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
controllerNode = e.On[0]
}
}
if busNode == "" {
return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
"bus to mint credentials for — register and assign it first")
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return err
}
if onNetwork[busNode] == "" {
return fmt.Errorf("%s runs the new bus and has no address on the private network, so no machine "+
"could be told where it is", busNode)
}
busAddress := onNetwork[busNode] + ":4222"
records, err := inv.BusRecords(ctx)
if err != nil {
return err
}
users, err := broker.Users(records)
if err != nil {
return err
}
kept, err := inv.BusUsers(ctx)
if err != nil {
return err
}
hashes := make(map[string]string, len(kept))
for name, u := range kept {
hashes[name] = u.PasswordHash
}
_, missing := broker.WithPasswords(users, hashes)
wanted := map[string]bool{}
for _, m := range missing {
wanted[m] = true
}
var machines, modules, skipped int
for _, p := range users {
if !wanted[p.Username()] {
continue
}
switch p.Kind {
case broker.KindController:
if controllerNode == "" {
return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
if err != nil {
return err
}
url := "nats://" + p.Username() + ":" + password + "@" + busAddress
if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
}
fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)
case broker.KindNode:
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
if err != nil {
return err
}
membership, _ := json.Marshal(map[string]string{
"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
})
key, err := inv.SealingKeyOf(ctx, p.Node)
if err != nil {
return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
}
sealed, err := secrets.Seal(key, membership)
if err != nil {
return err
}
if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
return err
}
machines++
case broker.KindModule:
m, inShelf := shelf[p.Module]
if !inShelf {
skipped++
continue
}
if _, reads := m.OwnSecrets["broker"]; !reads {
fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
skipped++
continue
}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
if err != nil {
return err
}
if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
return err
}
modules++
default:
skipped++
}
}
fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
machines, modules, skipped, busAddress)
fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;")
fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it")
return nil
}
// providesBus is whether a manifest provides the mesh's bus.
func providesBus(m catalogue.Manifest) bool {
for _, o := range m.Provides {
if o.Name == "mesh-bus" {
return true
}
}
return false
}
+24 -5
View File
@@ -156,17 +156,36 @@ func handOver(ctx context.Context, seatName, to string) error {
if m == nil {
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
}
if err := catalogue.CanHold(*m, seat); err != nil {
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
}
var was string
if holdings, err := inv.Holdings(ctx); err == nil {
holdings, err := inv.Holdings(ctx)
if err != nil {
return err
}
for _, h := range holdings {
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
was = h.Node
}
}
// **Recording who already holds the seat is not making a new holder, and is not judged like
// one.** On a mesh that predates the record, the first handover has to begin by writing down
// the standing holder — otherwise the next holder cannot be assigned beside it, because two
// eligible claimants with nothing on record are refused. That standing holder may no longer
// satisfy what the seat delivers (the row moved under it, on purpose, as ADR 0131's first step),
// and it holds regardless: derivation never read that column. So when nothing is on record and
// the named assignment is the one holding by derivation, only the claim itself is checked here.
// Every *change* of holder is judged in full.
claimsIt := false
for _, c := range m.Claims {
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
claimsIt = true
}
}
if was == "" && claimsIt {
fmt.Printf("nothing was on record for %s; recording %s on %s as its standing holder\n",
seat.Name, module, nodeName)
} else if err := catalogue.CanHold(*m, seat); err != nil {
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
}
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
return err
+19
View File
@@ -103,6 +103,11 @@ type Rendering struct {
// **Only the users, never the server's own settings**: those are the module's, in its image and
// its mounts (Manifest.BusUsers).
BusUsers string
// BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it
// (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads
// after the declaration has applied, so the bus it names is standing before the machine leaves
// the one it is on.
BusMembership string
// MeshRange is the private network's CIDR (the range node addresses are allocated from), for a
// module that must name the whole mesh rather than one machine — an intrusion filter that must
@@ -238,9 +243,23 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
if err != nil {
return Composed{}, err
}
if with.BusMembership != "" {
// The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a
// secret and placed where the host looks for exactly this (design 28, task 5.2).
resources = append(resources, map[string]any{
"id": BusMembershipID(), "type": "file", "path": BusMembershipPath,
"sealed": with.BusMembership, "mode": "0600",
})
}
return Composed{Resources: resources, Owner: owner}, nil
}
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
// BusMembershipPath is where the host reads it — the same constant on both sides.
func BusMembershipID() string { return "bus-membership" }
const BusMembershipPath = "/var/lib/mesh/membership-next.json"
func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) {
// Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings,
// credentials and contributions land are resolved against this node's directories, so every
+29
View File
@@ -114,3 +114,32 @@ func TestCanHoldJudgesClaimScopeAndWhatTheSeatDelivers(t *testing.T) {
t.Fatalf("with the row saying amqp, an amqp provider was refused: %v", err)
}
}
// A machine being moved is handed its membership for the new bus as a sealed file in its own
// declaration — the machine's, not any module's (design 28, task 5.2).
func TestAMembershipForTheNewBusIsComposedAsASealedFile(t *testing.T) {
r := Resolution{Node: "anchor"}
got, err := r.Compose(Rendering{BusMembership: "sealed-blob"})
if err != nil {
t.Fatal(err)
}
var found map[string]any
for _, res := range got.Resources {
if res["id"] == BusMembershipID() {
found = res
}
}
if found == nil {
t.Fatalf("no membership resource in %v", got.Resources)
}
if found["path"] != BusMembershipPath || found["sealed"] != "sealed-blob" || found["mode"] != "0600" {
t.Fatalf("the membership is not a sealed 0600 file where the host reads it: %v", found)
}
// And a machine not being moved is handed nothing.
got, _ = r.Compose(Rendering{})
for _, res := range got.Resources {
if res["id"] == BusMembershipID() {
t.Fatal("a machine with no membership on record was handed one")
}
}
}
+32
View File
@@ -230,3 +230,35 @@ func (i *Inventory) ForgetPerson(ctx context.Context, name string) error {
}
return i.ForgetBusUser(ctx, "person."+name)
}
// PutBusMembership records a machine's membership for the new bus, sealed to it (design 28, 5.2).
// Replaces any earlier one: a machine has one membership per bus, and re-minting is re-telling.
func (i *Inventory) PutBusMembership(ctx context.Context, nodeName, sealed string) error {
node, err := i.NodeByName(ctx, nodeName)
if err != nil {
return err
}
_, err = i.store.Pool().Exec(ctx,
`insert into bus_membership (node, sealed) values ($1, $2)
on conflict (node) do update set sealed = excluded.sealed, since = now()`, node.ID, sealed)
return err
}
// BusMemberships is every machine's sealed membership for the new bus, by node name.
func (i *Inventory) BusMemberships(ctx context.Context) (map[string]string, error) {
rows, err := i.store.Pool().Query(ctx,
`select n.name, b.sealed from bus_membership b join node n on n.id = b.node`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]string{}
for rows.Next() {
var name, sealed string
if err := rows.Scan(&name, &sealed); err != nil {
return nil, err
}
out[name] = sealed
}
return out, rows.Err()
}
+17
View File
@@ -80,3 +80,20 @@ func TestUnassigningTheHolderTakesTheHoldingWithIt(t *testing.T) {
t.Fatalf("the holding outlived the assignment it pointed at: %+v", held)
}
}
func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T) {
inv, ctx := twoBrokersOnTwoNodes(t)
if err := inv.PutBusMembership(ctx, "anchor", "first"); err != nil {
t.Fatal(err)
}
if err := inv.PutBusMembership(ctx, "anchor", "second"); err != nil {
t.Fatal(err)
}
got, err := inv.BusMemberships(ctx)
if err != nil {
t.Fatal(err)
}
if got["anchor"] != "second" || len(got) != 1 {
t.Fatalf("a re-told membership did not replace the first: %v", got)
}
}
@@ -0,0 +1,11 @@
-- A machine already enrolled is moved to the new bus by being told its membership for it
-- (novox/hq design 28, task 5.2). Until this, a membership — bus address, fingerprint, password,
-- transport — existed only in the enrolment reply, and nothing could hand one to a machine that
-- had already joined. The row is the membership sealed to that machine, composed into its
-- declaration as a file it reads after applying; the plaintext exists once, at minting, and then
-- only on the machine. One per node: the mesh moves to one bus.
create table bus_membership (
node uuid primary key references node(id) on delete cascade,
sealed text not null,
since timestamptz not null default now()
);
+2 -1
View File
@@ -23,7 +23,8 @@
"licences": "/var/lib/mesh/mesh-controller/licences",
"broker": "/var/lib/mesh/mesh-controller/broker",
"broker-management": "/var/lib/mesh/mesh-controller/broker-management",
"broker-address": "/var/lib/mesh/mesh-controller/broker-address"
"broker-address": "/var/lib/mesh/mesh-controller/broker-address",
"bus": "/var/lib/mesh/mesh-controller/bus"
},
"secrets-owner": "65534:65534",
"resources": [