The move mints every credential and tells each machine its membership

`rollout mint` gives every principal the new bus will have a credential it does
not yet have and puts each where its owner reads it: a machine's as a membership
— bus address, fingerprint, password, transport — sealed into its declaration
(migration 0041, the `bus-membership` resource the host reads after applying); a
module's as its broker secret, through the same delivery `module issue` uses; the
control plane's own as its `bus` secret. Idempotent, and worked out from where the
bus's module is assigned rather than from this process's environment, because this
process is still on the old bus when it runs and must be.

This is the half of design 28 task 5.2 the first live attempt found missing: a
credential was minted only at enrolment, at `module issue` and for a person, so no
machine already enrolled could ever be moved. `rollout check` was right to refuse;
now there is something to run first.
This commit is contained in:
2026-09-28 00:16:24 +02:00
parent 337aaea123
commit e8aa7ed9e7
9 changed files with 301 additions and 11 deletions
+22 -8
View File
@@ -614,6 +614,15 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
if err != nil {
return err
}
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
}
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
// secret, and the module's consumer created where the bus can be reached. Split from the minting
// so the move can issue every module against a bus whose address it worked out itself
// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
node, busAddress string, known broker.Broker, reachable, user, password string) error {
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
@@ -639,14 +648,19 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
Kind: broker.KindModule, Node: node, Module: m.Module,
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
}); needed {
js, err := broker.Dial(busAddress)
if err != nil {
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
"how %s hears what it consumes: %w", m.Module, err)
}
defer js.Close()
if err := js.EnsureConsumer(consumer); err != nil {
return err
if busAddress == "" {
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
"`rollout mint` again is harmless)\n", m.Module)
} else {
js, err := broker.Dial(busAddress)
if err != nil {
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
"how %s hears what it consumes: %w", m.Module, err)
}
defer js.Close()
if err := js.EnsureConsumer(consumer); err != nil {
return err
}
}
}
+6 -1
View File
@@ -636,8 +636,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
memberships, err := inv.BusMemberships(ctx)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
return catalogue.Rendering{
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines,
Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,
+163 -1
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
@@ -12,6 +13,7 @@ import (
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/secrets"
)
// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
@@ -31,12 +33,14 @@ import (
// ability to change things, not the services its modules are serving — measured on 2026-09-27, when
// a seat emptied mid-change and the control plane looped for two hours while every service stayed up.
const rolloutUsage = "rollout check | rollout --confirm"
const rolloutUsage = "rollout check | rollout mint | rollout --confirm"
func rolloutCommand(ctx context.Context, args []string) error {
switch {
case len(args) == 1 && args[0] == "check":
return rolloutCheck(ctx)
case len(args) == 1 && args[0] == "mint":
return rolloutMint(ctx)
case len(args) == 1 && args[0] == "--confirm":
return errors.New(
"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
@@ -190,3 +194,161 @@ func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node stri
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
// printing. The reasoning itself is the broker package's, where it is pure.
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
// sealed into its declaration, a module's as its broker secret, the control plane's own as its
// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
//
// **Before anything moves, and it is what makes moving possible.** A machine moved without a
// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
// the module that provides it is assigned, rather than read from this process's environment: this
// process is still on the old bus when this runs, and must be.
func rolloutMint(ctx context.Context) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
"must carry its fingerprint: %w", err)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
var busNode, controllerNode string
for _, e := range entries {
switch {
case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
busNode = e.On[0]
case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
controllerNode = e.On[0]
}
}
if busNode == "" {
return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
"bus to mint credentials for — register and assign it first")
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return err
}
if onNetwork[busNode] == "" {
return fmt.Errorf("%s runs the new bus and has no address on the private network, so no machine "+
"could be told where it is", busNode)
}
busAddress := onNetwork[busNode] + ":4222"
records, err := inv.BusRecords(ctx)
if err != nil {
return err
}
users, err := broker.Users(records)
if err != nil {
return err
}
kept, err := inv.BusUsers(ctx)
if err != nil {
return err
}
hashes := make(map[string]string, len(kept))
for name, u := range kept {
hashes[name] = u.PasswordHash
}
_, missing := broker.WithPasswords(users, hashes)
wanted := map[string]bool{}
for _, m := range missing {
wanted[m] = true
}
var machines, modules, skipped int
for _, p := range users {
if !wanted[p.Username()] {
continue
}
switch p.Kind {
case broker.KindController:
if controllerNode == "" {
return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
if err != nil {
return err
}
url := "nats://" + p.Username() + ":" + password + "@" + busAddress
if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
}
fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)
case broker.KindNode:
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
if err != nil {
return err
}
membership, _ := json.Marshal(map[string]string{
"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
})
key, err := inv.SealingKeyOf(ctx, p.Node)
if err != nil {
return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
}
sealed, err := secrets.Seal(key, membership)
if err != nil {
return err
}
if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
return err
}
machines++
case broker.KindModule:
m, inShelf := shelf[p.Module]
if !inShelf {
skipped++
continue
}
if _, reads := m.OwnSecrets["broker"]; !reads {
fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
skipped++
continue
}
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
if err != nil {
return err
}
if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
return err
}
modules++
default:
skipped++
}
}
fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
machines, modules, skipped, busAddress)
fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;")
fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it")
return nil
}
// providesBus is whether a manifest provides the mesh's bus.
func providesBus(m catalogue.Manifest) bool {
for _, o := range m.Provides {
if o.Name == "mesh-bus" {
return true
}
}
return false
}