Merge pull request 'The move mints every credential and tells each machine its membership' (#95) from feat/the-move-mints-and-delivers into main

cmd/mesh-controller/modules.go

@@ -614,6 +614,15 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
 	if err != nil {
 		return err
 	}
+	return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
+}
+
+// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
+// secret, and the module's consumer created where the bus can be reached. Split from the minting
+// so the move can issue every module against a bus whose address it worked out itself
+// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
+func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
+	node, busAddress string, known broker.Broker, reachable, user, password string) error {
 	held, err := json.Marshal(struct {
 		URL         string `json:"url"`
 		Fingerprint string `json:"fingerprint,omitempty"`
@@ -639,14 +648,19 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
 		Kind: broker.KindModule, Node: node, Module: m.Module,
 		Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
 	}); needed {
-		js, err := broker.Dial(busAddress)
+		if busAddress == "" {
-		if err != nil {
+			fmt.Printf("  %s consumes; its consumer is created when the bus is reachable (`push`, then "+
-			return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
+				"`rollout mint` again is harmless)\n", m.Module)
-				"how %s hears what it consumes: %w", m.Module, err)
+		} else {
-		}
+			js, err := broker.Dial(busAddress)
-		defer js.Close()
+			if err != nil {
-		if err := js.EnsureConsumer(consumer); err != nil {
+				return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
-			return err
+					"how %s hears what it consumes: %w", m.Module, err)
+			}
+			defer js.Close()
+			if err := js.EnsureConsumer(consumer); err != nil {
+				return err
+			}
 		}
 	}
 

cmd/mesh-controller/plan.go

@@ -636,8 +636,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
 	if err != nil {
 		return catalogue.Rendering{}, inventory.Node{}, err
 	}
+	memberships, err := inv.BusMemberships(ctx)
+	if err != nil {
+		return catalogue.Rendering{}, inventory.Node{}, err
+	}
 	return catalogue.Rendering{
-		Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
+		BusMembership: memberships[node],
+		Settings:      settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
 		Machines: machines,
 		Suffix:   overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,

cmd/mesh-controller/rollout.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
@@ -12,6 +13,7 @@ import (
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/secrets"
 )
 
 // Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
@@ -31,12 +33,14 @@ import (
 // ability to change things, not the services its modules are serving — measured on 2026-09-27, when
 // a seat emptied mid-change and the control plane looped for two hours while every service stayed up.
 
-const rolloutUsage = "rollout check | rollout --confirm"
+const rolloutUsage = "rollout check | rollout mint | rollout --confirm"
 
 func rolloutCommand(ctx context.Context, args []string) error {
 	switch {
 	case len(args) == 1 && args[0] == "check":
 		return rolloutCheck(ctx)
+	case len(args) == 1 && args[0] == "mint":
+		return rolloutMint(ctx)
 	case len(args) == 1 && args[0] == "--confirm":
 		return errors.New(
 			"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
@@ -190,3 +194,161 @@ func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node stri
 // notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
 // printing. The reasoning itself is the broker package's, where it is pure.
 func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
+
+// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
+// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
+// sealed into its declaration, a module's as its broker secret, the control plane's own as its
+// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
+//
+// **Before anything moves, and it is what makes moving possible.** A machine moved without a
+// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
+// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
+// the module that provides it is assigned, rather than read from this process's environment: this
+// process is still on the old bus when this runs, and must be.
+func rolloutMint(ctx context.Context) error {
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+
+	known, err := broker.FromEnvironment()
+	if err != nil {
+		return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
+			"must carry its fingerprint: %w", err)
+	}
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return err
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return err
+	}
+	var busNode, controllerNode string
+	for _, e := range entries {
+		switch {
+		case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
+			busNode = e.On[0]
+		case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
+			controllerNode = e.On[0]
+		}
+	}
+	if busNode == "" {
+		return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
+			"bus to mint credentials for — register and assign it first")
+	}
+	onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
+	if err != nil {
+		return err
+	}
+	if onNetwork[busNode] == "" {
+		return fmt.Errorf("%s runs the new bus and has no address on the private network, so no machine "+
+			"could be told where it is", busNode)
+	}
+	busAddress := onNetwork[busNode] + ":4222"
+
+	records, err := inv.BusRecords(ctx)
+	if err != nil {
+		return err
+	}
+	users, err := broker.Users(records)
+	if err != nil {
+		return err
+	}
+	kept, err := inv.BusUsers(ctx)
+	if err != nil {
+		return err
+	}
+	hashes := make(map[string]string, len(kept))
+	for name, u := range kept {
+		hashes[name] = u.PasswordHash
+	}
+	_, missing := broker.WithPasswords(users, hashes)
+	wanted := map[string]bool{}
+	for _, m := range missing {
+		wanted[m] = true
+	}
+
+	var machines, modules, skipped int
+	for _, p := range users {
+		if !wanted[p.Username()] {
+			continue
+		}
+		switch p.Kind {
+		case broker.KindController:
+			if controllerNode == "" {
+				return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
+			}
+			password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
+			if err != nil {
+				return err
+			}
+			url := "nats://" + p.Username() + ":" + password + "@" + busAddress
+			if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
+				return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
+			}
+			fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)
+
+		case broker.KindNode:
+			password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
+			if err != nil {
+				return err
+			}
+			membership, _ := json.Marshal(map[string]string{
+				"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
+			})
+			key, err := inv.SealingKeyOf(ctx, p.Node)
+			if err != nil {
+				return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
+			}
+			sealed, err := secrets.Seal(key, membership)
+			if err != nil {
+				return err
+			}
+			if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
+				return err
+			}
+			machines++
+
+		case broker.KindModule:
+			m, inShelf := shelf[p.Module]
+			if !inShelf {
+				skipped++
+				continue
+			}
+			if _, reads := m.OwnSecrets["broker"]; !reads {
+				fmt.Printf("  %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
+				skipped++
+				continue
+			}
+			password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
+			if err != nil {
+				return err
+			}
+			if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
+				return err
+			}
+			modules++
+
+		default:
+			skipped++
+		}
+	}
+	fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
+		machines, modules, skipped, busAddress)
+	fmt.Println("  each machine's membership and each module's credential arrive with the next push of its machine;")
+	fmt.Println("  push the machine running the bus first, so the bus stands with its user list before anything dials it")
+	return nil
+}
+
+// providesBus is whether a manifest provides the mesh's bus.
+func providesBus(m catalogue.Manifest) bool {
+	for _, o := range m.Provides {
+		if o.Name == "mesh-bus" {
+			return true
+		}
+	}
+	return false
+}

internal/catalogue/declaration.go

@@ -103,6 +103,11 @@ type Rendering struct {
 	// **Only the users, never the server's own settings**: those are the module's, in its image and
 	// its mounts (Manifest.BusUsers).
 	BusUsers string
+	// BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it
+	// (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads
+	// after the declaration has applied, so the bus it names is standing before the machine leaves
+	// the one it is on.
+	BusMembership string
 
 	// MeshRange is the private network's CIDR (the range node addresses are allocated from), for a
 	// module that must name the whole mesh rather than one machine — an intrusion filter that must
@@ -238,9 +243,23 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
 	if err != nil {
 		return Composed{}, err
 	}
+	if with.BusMembership != "" {
+		// The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a
+		// secret and placed where the host looks for exactly this (design 28, task 5.2).
+		resources = append(resources, map[string]any{
+			"id": BusMembershipID(), "type": "file", "path": BusMembershipPath,
+			"sealed": with.BusMembership, "mode": "0600",
+		})
+	}
 	return Composed{Resources: resources, Owner: owner}, nil
 }
 
+// BusMembershipID names the resource carrying a machine's membership for the new bus, and
+// BusMembershipPath is where the host reads it — the same constant on both sides.
+func BusMembershipID() string { return "bus-membership" }
+
+const BusMembershipPath = "/var/lib/mesh/membership-next.json"
+
 func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) {
 	// Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings,
 	// credentials and contributions land are resolved against this node's directories, so every

internal/catalogue/holdings_test.go

@@ -114,3 +114,32 @@ func TestCanHoldJudgesClaimScopeAndWhatTheSeatDelivers(t *testing.T) {
 		t.Fatalf("with the row saying amqp, an amqp provider was refused: %v", err)
 	}
 }
+
+// A machine being moved is handed its membership for the new bus as a sealed file in its own
+// declaration — the machine's, not any module's (design 28, task 5.2).
+func TestAMembershipForTheNewBusIsComposedAsASealedFile(t *testing.T) {
+	r := Resolution{Node: "anchor"}
+	got, err := r.Compose(Rendering{BusMembership: "sealed-blob"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var found map[string]any
+	for _, res := range got.Resources {
+		if res["id"] == BusMembershipID() {
+			found = res
+		}
+	}
+	if found == nil {
+		t.Fatalf("no membership resource in %v", got.Resources)
+	}
+	if found["path"] != BusMembershipPath || found["sealed"] != "sealed-blob" || found["mode"] != "0600" {
+		t.Fatalf("the membership is not a sealed 0600 file where the host reads it: %v", found)
+	}
+	// And a machine not being moved is handed nothing.
+	got, _ = r.Compose(Rendering{})
+	for _, res := range got.Resources {
+		if res["id"] == BusMembershipID() {
+			t.Fatal("a machine with no membership on record was handed one")
+		}
+	}
+}

internal/inventory/bususers.go

@@ -230,3 +230,35 @@ func (i *Inventory) ForgetPerson(ctx context.Context, name string) error {
 	}
 	return i.ForgetBusUser(ctx, "person."+name)
 }
+
+// PutBusMembership records a machine's membership for the new bus, sealed to it (design 28, 5.2).
+// Replaces any earlier one: a machine has one membership per bus, and re-minting is re-telling.
+func (i *Inventory) PutBusMembership(ctx context.Context, nodeName, sealed string) error {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return err
+	}
+	_, err = i.store.Pool().Exec(ctx,
+		`insert into bus_membership (node, sealed) values ($1, $2)
+		 on conflict (node) do update set sealed = excluded.sealed, since = now()`, node.ID, sealed)
+	return err
+}
+
+// BusMemberships is every machine's sealed membership for the new bus, by node name.
+func (i *Inventory) BusMemberships(ctx context.Context) (map[string]string, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select n.name, b.sealed from bus_membership b join node n on n.id = b.node`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]string{}
+	for rows.Next() {
+		var name, sealed string
+		if err := rows.Scan(&name, &sealed); err != nil {
+			return nil, err
+		}
+		out[name] = sealed
+	}
+	return out, rows.Err()
+}

internal/inventory/holdings_test.go

@@ -80,3 +80,20 @@ func TestUnassigningTheHolderTakesTheHoldingWithIt(t *testing.T) {
 		t.Fatalf("the holding outlived the assignment it pointed at: %+v", held)
 	}
 }
+
+func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T) {
+	inv, ctx := twoBrokersOnTwoNodes(t)
+	if err := inv.PutBusMembership(ctx, "anchor", "first"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.PutBusMembership(ctx, "anchor", "second"); err != nil {
+		t.Fatal(err)
+	}
+	got, err := inv.BusMemberships(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got["anchor"] != "second" || len(got) != 1 {
+		t.Fatalf("a re-told membership did not replace the first: %v", got)
+	}
+}

internal/inventory/migrations/0041-a-machine-is-told-its-membership-for-the-new-bus.sql

@@ -0,0 +1,11 @@
+-- A machine already enrolled is moved to the new bus by being told its membership for it
+-- (novox/hq design 28, task 5.2). Until this, a membership — bus address, fingerprint, password,
+-- transport — existed only in the enrolment reply, and nothing could hand one to a machine that
+-- had already joined. The row is the membership sealed to that machine, composed into its
+-- declaration as a file it reads after applying; the plaintext exists once, at minting, and then
+-- only on the machine. One per node: the mesh moves to one bus.
+create table bus_membership (
+  node   uuid        primary key references node(id) on delete cascade,
+  sealed text        not null,
+  since  timestamptz not null default now()
+);

module.json

@@ -23,7 +23,8 @@
     "licences": "/var/lib/mesh/mesh-controller/licences",
     "broker": "/var/lib/mesh/mesh-controller/broker",
     "broker-management": "/var/lib/mesh/mesh-controller/broker-management",
-    "broker-address": "/var/lib/mesh/mesh-controller/broker-address"
+    "broker-address": "/var/lib/mesh/mesh-controller/broker-address",
+    "bus": "/var/lib/mesh/mesh-controller/bus"
   },
   "secrets-owner": "65534:65534",
   "resources": [