From e8aa7ed9e77f3322c365f4092c25b5d84d8e8e1f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 28 Sep 2026 00:16:24 +0200 Subject: [PATCH] The move mints every credential and tells each machine its membership MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rollout mint` gives every principal the new bus will have a credential it does not yet have and puts each where its owner reads it: a machine's as a membership — bus address, fingerprint, password, transport — sealed into its declaration (migration 0041, the `bus-membership` resource the host reads after applying); a module's as its broker secret, through the same delivery `module issue` uses; the control plane's own as its `bus` secret. Idempotent, and worked out from where the bus's module is assigned rather than from this process's environment, because this process is still on the old bus when it runs and must be. This is the half of design 28 task 5.2 the first live attempt found missing: a credential was minted only at enrolment, at `module issue` and for a person, so no machine already enrolled could ever be moved. `rollout check` was right to refuse; now there is something to run first. --- cmd/mesh-controller/modules.go | 30 +++- cmd/mesh-controller/plan.go | 7 +- cmd/mesh-controller/rollout.go | 164 +++++++++++++++++- internal/catalogue/declaration.go | 19 ++ internal/catalogue/holdings_test.go | 29 ++++ internal/inventory/bususers.go | 32 ++++ internal/inventory/holdings_test.go | 17 ++ ...is-told-its-membership-for-the-new-bus.sql | 11 ++ module.json | 3 +- 9 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 internal/inventory/migrations/0041-a-machine-is-told-its-membership-for-the-new-bus.sql diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index fdb719f..6cfae4d 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -614,6 +614,15 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue if err != nil { return err } + return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password) +} + +// issueWith is the delivery half: the minted password sealed to the machine as the module's broker +// secret, and the module's consumer created where the bus can be reached. Split from the minting +// so the move can issue every module against a bus whose address it worked out itself +// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment. +func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, + node, busAddress string, known broker.Broker, reachable, user, password string) error { held, err := json.Marshal(struct { URL string `json:"url"` Fingerprint string `json:"fingerprint,omitempty"` @@ -639,14 +648,19 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue Kind: broker.KindModule, Node: node, Module: m.Module, Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools, }); needed { - js, err := broker.Dial(busAddress) - if err != nil { - return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+ - "how %s hears what it consumes: %w", m.Module, err) - } - defer js.Close() - if err := js.EnsureConsumer(consumer); err != nil { - return err + if busAddress == "" { + fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+ + "`rollout mint` again is harmless)\n", m.Module) + } else { + js, err := broker.Dial(busAddress) + if err != nil { + return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+ + "how %s hears what it consumes: %w", m.Module, err) + } + defer js.Close() + if err := js.EnsureConsumer(consumer); err != nil { + return err + } } } diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 0d90139..0280117 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -636,8 +636,13 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } + memberships, err := inv.BusMemberships(ctx) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } return catalogue.Rendering{ - Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, + BusMembership: memberships[node], + Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Machines: machines, Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation, diff --git a/cmd/mesh-controller/rollout.go b/cmd/mesh-controller/rollout.go index 13b9f24..e31ac01 100644 --- a/cmd/mesh-controller/rollout.go +++ b/cmd/mesh-controller/rollout.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -12,6 +13,7 @@ import ( "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/secrets" ) // Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5). @@ -31,12 +33,14 @@ import ( // ability to change things, not the services its modules are serving — measured on 2026-09-27, when // a seat emptied mid-change and the control plane looped for two hours while every service stayed up. -const rolloutUsage = "rollout check | rollout --confirm" +const rolloutUsage = "rollout check | rollout mint | rollout --confirm" func rolloutCommand(ctx context.Context, args []string) error { switch { case len(args) == 1 && args[0] == "check": return rolloutCheck(ctx) + case len(args) == 1 && args[0] == "mint": + return rolloutMint(ctx) case len(args) == 1 && args[0] == "--confirm": return errors.New( "the rollout itself is not built yet: `rollout check` answers whether it could run, and " + @@ -190,3 +194,161 @@ func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node stri // notReadyOf is the readiness reasoning, named here so a test can reach it without the command's // printing. The reasoning itself is the broker package's, where it is pure. func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) } + +// rolloutMint gives every principal the new bus will have a credential it does not yet have, and +// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership +// sealed into its declaration, a module's as its broker secret, the control plane's own as its +// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless. +// +// **Before anything moves, and it is what makes moving possible.** A machine moved without a +// credential cannot come back, and afterwards there is no bus to tell it anything over — which is +// why `rollout check` refuses until this has run. The bus's address is worked out here, from where +// the module that provides it is assigned, rather than read from this process's environment: this +// process is still on the old bus when this runs, and must be. +func rolloutMint(ctx context.Context) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + known, err := broker.FromEnvironment() + if err != nil { + return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+ + "must carry its fingerprint: %w", err) + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + return err + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return err + } + var busNode, controllerNode string + for _, e := range entries { + switch { + case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0: + busNode = e.On[0] + case e.Manifest.Module == "mesh-controller" && len(e.On) > 0: + controllerNode = e.On[0] + } + } + if busNode == "" { + return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " + + "bus to mint credentials for — register and assign it first") + } + onNetwork, err := whereEveryoneIs(ctx, inv, shelf) + if err != nil { + return err + } + if onNetwork[busNode] == "" { + return fmt.Errorf("%s runs the new bus and has no address on the private network, so no machine "+ + "could be told where it is", busNode) + } + busAddress := onNetwork[busNode] + ":4222" + + records, err := inv.BusRecords(ctx) + if err != nil { + return err + } + users, err := broker.Users(records) + if err != nil { + return err + } + kept, err := inv.BusUsers(ctx) + if err != nil { + return err + } + hashes := make(map[string]string, len(kept)) + for name, u := range kept { + hashes[name] = u.PasswordHash + } + _, missing := broker.WithPasswords(users, hashes) + wanted := map[string]bool{} + for _, m := range missing { + wanted[m] = true + } + + var machines, modules, skipped int + for _, p := range users { + if !wanted[p.Username()] { + continue + } + switch p.Kind { + case broker.KindController: + if controllerNode == "" { + return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go") + } + password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController}) + if err != nil { + return err + } + url := "nats://" + p.Username() + ":" + password + "@" + busAddress + if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil { + return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err) + } + fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode) + + case broker.KindNode: + password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node}) + if err != nil { + return err + } + membership, _ := json.Marshal(map[string]string{ + "broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats", + }) + key, err := inv.SealingKeyOf(ctx, p.Node) + if err != nil { + return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err) + } + sealed, err := secrets.Seal(key, membership) + if err != nil { + return err + } + if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil { + return err + } + machines++ + + case broker.KindModule: + m, inShelf := shelf[p.Module] + if !inShelf { + skipped++ + continue + } + if _, reads := m.OwnSecrets["broker"]; !reads { + fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node) + skipped++ + continue + } + password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module}) + if err != nil { + return err + } + if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil { + return err + } + modules++ + + default: + skipped++ + } + } + fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n", + machines, modules, skipped, busAddress) + fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;") + fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it") + return nil +} + +// providesBus is whether a manifest provides the mesh's bus. +func providesBus(m catalogue.Manifest) bool { + for _, o := range m.Provides { + if o.Name == "mesh-bus" { + return true + } + } + return false +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 1a9a410..fe6f76b 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -103,6 +103,11 @@ type Rendering struct { // **Only the users, never the server's own settings**: those are the module's, in its image and // its mounts (Manifest.BusUsers). BusUsers string + // BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it + // (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads + // after the declaration has applied, so the bus it names is standing before the machine leaves + // the one it is on. + BusMembership string // MeshRange is the private network's CIDR (the range node addresses are allocated from), for a // module that must name the whole mesh rather than one machine — an intrusion filter that must @@ -238,9 +243,23 @@ func (r Resolution) Compose(with Rendering) (Composed, error) { if err != nil { return Composed{}, err } + if with.BusMembership != "" { + // The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a + // secret and placed where the host looks for exactly this (design 28, task 5.2). + resources = append(resources, map[string]any{ + "id": BusMembershipID(), "type": "file", "path": BusMembershipPath, + "sealed": with.BusMembership, "mode": "0600", + }) + } return Composed{Resources: resources, Owner: owner}, nil } +// BusMembershipID names the resource carrying a machine's membership for the new bus, and +// BusMembershipPath is where the host reads it — the same constant on both sides. +func BusMembershipID() string { return "bus-membership" } + +const BusMembershipPath = "/var/lib/mesh/membership-next.json" + func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) { // Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings, // credentials and contributions land are resolved against this node's directories, so every diff --git a/internal/catalogue/holdings_test.go b/internal/catalogue/holdings_test.go index e89a4dc..6394447 100644 --- a/internal/catalogue/holdings_test.go +++ b/internal/catalogue/holdings_test.go @@ -114,3 +114,32 @@ func TestCanHoldJudgesClaimScopeAndWhatTheSeatDelivers(t *testing.T) { t.Fatalf("with the row saying amqp, an amqp provider was refused: %v", err) } } + +// A machine being moved is handed its membership for the new bus as a sealed file in its own +// declaration — the machine's, not any module's (design 28, task 5.2). +func TestAMembershipForTheNewBusIsComposedAsASealedFile(t *testing.T) { + r := Resolution{Node: "anchor"} + got, err := r.Compose(Rendering{BusMembership: "sealed-blob"}) + if err != nil { + t.Fatal(err) + } + var found map[string]any + for _, res := range got.Resources { + if res["id"] == BusMembershipID() { + found = res + } + } + if found == nil { + t.Fatalf("no membership resource in %v", got.Resources) + } + if found["path"] != BusMembershipPath || found["sealed"] != "sealed-blob" || found["mode"] != "0600" { + t.Fatalf("the membership is not a sealed 0600 file where the host reads it: %v", found) + } + // And a machine not being moved is handed nothing. + got, _ = r.Compose(Rendering{}) + for _, res := range got.Resources { + if res["id"] == BusMembershipID() { + t.Fatal("a machine with no membership on record was handed one") + } + } +} diff --git a/internal/inventory/bususers.go b/internal/inventory/bususers.go index b46a643..c8c983b 100644 --- a/internal/inventory/bususers.go +++ b/internal/inventory/bususers.go @@ -230,3 +230,35 @@ func (i *Inventory) ForgetPerson(ctx context.Context, name string) error { } return i.ForgetBusUser(ctx, "person."+name) } + +// PutBusMembership records a machine's membership for the new bus, sealed to it (design 28, 5.2). +// Replaces any earlier one: a machine has one membership per bus, and re-minting is re-telling. +func (i *Inventory) PutBusMembership(ctx context.Context, nodeName, sealed string) error { + node, err := i.NodeByName(ctx, nodeName) + if err != nil { + return err + } + _, err = i.store.Pool().Exec(ctx, + `insert into bus_membership (node, sealed) values ($1, $2) + on conflict (node) do update set sealed = excluded.sealed, since = now()`, node.ID, sealed) + return err +} + +// BusMemberships is every machine's sealed membership for the new bus, by node name. +func (i *Inventory) BusMemberships(ctx context.Context) (map[string]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select n.name, b.sealed from bus_membership b join node n on n.id = b.node`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var name, sealed string + if err := rows.Scan(&name, &sealed); err != nil { + return nil, err + } + out[name] = sealed + } + return out, rows.Err() +} diff --git a/internal/inventory/holdings_test.go b/internal/inventory/holdings_test.go index 3525cdc..c52f7d1 100644 --- a/internal/inventory/holdings_test.go +++ b/internal/inventory/holdings_test.go @@ -80,3 +80,20 @@ func TestUnassigningTheHolderTakesTheHoldingWithIt(t *testing.T) { t.Fatalf("the holding outlived the assignment it pointed at: %+v", held) } } + +func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T) { + inv, ctx := twoBrokersOnTwoNodes(t) + if err := inv.PutBusMembership(ctx, "anchor", "first"); err != nil { + t.Fatal(err) + } + if err := inv.PutBusMembership(ctx, "anchor", "second"); err != nil { + t.Fatal(err) + } + got, err := inv.BusMemberships(ctx) + if err != nil { + t.Fatal(err) + } + if got["anchor"] != "second" || len(got) != 1 { + t.Fatalf("a re-told membership did not replace the first: %v", got) + } +} diff --git a/internal/inventory/migrations/0041-a-machine-is-told-its-membership-for-the-new-bus.sql b/internal/inventory/migrations/0041-a-machine-is-told-its-membership-for-the-new-bus.sql new file mode 100644 index 0000000..ea34233 --- /dev/null +++ b/internal/inventory/migrations/0041-a-machine-is-told-its-membership-for-the-new-bus.sql @@ -0,0 +1,11 @@ +-- A machine already enrolled is moved to the new bus by being told its membership for it +-- (novox/hq design 28, task 5.2). Until this, a membership — bus address, fingerprint, password, +-- transport — existed only in the enrolment reply, and nothing could hand one to a machine that +-- had already joined. The row is the membership sealed to that machine, composed into its +-- declaration as a file it reads after applying; the plaintext exists once, at minting, and then +-- only on the machine. One per node: the mesh moves to one bus. +create table bus_membership ( + node uuid primary key references node(id) on delete cascade, + sealed text not null, + since timestamptz not null default now() +); diff --git a/module.json b/module.json index 277030c..85e1466 100644 --- a/module.json +++ b/module.json @@ -23,7 +23,8 @@ "licences": "/var/lib/mesh/mesh-controller/licences", "broker": "/var/lib/mesh/mesh-controller/broker", "broker-management": "/var/lib/mesh/mesh-controller/broker-management", - "broker-address": "/var/lib/mesh/mesh-controller/broker-address" + "broker-address": "/var/lib/mesh/mesh-controller/broker-address", + "bus": "/var/lib/mesh/mesh-controller/bus" }, "secrets-owner": "65534:65534", "resources": [