diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index fdb719f..6cfae4d 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -614,6 +614,15 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue if err != nil { return err } + return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password) +} + +// issueWith is the delivery half: the minted password sealed to the machine as the module's broker +// secret, and the module's consumer created where the bus can be reached. Split from the minting +// so the move can issue every module against a bus whose address it worked out itself +// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment. +func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, + node, busAddress string, known broker.Broker, reachable, user, password string) error { held, err := json.Marshal(struct { URL string `json:"url"` Fingerprint string `json:"fingerprint,omitempty"` @@ -639,14 +648,19 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue Kind: broker.KindModule, Node: node, Module: m.Module, Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools, }); needed { - js, err := broker.Dial(busAddress) - if err != nil { - return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+ - "how %s hears what it consumes: %w", m.Module, err) - } - defer js.Close() - if err := js.EnsureConsumer(consumer); err != nil { - return err + if busAddress == "" { + fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+ + "`rollout mint` again is harmless)\n", m.Module) + } else { + js, err := broker.Dial(busAddress) + if err != nil { + return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+ + "how %s hears what it consumes: %w", m.Module, err) + } + defer js.Close() + if err := js.EnsureConsumer(consumer); err != nil { + return err + } } } diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 0d90139..0280117 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -636,8 +636,13 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } + memberships, err := inv.BusMemberships(ctx) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } return catalogue.Rendering{ - Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, + BusMembership: memberships[node], + Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Machines: machines, Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation, diff --git a/cmd/mesh-controller/rollout.go b/cmd/mesh-controller/rollout.go index 13b9f24..e31ac01 100644 --- a/cmd/mesh-controller/rollout.go +++ b/cmd/mesh-controller/rollout.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -12,6 +13,7 @@ import ( "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/secrets" ) // Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5). @@ -31,12 +33,14 @@ import ( // ability to change things, not the services its modules are serving — measured on 2026-09-27, when // a seat emptied mid-change and the control plane looped for two hours while every service stayed up. -const rolloutUsage = "rollout check | rollout --confirm" +const rolloutUsage = "rollout check | rollout mint | rollout --confirm" func rolloutCommand(ctx context.Context, args []string) error { switch { case len(args) == 1 && args[0] == "check": return rolloutCheck(ctx) + case len(args) == 1 && args[0] == "mint": + return rolloutMint(ctx) case len(args) == 1 && args[0] == "--confirm": return errors.New( "the rollout itself is not built yet: `rollout check` answers whether it could run, and " + @@ -190,3 +194,161 @@ func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node stri // notReadyOf is the readiness reasoning, named here so a test can reach it without the command's // printing. The reasoning itself is the broker package's, where it is pure. func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) } + +// rolloutMint gives every principal the new bus will have a credential it does not yet have, and +// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership +// sealed into its declaration, a module's as its broker secret, the control plane's own as its +// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless. +// +// **Before anything moves, and it is what makes moving possible.** A machine moved without a +// credential cannot come back, and afterwards there is no bus to tell it anything over — which is +// why `rollout check` refuses until this has run. The bus's address is worked out here, from where +// the module that provides it is assigned, rather than read from this process's environment: this +// process is still on the old bus when this runs, and must be. +func rolloutMint(ctx context.Context) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + known, err := broker.FromEnvironment() + if err != nil { + return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+ + "must carry its fingerprint: %w", err) + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + return err + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return err + } + var busNode, controllerNode string + for _, e := range entries { + switch { + case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0: + busNode = e.On[0] + case e.Manifest.Module == "mesh-controller" && len(e.On) > 0: + controllerNode = e.On[0] + } + } + if busNode == "" { + return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " + + "bus to mint credentials for — register and assign it first") + } + onNetwork, err := whereEveryoneIs(ctx, inv, shelf) + if err != nil { + return err + } + if onNetwork[busNode] == "" { + return fmt.Errorf("%s runs the new bus and has no address on the private network, so no machine "+ + "could be told where it is", busNode) + } + busAddress := onNetwork[busNode] + ":4222" + + records, err := inv.BusRecords(ctx) + if err != nil { + return err + } + users, err := broker.Users(records) + if err != nil { + return err + } + kept, err := inv.BusUsers(ctx) + if err != nil { + return err + } + hashes := make(map[string]string, len(kept)) + for name, u := range kept { + hashes[name] = u.PasswordHash + } + _, missing := broker.WithPasswords(users, hashes) + wanted := map[string]bool{} + for _, m := range missing { + wanted[m] = true + } + + var machines, modules, skipped int + for _, p := range users { + if !wanted[p.Username()] { + continue + } + switch p.Kind { + case broker.KindController: + if controllerNode == "" { + return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go") + } + password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController}) + if err != nil { + return err + } + url := "nats://" + p.Username() + ":" + password + "@" + busAddress + if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil { + return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err) + } + fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode) + + case broker.KindNode: + password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node}) + if err != nil { + return err + } + membership, _ := json.Marshal(map[string]string{ + "broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats", + }) + key, err := inv.SealingKeyOf(ctx, p.Node) + if err != nil { + return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err) + } + sealed, err := secrets.Seal(key, membership) + if err != nil { + return err + } + if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil { + return err + } + machines++ + + case broker.KindModule: + m, inShelf := shelf[p.Module] + if !inShelf { + skipped++ + continue + } + if _, reads := m.OwnSecrets["broker"]; !reads { + fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node) + skipped++ + continue + } + password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module}) + if err != nil { + return err + } + if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil { + return err + } + modules++ + + default: + skipped++ + } + } + fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n", + machines, modules, skipped, busAddress) + fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;") + fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it") + return nil +} + +// providesBus is whether a manifest provides the mesh's bus. +func providesBus(m catalogue.Manifest) bool { + for _, o := range m.Provides { + if o.Name == "mesh-bus" { + return true + } + } + return false +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 1a9a410..fe6f76b 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -103,6 +103,11 @@ type Rendering struct { // **Only the users, never the server's own settings**: those are the module's, in its image and // its mounts (Manifest.BusUsers). BusUsers string + // BusMembership is this machine's membership for the bus the mesh is moving to, sealed to it + // (design 28, task 5.2). Empty for a machine not being moved. Written as a file the host reads + // after the declaration has applied, so the bus it names is standing before the machine leaves + // the one it is on. + BusMembership string // MeshRange is the private network's CIDR (the range node addresses are allocated from), for a // module that must name the whole mesh rather than one machine — an intrusion filter that must @@ -238,9 +243,23 @@ func (r Resolution) Compose(with Rendering) (Composed, error) { if err != nil { return Composed{}, err } + if with.BusMembership != "" { + // The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a + // secret and placed where the host looks for exactly this (design 28, task 5.2). + resources = append(resources, map[string]any{ + "id": BusMembershipID(), "type": "file", "path": BusMembershipPath, + "sealed": with.BusMembership, "mode": "0600", + }) + } return Composed{Resources: resources, Owner: owner}, nil } +// BusMembershipID names the resource carrying a machine's membership for the new bus, and +// BusMembershipPath is where the host reads it — the same constant on both sides. +func BusMembershipID() string { return "bus-membership" } + +const BusMembershipPath = "/var/lib/mesh/membership-next.json" + func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) { // Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings, // credentials and contributions land are resolved against this node's directories, so every diff --git a/internal/catalogue/holdings_test.go b/internal/catalogue/holdings_test.go index e89a4dc..6394447 100644 --- a/internal/catalogue/holdings_test.go +++ b/internal/catalogue/holdings_test.go @@ -114,3 +114,32 @@ func TestCanHoldJudgesClaimScopeAndWhatTheSeatDelivers(t *testing.T) { t.Fatalf("with the row saying amqp, an amqp provider was refused: %v", err) } } + +// A machine being moved is handed its membership for the new bus as a sealed file in its own +// declaration — the machine's, not any module's (design 28, task 5.2). +func TestAMembershipForTheNewBusIsComposedAsASealedFile(t *testing.T) { + r := Resolution{Node: "anchor"} + got, err := r.Compose(Rendering{BusMembership: "sealed-blob"}) + if err != nil { + t.Fatal(err) + } + var found map[string]any + for _, res := range got.Resources { + if res["id"] == BusMembershipID() { + found = res + } + } + if found == nil { + t.Fatalf("no membership resource in %v", got.Resources) + } + if found["path"] != BusMembershipPath || found["sealed"] != "sealed-blob" || found["mode"] != "0600" { + t.Fatalf("the membership is not a sealed 0600 file where the host reads it: %v", found) + } + // And a machine not being moved is handed nothing. + got, _ = r.Compose(Rendering{}) + for _, res := range got.Resources { + if res["id"] == BusMembershipID() { + t.Fatal("a machine with no membership on record was handed one") + } + } +} diff --git a/internal/inventory/bususers.go b/internal/inventory/bususers.go index b46a643..c8c983b 100644 --- a/internal/inventory/bususers.go +++ b/internal/inventory/bususers.go @@ -230,3 +230,35 @@ func (i *Inventory) ForgetPerson(ctx context.Context, name string) error { } return i.ForgetBusUser(ctx, "person."+name) } + +// PutBusMembership records a machine's membership for the new bus, sealed to it (design 28, 5.2). +// Replaces any earlier one: a machine has one membership per bus, and re-minting is re-telling. +func (i *Inventory) PutBusMembership(ctx context.Context, nodeName, sealed string) error { + node, err := i.NodeByName(ctx, nodeName) + if err != nil { + return err + } + _, err = i.store.Pool().Exec(ctx, + `insert into bus_membership (node, sealed) values ($1, $2) + on conflict (node) do update set sealed = excluded.sealed, since = now()`, node.ID, sealed) + return err +} + +// BusMemberships is every machine's sealed membership for the new bus, by node name. +func (i *Inventory) BusMemberships(ctx context.Context) (map[string]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select n.name, b.sealed from bus_membership b join node n on n.id = b.node`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var name, sealed string + if err := rows.Scan(&name, &sealed); err != nil { + return nil, err + } + out[name] = sealed + } + return out, rows.Err() +} diff --git a/internal/inventory/holdings_test.go b/internal/inventory/holdings_test.go index 3525cdc..c52f7d1 100644 --- a/internal/inventory/holdings_test.go +++ b/internal/inventory/holdings_test.go @@ -80,3 +80,20 @@ func TestUnassigningTheHolderTakesTheHoldingWithIt(t *testing.T) { t.Fatalf("the holding outlived the assignment it pointed at: %+v", held) } } + +func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T) { + inv, ctx := twoBrokersOnTwoNodes(t) + if err := inv.PutBusMembership(ctx, "anchor", "first"); err != nil { + t.Fatal(err) + } + if err := inv.PutBusMembership(ctx, "anchor", "second"); err != nil { + t.Fatal(err) + } + got, err := inv.BusMemberships(ctx) + if err != nil { + t.Fatal(err) + } + if got["anchor"] != "second" || len(got) != 1 { + t.Fatalf("a re-told membership did not replace the first: %v", got) + } +} diff --git a/internal/inventory/migrations/0041-a-machine-is-told-its-membership-for-the-new-bus.sql b/internal/inventory/migrations/0041-a-machine-is-told-its-membership-for-the-new-bus.sql new file mode 100644 index 0000000..ea34233 --- /dev/null +++ b/internal/inventory/migrations/0041-a-machine-is-told-its-membership-for-the-new-bus.sql @@ -0,0 +1,11 @@ +-- A machine already enrolled is moved to the new bus by being told its membership for it +-- (novox/hq design 28, task 5.2). Until this, a membership — bus address, fingerprint, password, +-- transport — existed only in the enrolment reply, and nothing could hand one to a machine that +-- had already joined. The row is the membership sealed to that machine, composed into its +-- declaration as a file it reads after applying; the plaintext exists once, at minting, and then +-- only on the machine. One per node: the mesh moves to one bus. +create table bus_membership ( + node uuid primary key references node(id) on delete cascade, + sealed text not null, + since timestamptz not null default now() +); diff --git a/module.json b/module.json index 277030c..85e1466 100644 --- a/module.json +++ b/module.json @@ -23,7 +23,8 @@ "licences": "/var/lib/mesh/mesh-controller/licences", "broker": "/var/lib/mesh/mesh-controller/broker", "broker-management": "/var/lib/mesh/mesh-controller/broker-management", - "broker-address": "/var/lib/mesh/mesh-controller/broker-address" + "broker-address": "/var/lib/mesh/mesh-controller/broker-address", + "bus": "/var/lib/mesh/mesh-controller/bus" }, "secrets-owner": "65534:65534", "resources": [