package main import ( "context" "encoding/json" "errors" "fmt" "os" "strings" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/secrets" ) // Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5). // // **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both // transports and every one of them chooses by a single fact; this is the step that flips it. That // shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays // bounded until here — and it means the useful work is almost all in the checking. // // So `rollout check` is the command that matters and the one that can be run any number of times // against a mesh that is serving. It answers from records: what is missing, and what would happen. // `rollout` itself refuses unless the check is clean. // // **The old broker goes with the move, and goes last** (novox/hq ADR 0131): AMQP is not a provision, // so once every machine reports on the new bus its module is unassigned. Only the mesh's own traffic // is what moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's // ability to change things, not the services its modules are serving — measured on 2026-09-27, when // a seat emptied mid-change and the control plane looped for two hours while every service stayed up. const rolloutUsage = "rollout check | rollout mint [--again] | rollout hand | rollout --confirm" func rolloutCommand(ctx context.Context, args []string) error { switch { case len(args) == 1 && args[0] == "check": return rolloutCheck(ctx) case len(args) == 1 && args[0] == "mint": return rolloutMint(ctx, false) case len(args) == 2 && args[0] == "hand": return rolloutHand(ctx, args[1]) case len(args) == 2 && args[0] == "mint" && args[1] == "--again": // Every credential minted afresh, whether or not one exists — for a mint that was wrong // before anything was pushed. Afterwards nothing that received the old one still works, // which is fine exactly when nothing received it. return rolloutMint(ctx, true) case len(args) == 1 && args[0] == "--confirm": return errors.New( "the rollout itself is not built yet: `rollout check` answers whether it could run, and " + "what is missing. Moving every node at once is the one step with nothing to inspect " + "afterwards, so it is not being written before the check it depends on has been run " + "against a real mesh") default: return errors.New(rolloutUsage) } } // rolloutCheck says whether the mesh could move, and what would happen if it did. func rolloutCheck(ctx context.Context) error { open, err := openStores(ctx) if err != nil { return err } defer open.Close() inv := open.inventory state, err := readinessOf(ctx, inv) if err != nil { return err } fmt.Println("the bus this mesh would move to") if state.TheBus == "" { fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar) } else { standing := "not answering" if state.ServerStanding { standing = "answering" } fmt.Printf(" %s — %s\n", state.TheBus, standing) } fmt.Println() fmt.Println("what would move") for _, step := range broker.WhatMoves(state) { fmt.Printf(" %s\n", step) } fmt.Println() why := notReadyOf(state) if len(why) == 0 { fmt.Println("nothing is missing: this mesh could move its bus.") fmt.Println() fmt.Println("Read `what would move` above once more before running it. Every node moves at the") fmt.Println("same moment and there is no half-moved state to look at afterwards.") return nil } fmt.Printf("not ready — %d thing(s) to do first:\n", len(why)) for i, w := range why { fmt.Printf(" %d. %s\n", i+1, w) } return nil } // readinessOf gathers what the mesh knows about its own ability to move. // // Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the // point: the answer is only useful if it can be had without committing to anything. func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) { state := broker.Readiness{ Credentialled: map[string]bool{}, ModuleCredentialled: map[string]bool{}, // The old broker keeps its other clients on this installation, and saying so is how the plan // stops reading as a retirement. } address, _, err := broker.OnNATS() if err != nil { return state, err } state.TheBus = address if address != "" { // One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about // to move onto a server that is not there should hear it here rather than afterwards. // // **Dialled the way the mesh dials it** — credential and pin — because a bare connect to a // bus that requires TLS and a user fails at the handshake, and the check then reported a // standing server as absent (seen live, 2026-09-28). if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil { state.ServerStanding = true js.Close() } } nodes, err := inv.Nodes(ctx) if err != nil { return state, err } kept, err := inv.BusUsers(ctx) if err != nil { return state, err } shelf, err := inv.Catalogue(ctx) if err != nil { return state, err } for _, n := range nodes { state.Nodes = append(state.Nodes, n.Name) _, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()] state.Credentialled[n.Name] = has assigned, err := inv.Assigned(ctx, n.Name) if err != nil { return state, err } for _, module := range assigned { m, known := shelf[module] if !known { continue } // The machine that holds the bus seat is the one that would be sent the user list. if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") { state.Holder = n.Name state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name) } // A module that never speaks needs no credential, so it is not counted as missing one. if !speaksOnTheBus(m) { continue } named := n.Name + "/" + module state.Modules = append(state.Modules, named) _, hasOne := kept[broker.Principal{ Kind: broker.KindModule, Node: n.Name, Module: module, }.Username()] state.ModuleCredentialled[named] = hasOne } } return state, nil } // speaksOnTheBus says whether a module reaches the bus at all. // // A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential // would bury the ones that matter under a list nobody can act on. func speaksOnTheBus(m catalogue.Manifest) bool { return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 || len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0 } // wasSentTheUserList says whether the machine holding the bus has had a declaration since the user // list became part of one. // // Read from what the mesh recorded sending rather than asked of the machine: a machine that is away // has still been sent it, and this question is about whether the mesh did its part. func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool { digest, err := inv.Outstanding(ctx, node) return err == nil && strings.TrimSpace(digest) != "" } // notReadyOf is the readiness reasoning, named here so a test can reach it without the command's // printing. The reasoning itself is the broker package's, where it is pure. func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) } // rolloutMint gives every principal the new bus will have a credential it does not yet have, and // puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership // sealed into its declaration, a module's as its broker secret, the control plane's own as its // `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless. // // **Before anything moves, and it is what makes moving possible.** A machine moved without a // credential cannot come back, and afterwards there is no bus to tell it anything over — which is // why `rollout check` refuses until this has run. The bus's address is worked out here, from where // the module that provides it is assigned, rather than read from this process's environment: this // process is still on the old bus when this runs, and must be. func rolloutMint(ctx context.Context, again bool) error { open, err := openStores(ctx) if err != nil { return err } defer open.Close() inv := open.inventory known, err := broker.FromEnvironment() if err != nil { return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+ "must carry its fingerprint: %w", err) } shelf, err := inv.Catalogue(ctx) if err != nil { return err } entries, err := inv.Catalogued(ctx) if err != nil { return err } var busNode, controllerNode string for _, e := range entries { switch { case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0: busNode = e.On[0] case e.Manifest.Module == "mesh-controller" && len(e.On) > 0: controllerNode = e.On[0] } } if busNode == "" { return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " + "bus to mint credentials for — register and assign it first") } onNetwork, err := whereEveryoneIs(ctx, inv, shelf) if err != nil { return err } busHost := onNetwork[busNode] if busHost == "" { // **The hub is not in that map.** The machine that took over the tunnel is where the current // bus already answers, and every machine dials it at the address the mesh handed them — so // when the new bus runs on the same machine, that address is the one to tell them, with the // new port. Found live: the control node is the hub, and the map lists the machines placed // around it. // The host alone: no scheme (BareAddress adds one where none was, which is the wrong // direction here — every URL built below adds its own) and no port. _, _, host := broker.CredentialIn(known.Address) if host == "" { host = known.Address } if _, after, hasScheme := strings.Cut(host, "://"); hasScheme { host = after } host = strings.TrimSpace(host) if i := strings.LastIndex(host, ":"); i > 0 && !strings.Contains(host[i:], "]") { host = host[:i] } if host == "" { return fmt.Errorf("%s runs the new bus and has no address on the private network, and the "+ "current bus's address is unknown too, so no machine could be told where it is", busNode) } busHost = host } busAddress := busHost + ":4222" records, err := inv.BusRecords(ctx) if err != nil { return err } users, err := broker.Users(records) if err != nil { return err } kept, err := inv.BusUsers(ctx) if err != nil { return err } hashes := make(map[string]string, len(kept)) for name, u := range kept { hashes[name] = u.PasswordHash } _, missing := broker.WithPasswords(users, hashes) wanted := map[string]bool{} for _, m := range missing { wanted[m] = true } var machines, modules, skipped int for _, p := range users { if !again && !wanted[p.Username()] { continue } switch p.Kind { case broker.KindController: if controllerNode == "" { return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go") } password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController}) if err != nil { return err } url := "nats://" + p.Username() + ":" + password + "@" + busAddress if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil { return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err) } fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode) case broker.KindNode: password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node}) if err != nil { return err } membership, _ := json.Marshal(map[string]string{ "broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats", }) key, err := inv.SealingKeyOf(ctx, p.Node) if err != nil { return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err) } sealed, err := secrets.Seal(key, membership) if err != nil { return err } if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil { return err } machines++ case broker.KindModule: if p.Module == "mesh-controller" { // The control plane is a module too, and its `broker` secret is the old bus's // credential it is still using while this runs. Writing the new bus's blob there // cut the mesh off from its own old bus mid-move (2026-09-28). Its new-bus credential // is the controller principal's `bus` secret above; nothing else is needed here. skipped++ continue } m, inShelf := shelf[p.Module] if !inShelf { skipped++ continue } if _, reads := m.OwnSecrets["broker"]; !reads { fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node) skipped++ continue } password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module}) if err != nil { return err } if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil { return err } modules++ default: skipped++ } } fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n", machines, modules, skipped, busAddress) fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;") fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it") return nil } // providesBus is whether a manifest provides the mesh's bus. func providesBus(m catalogue.Manifest) bool { for _, o := range m.Provides { if o.Name == "mesh-bus" { return true } } return false } // rolloutHand mints a machine its credential for the new bus afresh and prints its membership // once, for an operator to carry by hand — the rescue for a machine that cannot be reached over // any bus: rotated while it still held the old password, or reachable only by ssh. The plaintext // exists on this terminal and then only where it is written; the store keeps the hash, and the // sealed copy in the machine's declaration is replaced too, so the next push says the same. func rolloutHand(ctx context.Context, node string) error { open, err := openStores(ctx) if err != nil { return err } defer open.Close() inv := open.inventory known, err := broker.FromEnvironment() if err != nil { return fmt.Errorf("the bus's certificate is not known to this process: %w", err) } busAddress, _, err := broker.OnNATS() if err != nil { return err } if busAddress == "" { return errors.New("this control plane is not on the new bus, so there is no membership to hand out") } _, _, bare := broker.CredentialIn(busAddress) if _, after, has := strings.Cut(bare, "://"); has { bare = after } if _, err := inv.NodeByName(ctx, node); err != nil { return err } p := broker.Principal{Kind: broker.KindNode, Node: node} password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: node}) if err != nil { return err } membership, _ := json.Marshal(map[string]string{ "broker": bare, "fingerprint": known.Fingerprint, "password": password, "transport": "nats", }) key, err := inv.SealingKeyOf(ctx, node) if err != nil { return err } sealed, err := secrets.Seal(key, membership) if err != nil { return err } if err := inv.PutBusMembership(ctx, node, sealed); err != nil { return err } // The one line of output is the membership itself, so it can be piped to the machine without // being read on the way. Everything else goes to stderr. fmt.Fprintf(os.Stderr, "%s's credential is minted afresh. Write this to %s on it and restart its host; "+ "then push the machine running the bus so the user list carries the new hash.\n", node, catalogue.BusMembershipPath) fmt.Println(string(membership)) return nil }