mesh-controller/cmd/mesh-controller/rollout.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/nats-io/nats.go"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/secrets"
)

// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
//
// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both
// transports and every one of them chooses by a single fact; this is the step that flips it. That
// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays
// bounded until here — and it means the useful work is almost all in the checking.
//
// So `rollout check` is the command that matters and the one that can be run any number of times
// against a mesh that is serving. It answers from records: what is missing, and what would happen.
// `rollout` itself refuses unless the check is clean.
//
// **The old broker goes with the move, and goes last** (novox/hq ADR 0131): AMQP is not a provision,
// so once every machine reports on the new bus its module is unassigned. Only the mesh's own traffic
// is what moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's
// ability to change things, not the services its modules are serving — measured on 2026-09-27, when
// a seat emptied mid-change and the control plane looped for two hours while every service stayed up.

const rolloutUsage = "rollout check | rollout mint [--again] | rollout hand <node> | rollout --confirm"

func rolloutCommand(ctx context.Context, args []string) error {
	switch {
	case len(args) == 1 && args[0] == "check":
		return rolloutCheck(ctx)
	case len(args) == 1 && args[0] == "mint":
		return rolloutMint(ctx, false)
	case len(args) == 2 && args[0] == "hand":
		return rolloutHand(ctx, args[1])
	case len(args) == 2 && args[0] == "mint" && args[1] == "--again":
		// Every credential minted afresh, whether or not one exists — for a mint that was wrong
		// before anything was pushed. Afterwards nothing that received the old one still works,
		// which is fine exactly when nothing received it.
		return rolloutMint(ctx, true)
	case len(args) == 1 && args[0] == "--confirm":
		return errors.New(
			"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
				"what is missing. Moving every node at once is the one step with nothing to inspect " +
				"afterwards, so it is not being written before the check it depends on has been run " +
				"against a real mesh")
	default:
		return errors.New(rolloutUsage)
	}
}

// rolloutCheck says whether the mesh could move, and what would happen if it did.
func rolloutCheck(ctx context.Context) error {
	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory

	state, err := readinessOf(ctx, inv)
	if err != nil {
		return err
	}

	fmt.Println("the bus this mesh would move to")
	if state.TheBus == "" {
		fmt.Printf("  nothing names one (%s is unset)\n", broker.NATSVar)
	} else {
		standing := "not answering"
		if state.ServerStanding {
			standing = "answering"
		}
		fmt.Printf("  %s — %s\n", state.TheBus, standing)
	}
	fmt.Println()

	fmt.Println("what would move")
	for _, step := range broker.WhatMoves(state) {
		fmt.Printf("  %s\n", step)
	}
	fmt.Println()

	why := notReadyOf(state)
	if len(why) == 0 {
		fmt.Println("nothing is missing: this mesh could move its bus.")
		fmt.Println()
		fmt.Println("Read `what would move` above once more before running it. Every node moves at the")
		fmt.Println("same moment and there is no half-moved state to look at afterwards.")
		return nil
	}
	fmt.Printf("not ready — %d thing(s) to do first:\n", len(why))
	for i, w := range why {
		fmt.Printf("  %d. %s\n", i+1, w)
	}
	return nil
}

// readinessOf gathers what the mesh knows about its own ability to move.
//
// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the
// point: the answer is only useful if it can be had without committing to anything.
func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) {
	state := broker.Readiness{
		Credentialled:       map[string]bool{},
		ModuleCredentialled: map[string]bool{},
		// The old broker keeps its other clients on this installation, and saying so is how the plan
		// stops reading as a retirement.
	}

	address, _, err := broker.OnNATS()
	if err != nil {
		return state, err
	}
	state.TheBus = address
	if address != "" {
		// One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about
		// to move onto a server that is not there should hear it here rather than afterwards.
		if conn, err := nats.Connect(broker.BareAddress(address), nats.Timeout(5*time.Second)); err == nil {
			state.ServerStanding = true
			conn.Close()
		}
	}

	nodes, err := inv.Nodes(ctx)
	if err != nil {
		return state, err
	}
	kept, err := inv.BusUsers(ctx)
	if err != nil {
		return state, err
	}
	shelf, err := inv.Catalogue(ctx)
	if err != nil {
		return state, err
	}

	for _, n := range nodes {
		state.Nodes = append(state.Nodes, n.Name)
		_, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()]
		state.Credentialled[n.Name] = has

		assigned, err := inv.Assigned(ctx, n.Name)
		if err != nil {
			return state, err
		}
		for _, module := range assigned {
			m, known := shelf[module]
			if !known {
				continue
			}
			// The machine that holds the bus seat is the one that would be sent the user list.
			if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
				state.Holder = n.Name
				state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
			}
			// A module that never speaks needs no credential, so it is not counted as missing one.
			if !speaksOnTheBus(m) {
				continue
			}
			named := n.Name + "/" + module
			state.Modules = append(state.Modules, named)
			_, hasOne := kept[broker.Principal{
				Kind: broker.KindModule, Node: n.Name, Module: module,
			}.Username()]
			state.ModuleCredentialled[named] = hasOne
		}
	}
	return state, nil
}

// speaksOnTheBus says whether a module reaches the bus at all.
//
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
// would bury the ones that matter under a list nobody can act on.
func speaksOnTheBus(m catalogue.Manifest) bool {
	return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
		len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
}

// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
// list became part of one.
//
// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
// has still been sent it, and this question is about whether the mesh did its part.
func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
	digest, err := inv.Outstanding(ctx, node)
	return err == nil && strings.TrimSpace(digest) != ""
}

// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
// printing. The reasoning itself is the broker package's, where it is pure.
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }

// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
// sealed into its declaration, a module's as its broker secret, the control plane's own as its
// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
//
// **Before anything moves, and it is what makes moving possible.** A machine moved without a
// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
// the module that provides it is assigned, rather than read from this process's environment: this
// process is still on the old bus when this runs, and must be.
func rolloutMint(ctx context.Context, again bool) error {
	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory

	known, err := broker.FromEnvironment()
	if err != nil {
		return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
			"must carry its fingerprint: %w", err)
	}
	shelf, err := inv.Catalogue(ctx)
	if err != nil {
		return err
	}
	entries, err := inv.Catalogued(ctx)
	if err != nil {
		return err
	}
	var busNode, controllerNode string
	for _, e := range entries {
		switch {
		case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
			busNode = e.On[0]
		case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
			controllerNode = e.On[0]
		}
	}
	if busNode == "" {
		return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
			"bus to mint credentials for — register and assign it first")
	}
	onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
	if err != nil {
		return err
	}
	busHost := onNetwork[busNode]
	if busHost == "" {
		// **The hub is not in that map.** The machine that took over the tunnel is where the current
		// bus already answers, and every machine dials it at the address the mesh handed them — so
		// when the new bus runs on the same machine, that address is the one to tell them, with the
		// new port. Found live: the control node is the hub, and the map lists the machines placed
		// around it.
		// The host alone: no scheme (BareAddress adds one where none was, which is the wrong
		// direction here — every URL built below adds its own) and no port.
		_, _, host := broker.CredentialIn(known.Address)
		if host == "" {
			host = known.Address
		}
		if _, after, hasScheme := strings.Cut(host, "://"); hasScheme {
			host = after
		}
		host = strings.TrimSpace(host)
		if i := strings.LastIndex(host, ":"); i > 0 && !strings.Contains(host[i:], "]") {
			host = host[:i]
		}
		if host == "" {
			return fmt.Errorf("%s runs the new bus and has no address on the private network, and the "+
				"current bus's address is unknown too, so no machine could be told where it is", busNode)
		}
		busHost = host
	}
	busAddress := busHost + ":4222"

	records, err := inv.BusRecords(ctx)
	if err != nil {
		return err
	}
	users, err := broker.Users(records)
	if err != nil {
		return err
	}
	kept, err := inv.BusUsers(ctx)
	if err != nil {
		return err
	}
	hashes := make(map[string]string, len(kept))
	for name, u := range kept {
		hashes[name] = u.PasswordHash
	}
	_, missing := broker.WithPasswords(users, hashes)
	wanted := map[string]bool{}
	for _, m := range missing {
		wanted[m] = true
	}

	var machines, modules, skipped int
	for _, p := range users {
		if !again && !wanted[p.Username()] {
			continue
		}
		switch p.Kind {
		case broker.KindController:
			if controllerNode == "" {
				return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
			}
			password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
			if err != nil {
				return err
			}
			url := "nats://" + p.Username() + ":" + password + "@" + busAddress
			if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
				return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
			}
			fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)

		case broker.KindNode:
			password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
			if err != nil {
				return err
			}
			membership, _ := json.Marshal(map[string]string{
				"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
			})
			key, err := inv.SealingKeyOf(ctx, p.Node)
			if err != nil {
				return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
			}
			sealed, err := secrets.Seal(key, membership)
			if err != nil {
				return err
			}
			if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
				return err
			}
			machines++

		case broker.KindModule:
			if p.Module == "mesh-controller" {
				// The control plane is a module too, and its `broker` secret is the old bus's
				// credential it is still using while this runs. Writing the new bus's blob there
				// cut the mesh off from its own old bus mid-move (2026-09-28). Its new-bus credential
				// is the controller principal's `bus` secret above; nothing else is needed here.
				skipped++
				continue
			}
			m, inShelf := shelf[p.Module]
			if !inShelf {
				skipped++
				continue
			}
			if _, reads := m.OwnSecrets["broker"]; !reads {
				fmt.Printf("  %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
				skipped++
				continue
			}
			password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
			if err != nil {
				return err
			}
			if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
				return err
			}
			modules++

		default:
			skipped++
		}
	}
	fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
		machines, modules, skipped, busAddress)
	fmt.Println("  each machine's membership and each module's credential arrive with the next push of its machine;")
	fmt.Println("  push the machine running the bus first, so the bus stands with its user list before anything dials it")
	return nil
}

// providesBus is whether a manifest provides the mesh's bus.
func providesBus(m catalogue.Manifest) bool {
	for _, o := range m.Provides {
		if o.Name == "mesh-bus" {
			return true
		}
	}
	return false
}

// rolloutHand mints a machine its credential for the new bus afresh and prints its membership
// once, for an operator to carry by hand — the rescue for a machine that cannot be reached over
// any bus: rotated while it still held the old password, or reachable only by ssh. The plaintext
// exists on this terminal and then only where it is written; the store keeps the hash, and the
// sealed copy in the machine's declaration is replaced too, so the next push says the same.
func rolloutHand(ctx context.Context, node string) error {
	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory

	known, err := broker.FromEnvironment()
	if err != nil {
		return fmt.Errorf("the bus's certificate is not known to this process: %w", err)
	}
	busAddress, _, err := broker.OnNATS()
	if err != nil {
		return err
	}
	if busAddress == "" {
		return errors.New("this control plane is not on the new bus, so there is no membership to hand out")
	}
	_, _, bare := broker.CredentialIn(busAddress)
	if _, after, has := strings.Cut(bare, "://"); has {
		bare = after
	}
	if _, err := inv.NodeByName(ctx, node); err != nil {
		return err
	}
	p := broker.Principal{Kind: broker.KindNode, Node: node}
	password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: node})
	if err != nil {
		return err
	}
	membership, _ := json.Marshal(map[string]string{
		"broker": bare, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
	})
	key, err := inv.SealingKeyOf(ctx, node)
	if err != nil {
		return err
	}
	sealed, err := secrets.Seal(key, membership)
	if err != nil {
		return err
	}
	if err := inv.PutBusMembership(ctx, node, sealed); err != nil {
		return err
	}
	// The one line of output is the membership itself, so it can be piped to the machine without
	// being read on the way. Everything else goes to stderr.
	fmt.Fprintf(os.Stderr, "%s's credential is minted afresh. Write this to %s on it and restart its host; "+
		"then push the machine running the bus so the user list carries the new hash.\n",
		node, catalogue.BusMembershipPath)
	fmt.Println(string(membership))
	return nil
}