jschoubben
e62201e227
The probe connected bare, and a bus that requires TLS and a user refused it at the handshake — so the check reported the standing server as absent. It now dials with the controller's own credential and pin, which is the one fact the check is there to report.
456 lines
16 KiB
Go
456 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/secrets"
|
|
)
|
|
|
|
// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
|
|
//
|
|
// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both
|
|
// transports and every one of them chooses by a single fact; this is the step that flips it. That
|
|
// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays
|
|
// bounded until here — and it means the useful work is almost all in the checking.
|
|
//
|
|
// So `rollout check` is the command that matters and the one that can be run any number of times
|
|
// against a mesh that is serving. It answers from records: what is missing, and what would happen.
|
|
// `rollout` itself refuses unless the check is clean.
|
|
//
|
|
// **The old broker goes with the move, and goes last** (novox/hq ADR 0131): AMQP is not a provision,
|
|
// so once every machine reports on the new bus its module is unassigned. Only the mesh's own traffic
|
|
// is what moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's
|
|
// ability to change things, not the services its modules are serving — measured on 2026-09-27, when
|
|
// a seat emptied mid-change and the control plane looped for two hours while every service stayed up.
|
|
|
|
const rolloutUsage = "rollout check | rollout mint [--again] | rollout hand <node> | rollout --confirm"
|
|
|
|
func rolloutCommand(ctx context.Context, args []string) error {
|
|
switch {
|
|
case len(args) == 1 && args[0] == "check":
|
|
return rolloutCheck(ctx)
|
|
case len(args) == 1 && args[0] == "mint":
|
|
return rolloutMint(ctx, false)
|
|
case len(args) == 2 && args[0] == "hand":
|
|
return rolloutHand(ctx, args[1])
|
|
case len(args) == 2 && args[0] == "mint" && args[1] == "--again":
|
|
// Every credential minted afresh, whether or not one exists — for a mint that was wrong
|
|
// before anything was pushed. Afterwards nothing that received the old one still works,
|
|
// which is fine exactly when nothing received it.
|
|
return rolloutMint(ctx, true)
|
|
case len(args) == 1 && args[0] == "--confirm":
|
|
return errors.New(
|
|
"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
|
|
"what is missing. Moving every node at once is the one step with nothing to inspect " +
|
|
"afterwards, so it is not being written before the check it depends on has been run " +
|
|
"against a real mesh")
|
|
default:
|
|
return errors.New(rolloutUsage)
|
|
}
|
|
}
|
|
|
|
// rolloutCheck says whether the mesh could move, and what would happen if it did.
|
|
func rolloutCheck(ctx context.Context) error {
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
inv := open.inventory
|
|
|
|
state, err := readinessOf(ctx, inv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Println("the bus this mesh would move to")
|
|
if state.TheBus == "" {
|
|
fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar)
|
|
} else {
|
|
standing := "not answering"
|
|
if state.ServerStanding {
|
|
standing = "answering"
|
|
}
|
|
fmt.Printf(" %s — %s\n", state.TheBus, standing)
|
|
}
|
|
fmt.Println()
|
|
|
|
fmt.Println("what would move")
|
|
for _, step := range broker.WhatMoves(state) {
|
|
fmt.Printf(" %s\n", step)
|
|
}
|
|
fmt.Println()
|
|
|
|
why := notReadyOf(state)
|
|
if len(why) == 0 {
|
|
fmt.Println("nothing is missing: this mesh could move its bus.")
|
|
fmt.Println()
|
|
fmt.Println("Read `what would move` above once more before running it. Every node moves at the")
|
|
fmt.Println("same moment and there is no half-moved state to look at afterwards.")
|
|
return nil
|
|
}
|
|
fmt.Printf("not ready — %d thing(s) to do first:\n", len(why))
|
|
for i, w := range why {
|
|
fmt.Printf(" %d. %s\n", i+1, w)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// readinessOf gathers what the mesh knows about its own ability to move.
|
|
//
|
|
// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the
|
|
// point: the answer is only useful if it can be had without committing to anything.
|
|
func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) {
|
|
state := broker.Readiness{
|
|
Credentialled: map[string]bool{},
|
|
ModuleCredentialled: map[string]bool{},
|
|
// The old broker keeps its other clients on this installation, and saying so is how the plan
|
|
// stops reading as a retirement.
|
|
}
|
|
|
|
address, _, err := broker.OnNATS()
|
|
if err != nil {
|
|
return state, err
|
|
}
|
|
state.TheBus = address
|
|
if address != "" {
|
|
// One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about
|
|
// to move onto a server that is not there should hear it here rather than afterwards.
|
|
//
|
|
// **Dialled the way the mesh dials it** — credential and pin — because a bare connect to a
|
|
// bus that requires TLS and a user fails at the handshake, and the check then reported a
|
|
// standing server as absent (seen live, 2026-09-28).
|
|
if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
|
|
state.ServerStanding = true
|
|
js.Close()
|
|
}
|
|
}
|
|
|
|
nodes, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
return state, err
|
|
}
|
|
kept, err := inv.BusUsers(ctx)
|
|
if err != nil {
|
|
return state, err
|
|
}
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return state, err
|
|
}
|
|
|
|
for _, n := range nodes {
|
|
state.Nodes = append(state.Nodes, n.Name)
|
|
_, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()]
|
|
state.Credentialled[n.Name] = has
|
|
|
|
assigned, err := inv.Assigned(ctx, n.Name)
|
|
if err != nil {
|
|
return state, err
|
|
}
|
|
for _, module := range assigned {
|
|
m, known := shelf[module]
|
|
if !known {
|
|
continue
|
|
}
|
|
// The machine that holds the bus seat is the one that would be sent the user list.
|
|
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
|
state.Holder = n.Name
|
|
state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
|
|
}
|
|
// A module that never speaks needs no credential, so it is not counted as missing one.
|
|
if !speaksOnTheBus(m) {
|
|
continue
|
|
}
|
|
named := n.Name + "/" + module
|
|
state.Modules = append(state.Modules, named)
|
|
_, hasOne := kept[broker.Principal{
|
|
Kind: broker.KindModule, Node: n.Name, Module: module,
|
|
}.Username()]
|
|
state.ModuleCredentialled[named] = hasOne
|
|
}
|
|
}
|
|
return state, nil
|
|
}
|
|
|
|
// speaksOnTheBus says whether a module reaches the bus at all.
|
|
//
|
|
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
|
// would bury the ones that matter under a list nobody can act on.
|
|
func speaksOnTheBus(m catalogue.Manifest) bool {
|
|
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
|
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
|
}
|
|
|
|
// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
|
|
// list became part of one.
|
|
//
|
|
// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
|
|
// has still been sent it, and this question is about whether the mesh did its part.
|
|
func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
|
|
digest, err := inv.Outstanding(ctx, node)
|
|
return err == nil && strings.TrimSpace(digest) != ""
|
|
}
|
|
|
|
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
|
|
// printing. The reasoning itself is the broker package's, where it is pure.
|
|
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
|
|
|
|
// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
|
|
// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
|
|
// sealed into its declaration, a module's as its broker secret, the control plane's own as its
|
|
// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
|
|
//
|
|
// **Before anything moves, and it is what makes moving possible.** A machine moved without a
|
|
// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
|
|
// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
|
|
// the module that provides it is assigned, rather than read from this process's environment: this
|
|
// process is still on the old bus when this runs, and must be.
|
|
func rolloutMint(ctx context.Context, again bool) error {
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
inv := open.inventory
|
|
|
|
known, err := broker.FromEnvironment()
|
|
if err != nil {
|
|
return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
|
|
"must carry its fingerprint: %w", err)
|
|
}
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entries, err := inv.Catalogued(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var busNode, controllerNode string
|
|
for _, e := range entries {
|
|
switch {
|
|
case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
|
|
busNode = e.On[0]
|
|
case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
|
|
controllerNode = e.On[0]
|
|
}
|
|
}
|
|
if busNode == "" {
|
|
return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
|
|
"bus to mint credentials for — register and assign it first")
|
|
}
|
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
busHost := onNetwork[busNode]
|
|
if busHost == "" {
|
|
// **The hub is not in that map.** The machine that took over the tunnel is where the current
|
|
// bus already answers, and every machine dials it at the address the mesh handed them — so
|
|
// when the new bus runs on the same machine, that address is the one to tell them, with the
|
|
// new port. Found live: the control node is the hub, and the map lists the machines placed
|
|
// around it.
|
|
// The host alone: no scheme (BareAddress adds one where none was, which is the wrong
|
|
// direction here — every URL built below adds its own) and no port.
|
|
_, _, host := broker.CredentialIn(known.Address)
|
|
if host == "" {
|
|
host = known.Address
|
|
}
|
|
if _, after, hasScheme := strings.Cut(host, "://"); hasScheme {
|
|
host = after
|
|
}
|
|
host = strings.TrimSpace(host)
|
|
if i := strings.LastIndex(host, ":"); i > 0 && !strings.Contains(host[i:], "]") {
|
|
host = host[:i]
|
|
}
|
|
if host == "" {
|
|
return fmt.Errorf("%s runs the new bus and has no address on the private network, and the "+
|
|
"current bus's address is unknown too, so no machine could be told where it is", busNode)
|
|
}
|
|
busHost = host
|
|
}
|
|
busAddress := busHost + ":4222"
|
|
|
|
records, err := inv.BusRecords(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
users, err := broker.Users(records)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
kept, err := inv.BusUsers(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
hashes := make(map[string]string, len(kept))
|
|
for name, u := range kept {
|
|
hashes[name] = u.PasswordHash
|
|
}
|
|
_, missing := broker.WithPasswords(users, hashes)
|
|
wanted := map[string]bool{}
|
|
for _, m := range missing {
|
|
wanted[m] = true
|
|
}
|
|
|
|
var machines, modules, skipped int
|
|
for _, p := range users {
|
|
if !again && !wanted[p.Username()] {
|
|
continue
|
|
}
|
|
switch p.Kind {
|
|
case broker.KindController:
|
|
if controllerNode == "" {
|
|
return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
|
|
}
|
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
url := "nats://" + p.Username() + ":" + password + "@" + busAddress
|
|
if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
|
|
return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
|
|
}
|
|
fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)
|
|
|
|
case broker.KindNode:
|
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
membership, _ := json.Marshal(map[string]string{
|
|
"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
|
|
})
|
|
key, err := inv.SealingKeyOf(ctx, p.Node)
|
|
if err != nil {
|
|
return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
|
|
}
|
|
sealed, err := secrets.Seal(key, membership)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
|
|
return err
|
|
}
|
|
machines++
|
|
|
|
case broker.KindModule:
|
|
if p.Module == "mesh-controller" {
|
|
// The control plane is a module too, and its `broker` secret is the old bus's
|
|
// credential it is still using while this runs. Writing the new bus's blob there
|
|
// cut the mesh off from its own old bus mid-move (2026-09-28). Its new-bus credential
|
|
// is the controller principal's `bus` secret above; nothing else is needed here.
|
|
skipped++
|
|
continue
|
|
}
|
|
m, inShelf := shelf[p.Module]
|
|
if !inShelf {
|
|
skipped++
|
|
continue
|
|
}
|
|
if _, reads := m.OwnSecrets["broker"]; !reads {
|
|
fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
|
|
skipped++
|
|
continue
|
|
}
|
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusModule, Node: p.Node, Module: p.Module})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
|
|
return err
|
|
}
|
|
modules++
|
|
|
|
default:
|
|
skipped++
|
|
}
|
|
}
|
|
fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
|
|
machines, modules, skipped, busAddress)
|
|
fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;")
|
|
fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it")
|
|
return nil
|
|
}
|
|
|
|
// providesBus is whether a manifest provides the mesh's bus.
|
|
func providesBus(m catalogue.Manifest) bool {
|
|
for _, o := range m.Provides {
|
|
if o.Name == "mesh-bus" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// rolloutHand mints a machine its credential for the new bus afresh and prints its membership
|
|
// once, for an operator to carry by hand — the rescue for a machine that cannot be reached over
|
|
// any bus: rotated while it still held the old password, or reachable only by ssh. The plaintext
|
|
// exists on this terminal and then only where it is written; the store keeps the hash, and the
|
|
// sealed copy in the machine's declaration is replaced too, so the next push says the same.
|
|
func rolloutHand(ctx context.Context, node string) error {
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
inv := open.inventory
|
|
|
|
known, err := broker.FromEnvironment()
|
|
if err != nil {
|
|
return fmt.Errorf("the bus's certificate is not known to this process: %w", err)
|
|
}
|
|
busAddress, _, err := broker.OnNATS()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if busAddress == "" {
|
|
return errors.New("this control plane is not on the new bus, so there is no membership to hand out")
|
|
}
|
|
_, _, bare := broker.CredentialIn(busAddress)
|
|
if _, after, has := strings.Cut(bare, "://"); has {
|
|
bare = after
|
|
}
|
|
if _, err := inv.NodeByName(ctx, node); err != nil {
|
|
return err
|
|
}
|
|
p := broker.Principal{Kind: broker.KindNode, Node: node}
|
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: node})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
membership, _ := json.Marshal(map[string]string{
|
|
"broker": bare, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
|
|
})
|
|
key, err := inv.SealingKeyOf(ctx, node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sealed, err := secrets.Seal(key, membership)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := inv.PutBusMembership(ctx, node, sealed); err != nil {
|
|
return err
|
|
}
|
|
// The one line of output is the membership itself, so it can be piped to the machine without
|
|
// being read on the way. Everything else goes to stderr.
|
|
fmt.Fprintf(os.Stderr, "%s's credential is minted afresh. Write this to %s on it and restart its host; "+
|
|
"then push the machine running the bus so the user list carries the new hash.\n",
|
|
node, catalogue.BusMembershipPath)
|
|
fmt.Println(string(membership))
|
|
return nil
|
|
}
|