rollout check: whether this mesh could move its bus, and what is missing

The rollout moves every node at once, so there is nothing to inspect afterwards and no
half to roll back — either the mesh was ready or it was not. That makes the readiness
question the valuable half: it costs nothing, it can be asked of a mesh that is serving as
many times as you like, and every answer is a thing somebody can go and fix.

It reads from records and dials once. Is a bus answering, does a machine hold the seat, has
that machine been sent the composed user list, does every machine have a credential for the
new bus, does every module that speaks. Each missing thing names its own next step, because
"not ready" that cannot be acted on is not an answer — and this is read at the point where
the next step is irreversible.

**A machine with no credential is the one that must stop it.** It keeps running and cannot
come back, and afterwards there is no bus to tell it anything over, so the remedy has to
happen first. The message says so.

A module that never reaches the bus is not counted as missing a credential. A third of the
catalogue never speaks, and listing those would bury the ones that matter.

`rollout --confirm` refuses and says why: the move is not being written before its check has
been run against a real mesh. And the plan it prints says the old broker stays — it remains
an ordinary provider of `amqp` for whatever else uses it, which on this installation is a
whole automation layer that has nothing to do with the mesh. This move is not its retirement,
and that is why it is survivable: what breaks if it goes wrong is the mesh's ability to
change things, not the services its modules serve.

cmd/mesh-controller/main.go

@@ -114,6 +114,8 @@ func run() error {
 		return planCommand(ctx, args[1:])
 	case "push":
 		return pushCommand(ctx, args[1:])
+	case "rollout":
+		return rolloutCommand(ctx, args[1:])
 	case "seats":
 		return seatsCommand(ctx, args[1:])
 	case "status":

cmd/mesh-controller/rollout.go

@@ -0,0 +1,193 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
+//
+// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both
+// transports and every one of them chooses by a single fact; this is the step that flips it. That
+// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays
+// bounded until here — and it means the useful work is almost all in the checking.
+//
+// So `rollout check` is the command that matters and the one that can be run any number of times
+// against a mesh that is serving. It answers from records: what is missing, and what would happen.
+// `rollout` itself refuses unless the check is clean.
+//
+// **The old broker is not switched off by this.** It stays an ordinary provider of `amqp` for whatever
+// else uses it — on this installation, a whole automation layer that has nothing to do with the mesh
+// ([ADR 0119](../../02-DECISIONS/0119-amqp-is-a-provision-not-the-bus.md)). Only the mesh's own
+// traffic moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's
+// ability to change things, not the services its modules are serving.
+
+const rolloutUsage = "rollout check | rollout --confirm"
+
+func rolloutCommand(ctx context.Context, args []string) error {
+	switch {
+	case len(args) == 1 && args[0] == "check":
+		return rolloutCheck(ctx)
+	case len(args) == 1 && args[0] == "--confirm":
+		return errors.New(
+			"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
+				"what is missing. Moving every node at once is the one step with nothing to inspect " +
+				"afterwards, so it is not being written before the check it depends on has been run " +
+				"against a real mesh")
+	default:
+		return errors.New(rolloutUsage)
+	}
+}
+
+// rolloutCheck says whether the mesh could move, and what would happen if it did.
+func rolloutCheck(ctx context.Context) error {
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+
+	state, err := readinessOf(ctx, inv)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("the bus this mesh would move to")
+	if state.TheBus == "" {
+		fmt.Printf("  nothing names one (%s is unset)\n", broker.NATSVar)
+	} else {
+		standing := "not answering"
+		if state.ServerStanding {
+			standing = "answering"
+		}
+		fmt.Printf("  %s — %s\n", state.TheBus, standing)
+	}
+	fmt.Println()
+
+	fmt.Println("what would move")
+	for _, step := range broker.WhatMoves(state) {
+		fmt.Printf("  %s\n", step)
+	}
+	fmt.Println()
+
+	why := notReadyOf(state)
+	if len(why) == 0 {
+		fmt.Println("nothing is missing: this mesh could move its bus.")
+		fmt.Println()
+		fmt.Println("Read `what would move` above once more before running it. Every node moves at the")
+		fmt.Println("same moment and there is no half-moved state to look at afterwards.")
+		return nil
+	}
+	fmt.Printf("not ready — %d thing(s) to do first:\n", len(why))
+	for i, w := range why {
+		fmt.Printf("  %d. %s\n", i+1, w)
+	}
+	return nil
+}
+
+// readinessOf gathers what the mesh knows about its own ability to move.
+//
+// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the
+// point: the answer is only useful if it can be had without committing to anything.
+func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) {
+	state := broker.Readiness{
+		Credentialled:       map[string]bool{},
+		ModuleCredentialled: map[string]bool{},
+		// The old broker keeps its other clients on this installation, and saying so is how the plan
+		// stops reading as a retirement.
+		OldBusHasOtherClients: true,
+	}
+
+	address, _, err := broker.OnNATS()
+	if err != nil {
+		return state, err
+	}
+	state.TheBus = address
+	if address != "" {
+		// One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about
+		// to move onto a server that is not there should hear it here rather than afterwards.
+		if conn, err := nats.Connect(broker.BareAddress(address), nats.Timeout(5*time.Second)); err == nil {
+			state.ServerStanding = true
+			conn.Close()
+		}
+	}
+
+	nodes, err := inv.Nodes(ctx)
+	if err != nil {
+		return state, err
+	}
+	kept, err := inv.BusUsers(ctx)
+	if err != nil {
+		return state, err
+	}
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return state, err
+	}
+
+	for _, n := range nodes {
+		state.Nodes = append(state.Nodes, n.Name)
+		_, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()]
+		state.Credentialled[n.Name] = has
+
+		assigned, err := inv.Assigned(ctx, n.Name)
+		if err != nil {
+			return state, err
+		}
+		for _, module := range assigned {
+			m, known := shelf[module]
+			if !known {
+				continue
+			}
+			// The machine that holds the bus seat is the one that would be sent the user list.
+			if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
+				state.Holder = n.Name
+				state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
+			}
+			// A module that never speaks needs no credential, so it is not counted as missing one.
+			if !speaksOnTheBus(m) {
+				continue
+			}
+			named := n.Name + "/" + module
+			state.Modules = append(state.Modules, named)
+			_, hasOne := kept[broker.Principal{
+				Kind: broker.KindModule, Node: n.Name, Module: module,
+			}.Username()]
+			state.ModuleCredentialled[named] = hasOne
+		}
+	}
+	return state, nil
+}
+
+// speaksOnTheBus says whether a module reaches the bus at all.
+//
+// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
+// would bury the ones that matter under a list nobody can act on.
+func speaksOnTheBus(m catalogue.Manifest) bool {
+	return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
+		len(m.Seats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
+}
+
+// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
+// list became part of one.
+//
+// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
+// has still been sent it, and this question is about whether the mesh did its part.
+func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
+	digest, err := inv.Outstanding(ctx, node)
+	return err == nil && strings.TrimSpace(digest) != ""
+}
+
+// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
+// printing. The reasoning itself is the broker package's, where it is pure.
+func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }

cmd/mesh-controller/rollout_test.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// Whether a mesh could move its bus, read from a real store.
+//
+// The readiness reasoning has its own tests; this is about the gathering — that the question is
+// answered from what the mesh actually holds, on a store with machines and modules in it, without
+// writing anything.
+
+func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) {
+	inv := inventory.ForTest(t)
+	ctx := context.Background()
+
+	// A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a
+	// module that never does.
+	for _, m := range []catalogue.Manifest{
+		{Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf",
+			Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}},
+		{Module: "gitea", Version: "1", Tools: []string{"repo_create"}},
+		{Module: "wallpaper", Version: "1"},
+	} {
+		if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, n := range []string{"anchor", "laptop"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} {
+		if err := inv.Assign(ctx, a[0], a[1]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	state, err := readinessOf(ctx, inv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if state.Holder != "anchor" {
+		t.Errorf("the machine holding the bus reads as %q", state.Holder)
+	}
+	if len(state.Nodes) != 2 {
+		t.Errorf("machines read as %v", state.Nodes)
+	}
+	// **A module that never speaks is not counted as missing a credential.** A third of the catalogue
+	// never reaches the bus, and listing those would bury the ones that matter.
+	for _, m := range state.Modules {
+		if strings.HasSuffix(m, "/wallpaper") {
+			t.Errorf("a module that never speaks was counted: %v", state.Modules)
+		}
+	}
+	if len(state.Modules) != 2 {
+		t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool",
+			state.Modules)
+	}
+
+	// Nothing has been minted, so it is not ready — and it says so about each machine by name.
+	why := notReadyOf(state)
+	if len(why) == 0 {
+		t.Fatal("a mesh where nothing has a credential was reported ready to move")
+	}
+	said := strings.Join(why, "\n")
+	for _, name := range []string{"anchor", "laptop"} {
+		if !strings.Contains(said, name) {
+			t.Errorf("the refusal does not name %s: %s", name, said)
+		}
+	}
+
+	// Mint for one machine and it drops out of the complaint, which is how somebody works through it.
+	if _, err := inv.MintBusPassword(ctx, inventory.BusUser{
+		Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	state, err = readinessOf(ctx, inv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !state.Credentialled["laptop"] {
+		t.Error("a machine that was minted a credential still reads as having none")
+	}
+}

internal/broker/onnats.go

@@ -55,6 +55,19 @@ func CredentialIn(address string) (user, password, bare string) {
 	return user, password, scheme + address[at+1:]
 }
 
+// BareAddress is a bus address with any credential stripped, for something that only needs to know
+// whether a server is answering there.
+func BareAddress(address string) string {
+	_, _, bare := CredentialIn(address)
+	if bare == "" {
+		return address
+	}
+	if strings.Contains(bare, "://") {
+		return bare
+	}
+	return "nats://" + bare
+}
+
 // MustBeOneBus refuses a configuration that names both buses for the mesh's own traffic.
 //
 // **Both clients ship and that is the point; both being live is not.** The rollout moves every node

internal/broker/readiness.go

@@ -0,0 +1,142 @@
+package broker
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// Whether a mesh could move its bus, and what is missing if not.
+//
+// **Asked before anything moves, and answerable from records alone.** The rollout moves every node at
+// once (novox/hq ADR 0116 step 5), so there is no partial state to inspect afterwards and no half to
+// roll back: either the mesh was ready or it was not. That makes a readiness question the most
+// valuable thing here — it costs nothing, it can be asked of a running mesh any number of times, and
+// every answer is a thing somebody can go and fix.
+//
+// Deliberately pure. It is handed what the mesh knows and returns sentences; nothing here connects to
+// anything, so it can be asked on a workstation about a mesh it has never reached.
+
+// Readiness is what the mesh knows about its own ability to move.
+type Readiness struct {
+	// TheBus is the address the mesh's own traffic would move to, empty when nothing names one.
+	TheBus string
+	// ServerStanding is whether a bus is reachable at that address, as somebody checked.
+	ServerStanding bool
+	// Holder is the node running the module that holds the bus seat, empty when nothing does.
+	Holder string
+	// AccountsComposed is whether that node has been sent the composed user list.
+	AccountsComposed bool
+	// Nodes is every machine the mesh knows.
+	Nodes []string
+	// Credentialled is which of them has a credential for the new bus.
+	Credentialled map[string]bool
+	// Modules is every assigned module, as `<node>/<module>`.
+	Modules []string
+	// ModuleCredentialled is which of those has one.
+	ModuleCredentialled map[string]bool
+	// StillOnTheOldBus is whether anything of the mesh's own still needs the bus it is leaving —
+	// which is not a reason to stop, because that broker stays as an ordinary provider of `amqp`
+	// (ADR 0119). Recorded so nobody reads the move as a retirement.
+	OldBusHasOtherClients bool
+}
+
+// NotReady is every reason this mesh cannot move its bus yet, in the order somebody would fix them.
+//
+// Empty means ready. **Each entry names one thing and what to do about it**, because a readiness check
+// that says "not ready" is a check nobody can act on — and this is read at the point where the next
+// step is irreversible.
+func NotReady(r Readiness) []string {
+	var why []string
+
+	if strings.TrimSpace(r.TheBus) == "" {
+		why = append(why, "nothing names the bus to move to: set "+NATSVar+" on the control node "+
+			"to the address the new server answers on")
+	}
+	if !r.ServerStanding {
+		why = append(why, "no bus is answering at that address. Step 2 of the change raises it beside "+
+			"the one the mesh is on, carrying nothing — assign the module that holds "+
+			"mesh-broker and push the machine that runs it")
+	}
+	if r.Holder == "" {
+		why = append(why, "no machine holds mesh-broker, so nothing would compose the bus's user "+
+			"list. Assign the module that claims it")
+	} else if !r.AccountsComposed {
+		why = append(why, fmt.Sprintf(
+			"%s holds mesh-broker and has not been sent the composed user list, so the bus would "+
+				"refuse every connection. `push %s`", r.Holder, r.Holder))
+	}
+
+	// A node with no credential cannot come back after the move, and a node that cannot come back is
+	// a machine the mesh has lost until somebody goes to it.
+	var missing []string
+	for _, n := range r.Nodes {
+		if !r.Credentialled[n] {
+			missing = append(missing, n)
+		}
+	}
+	sort.Strings(missing)
+	if len(missing) > 0 {
+		why = append(why, fmt.Sprintf(
+			"%d machine(s) have no credential for the new bus and would not come back: %s. Each needs "+
+				"one minted before the move, not after — after, there is no bus to ask over",
+			len(missing), strings.Join(missing, ", ")))
+	}
+
+	// A module without one keeps running and stops being reachable, which is a smaller fault and still
+	// one somebody should choose rather than discover.
+	var quiet []string
+	for _, m := range r.Modules {
+		if !r.ModuleCredentialled[m] {
+			quiet = append(quiet, m)
+		}
+	}
+	sort.Strings(quiet)
+	if len(quiet) > 0 {
+		why = append(why, fmt.Sprintf(
+			"%d module(s) have no credential for the new bus: %s. Each keeps serving and stops "+
+				"answering tools and hearing events until it is issued one",
+			len(quiet), strings.Join(quiet, ", ")))
+	}
+
+	return why
+}
+
+// WhatMoves is what the rollout would do, in order, for somebody reading before they commit.
+//
+// **Written out rather than summarised.** This is the one step with nothing to inspect afterwards, so
+// the last useful moment to disagree with it is while reading this.
+func WhatMoves(r Readiness) []string {
+	out := []string{
+		fmt.Sprintf("compose the bus's user list and send it to %s", holderOr(r.Holder)),
+		fmt.Sprintf("move this control plane to %s, and confirm it is heard", busOr(r.TheBus)),
+	}
+	nodes := append([]string(nil), r.Nodes...)
+	sort.Strings(nodes)
+	for _, n := range nodes {
+		out = append(out, fmt.Sprintf("move %s, and confirm it reports", n))
+	}
+	if len(r.Modules) > 0 {
+		out = append(out, fmt.Sprintf("move %d module runtime(s), and confirm each answers",
+			len(r.Modules)))
+	}
+	if r.OldBusHasOtherClients {
+		out = append(out, "leave the old broker running: it stays an ordinary provider of `amqp` for "+
+			"whatever else uses it (ADR 0119), and this move is not its retirement")
+	}
+	return out
+}
+
+func holderOr(node string) string {
+	if node == "" {
+		return "whichever machine holds mesh-broker"
+	}
+	return node
+}
+
+func busOr(address string) string {
+	if address == "" {
+		return "the new bus"
+	}
+	return address
+}

internal/broker/readiness_test.go

@@ -0,0 +1,97 @@
+package broker
+
+import (
+	"strings"
+	"testing"
+)
+
+// Whether a mesh could move its bus.
+//
+// Every case here is a way of moving that leaves something behind, and the one that matters most is a
+// machine with no credential: after the move there is no bus to ask it over, so it is lost until
+// somebody walks to it.
+
+func aMeshReadyToMove() Readiness {
+	return Readiness{
+		TheBus: "nats://127.0.0.1:5671", ServerStanding: true,
+		Holder: "anchor", AccountsComposed: true,
+		Nodes:               []string{"anchor", "laptop"},
+		Credentialled:       map[string]bool{"anchor": true, "laptop": true},
+		Modules:             []string{"anchor/gitea"},
+		ModuleCredentialled: map[string]bool{"anchor/gitea": true},
+	}
+}
+
+func TestAMeshWithEverythingInPlaceIsReady(t *testing.T) {
+	if why := NotReady(aMeshReadyToMove()); len(why) != 0 {
+		t.Fatalf("a mesh with everything in place was refused: %v", why)
+	}
+}
+
+// **A machine with no credential is the one that must stop this.** It keeps running and cannot come
+// back, and there is no bus left to tell it anything over — so the remedy has to happen before, and
+// the message says so.
+func TestAMachineWithNoCredentialStopsTheMove(t *testing.T) {
+	r := aMeshReadyToMove()
+	r.Credentialled = map[string]bool{"anchor": true}
+
+	why := NotReady(r)
+	if len(why) == 0 {
+		t.Fatal("a machine that could not come back did not stop the move")
+	}
+	said := strings.Join(why, "\n")
+	if !strings.Contains(said, "laptop") {
+		t.Errorf("the refusal does not name the machine: %s", said)
+	}
+	if !strings.Contains(said, "before the move") {
+		t.Errorf("the refusal does not say the remedy comes first: %s", said)
+	}
+}
+
+// A bus nobody has raised, a seat nobody holds, and a user list nobody has been sent: each stops it,
+// and each names its own next step, because "not ready" that cannot be acted on is not an answer.
+func TestEachThingMissingNamesItsOwnRemedy(t *testing.T) {
+	for _, c := range []struct {
+		what   string
+		break_ func(*Readiness)
+		says   string
+	}{
+		{"no address", func(r *Readiness) { r.TheBus = "" }, NATSVar},
+		{"no server", func(r *Readiness) { r.ServerStanding = false }, "carrying nothing"},
+		{"no holder", func(r *Readiness) { r.Holder = "" }, "mesh-broker"},
+		{"no user list", func(r *Readiness) { r.AccountsComposed = false }, "push anchor"},
+		{"a module with none", func(r *Readiness) {
+			r.ModuleCredentialled = map[string]bool{}
+		}, "anchor/gitea"},
+	} {
+		r := aMeshReadyToMove()
+		c.break_(&r)
+		why := NotReady(r)
+		if len(why) == 0 {
+			t.Errorf("%s did not stop the move", c.what)
+			continue
+		}
+		if !strings.Contains(strings.Join(why, "\n"), c.says) {
+			t.Errorf("%s: the refusal does not mention %q: %v", c.what, c.says, why)
+		}
+	}
+}
+
+// What the move would do is written out rather than summarised, because this is the one step with
+// nothing to inspect afterwards — so reading it is the last chance to disagree.
+func TestWhatMovesNamesEveryMachineAndSaysTheOldBrokerStays(t *testing.T) {
+	r := aMeshReadyToMove()
+	r.OldBusHasOtherClients = true
+	steps := strings.Join(WhatMoves(r), "\n")
+
+	for _, want := range []string{"anchor", "laptop", "user list", "module runtime"} {
+		if !strings.Contains(steps, want) {
+			t.Errorf("the plan does not mention %q:\n%s", want, steps)
+		}
+	}
+	// Said explicitly, so nobody reads the move as switching the old broker off — it stays serving
+	// whatever else uses it, and that is a decision already taken.
+	if !strings.Contains(steps, "not its retirement") {
+		t.Errorf("the plan does not say the old broker stays:\n%s", steps)
+	}
+}