cmd/mesh-controller/main.go

@@ -114,6 +114,8 @@ func run() error {
 		return planCommand(ctx, args[1:])
 	case "push":
 		return pushCommand(ctx, args[1:])
+	case "rollout":
+		return rolloutCommand(ctx, args[1:])
 	case "seats":
 		return seatsCommand(ctx, args[1:])
 	case "status":

cmd/mesh-controller/rollout.go

@@ -0,0 +1,193 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
+//
+// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both
+// transports and every one of them chooses by a single fact; this is the step that flips it. That
+// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays
+// bounded until here — and it means the useful work is almost all in the checking.
+//
+// So `rollout check` is the command that matters and the one that can be run any number of times
+// against a mesh that is serving. It answers from records: what is missing, and what would happen.
+// `rollout` itself refuses unless the check is clean.
+//
+// **The old broker is not switched off by this.** It stays an ordinary provider of `amqp` for whatever
+// else uses it — on this installation, a whole automation layer that has nothing to do with the mesh
+// ([ADR 0119](../../02-DECISIONS/0119-amqp-is-a-provision-not-the-bus.md)). Only the mesh's own
+// traffic moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's
+// ability to change things, not the services its modules are serving.
+
+const rolloutUsage = "rollout check | rollout --confirm"
+
+func rolloutCommand(ctx context.Context, args []string) error {
+	switch {
+	case len(args) == 1 && args[0] == "check":
+		return rolloutCheck(ctx)
+	case len(args) == 1 && args[0] == "--confirm":
+		return errors.New(
+			"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
+				"what is missing. Moving every node at once is the one step with nothing to inspect " +
+				"afterwards, so it is not being written before the check it depends on has been run " +
+				"against a real mesh")
+	default:
+		return errors.New(rolloutUsage)
+	}
+}
+
+// rolloutCheck says whether the mesh could move, and what would happen if it did.
+func rolloutCheck(ctx context.Context) error {
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+
+	state, err := readinessOf(ctx, inv)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("the bus this mesh would move to")
+	if state.TheBus == "" {
+		fmt.Printf("  nothing names one (%s is unset)\n", broker.NATSVar)
+	} else {
+		standing := "not answering"
+		if state.ServerStanding {
+			standing = "answering"
+		}
+		fmt.Printf("  %s — %s\n", state.TheBus, standing)
+	}
+	fmt.Println()
+
+	fmt.Println("what would move")
+	for _, step := range broker.WhatMoves(state) {
+		fmt.Printf("  %s\n", step)
+	}
+	fmt.Println()
+
+	why := notReadyOf(state)
+	if len(why) == 0 {
+		fmt.Println("nothing is missing: this mesh could move its bus.")
+		fmt.Println()
+		fmt.Println("Read `what would move` above once more before running it. Every node moves at the")
+		fmt.Println("same moment and there is no half-moved state to look at afterwards.")
+		return nil
+	}
+	fmt.Printf("not ready — %d thing(s) to do first:\n", len(why))
+	for i, w := range why {
+		fmt.Printf("  %d. %s\n", i+1, w)
+	}
+	return nil
+}
+
+// readinessOf gathers what the mesh knows about its own ability to move.
+//
+// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the
+// point: the answer is only useful if it can be had without committing to anything.
+func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) {
+	state := broker.Readiness{
+		Credentialled:       map[string]bool{},
+		ModuleCredentialled: map[string]bool{},
+		// The old broker keeps its other clients on this installation, and saying so is how the plan
+		// stops reading as a retirement.
+		OldBusHasOtherClients: true,
+	}
+
+	address, _, err := broker.OnNATS()
+	if err != nil {
+		return state, err
+	}
+	state.TheBus = address
+	if address != "" {
+		// One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about
+		// to move onto a server that is not there should hear it here rather than afterwards.
+		if conn, err := nats.Connect(broker.BareAddress(address), nats.Timeout(5*time.Second)); err == nil {
+			state.ServerStanding = true
+			conn.Close()
+		}
+	}
+
+	nodes, err := inv.Nodes(ctx)
+	if err != nil {
+		return state, err
+	}
+	kept, err := inv.BusUsers(ctx)
+	if err != nil {
+		return state, err
+	}
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return state, err
+	}
+
+	for _, n := range nodes {
+		state.Nodes = append(state.Nodes, n.Name)
+		_, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()]
+		state.Credentialled[n.Name] = has
+
+		assigned, err := inv.Assigned(ctx, n.Name)
+		if err != nil {
+			return state, err
+		}
+		for _, module := range assigned {
+			m, known := shelf[module]
+			if !known {
+				continue
+			}
+			// The machine that holds the bus seat is the one that would be sent the user list.
+			if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
+				state.Holder = n.Name
+				state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
+			}
+			// A module that never speaks needs no credential, so it is not counted as missing one.
+			if !speaksOnTheBus(m) {
+				continue
+			}
+			named := n.Name + "/" + module
+			state.Modules = append(state.Modules, named)
+			_, hasOne := kept[broker.Principal{
+				Kind: broker.KindModule, Node: n.Name, Module: module,
+			}.Username()]
+			state.ModuleCredentialled[named] = hasOne
+		}
+	}
+	return state, nil
+}
+
+// speaksOnTheBus says whether a module reaches the bus at all.
+//
+// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
+// would bury the ones that matter under a list nobody can act on.
+func speaksOnTheBus(m catalogue.Manifest) bool {
+	return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
+		len(m.Seats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
+}
+
+// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
+// list became part of one.
+//
+// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
+// has still been sent it, and this question is about whether the mesh did its part.
+func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
+	digest, err := inv.Outstanding(ctx, node)
+	return err == nil && strings.TrimSpace(digest) != ""
+}
+
+// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
+// printing. The reasoning itself is the broker package's, where it is pure.
+func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }

cmd/mesh-controller/rollout_test.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// Whether a mesh could move its bus, read from a real store.
+//
+// The readiness reasoning has its own tests; this is about the gathering — that the question is
+// answered from what the mesh actually holds, on a store with machines and modules in it, without
+// writing anything.
+
+func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) {
+	inv := inventory.ForTest(t)
+	ctx := context.Background()
+
+	// A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a
+	// module that never does.
+	for _, m := range []catalogue.Manifest{
+		{Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf",
+			Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}},
+		{Module: "gitea", Version: "1", Tools: []string{"repo_create"}},
+		{Module: "wallpaper", Version: "1"},
+	} {
+		if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, n := range []string{"anchor", "laptop"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} {
+		if err := inv.Assign(ctx, a[0], a[1]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	state, err := readinessOf(ctx, inv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if state.Holder != "anchor" {
+		t.Errorf("the machine holding the bus reads as %q", state.Holder)
+	}
+	if len(state.Nodes) != 2 {
+		t.Errorf("machines read as %v", state.Nodes)
+	}
+	// **A module that never speaks is not counted as missing a credential.** A third of the catalogue
+	// never reaches the bus, and listing those would bury the ones that matter.
+	for _, m := range state.Modules {
+		if strings.HasSuffix(m, "/wallpaper") {
+			t.Errorf("a module that never speaks was counted: %v", state.Modules)
+		}
+	}
+	if len(state.Modules) != 2 {
+		t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool",
+			state.Modules)
+	}
+
+	// Nothing has been minted, so it is not ready — and it says so about each machine by name.
+	why := notReadyOf(state)
+	if len(why) == 0 {
+		t.Fatal("a mesh where nothing has a credential was reported ready to move")
+	}
+	said := strings.Join(why, "\n")
+	for _, name := range []string{"anchor", "laptop"} {
+		if !strings.Contains(said, name) {
+			t.Errorf("the refusal does not name %s: %s", name, said)
+		}
+	}
+
+	// Mint for one machine and it drops out of the complaint, which is how somebody works through it.
+	if _, err := inv.MintBusPassword(ctx, inventory.BusUser{
+		Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	state, err = readinessOf(ctx, inv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !state.Credentialled["laptop"] {
+		t.Error("a machine that was minted a credential still reads as having none")
+	}
+}

internal/broker/onnats.go

@@ -55,6 +55,19 @@ func CredentialIn(address string) (user, password, bare string) {
 	return user, password, scheme + address[at+1:]
 }
 
+// BareAddress is a bus address with any credential stripped, for something that only needs to know
+// whether a server is answering there.
+func BareAddress(address string) string {
+	_, _, bare := CredentialIn(address)
+	if bare == "" {
+		return address
+	}
+	if strings.Contains(bare, "://") {
+		return bare
+	}
+	return "nats://" + bare
+}
+
 // MustBeOneBus refuses a configuration that names both buses for the mesh's own traffic.
 //
 // **Both clients ship and that is the point; both being live is not.** The rollout moves every node

internal/broker/readiness.go

@@ -0,0 +1,142 @@
+package broker
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// Whether a mesh could move its bus, and what is missing if not.
+//
+// **Asked before anything moves, and answerable from records alone.** The rollout moves every node at
+// once (novox/hq ADR 0116 step 5), so there is no partial state to inspect afterwards and no half to
+// roll back: either the mesh was ready or it was not. That makes a readiness question the most
+// valuable thing here — it costs nothing, it can be asked of a running mesh any number of times, and
+// every answer is a thing somebody can go and fix.
+//
+// Deliberately pure. It is handed what the mesh knows and returns sentences; nothing here connects to
+// anything, so it can be asked on a workstation about a mesh it has never reached.
+
+// Readiness is what the mesh knows about its own ability to move.
+type Readiness struct {
+	// TheBus is the address the mesh's own traffic would move to, empty when nothing names one.
+	TheBus string
+	// ServerStanding is whether a bus is reachable at that address, as somebody checked.
+	ServerStanding bool
+	// Holder is the node running the module that holds the bus seat, empty when nothing does.
+	Holder string
+	// AccountsComposed is whether that node has been sent the composed user list.
+	AccountsComposed bool
+	// Nodes is every machine the mesh knows.
+	Nodes []string
+	// Credentialled is which of them has a credential for the new bus.
+	Credentialled map[string]bool
+	// Modules is every assigned module, as `<node>/<module>`.
+	Modules []string
+	// ModuleCredentialled is which of those has one.
+	ModuleCredentialled map[string]bool
+	// StillOnTheOldBus is whether anything of the mesh's own still needs the bus it is leaving —
+	// which is not a reason to stop, because that broker stays as an ordinary provider of `amqp`
+	// (ADR 0119). Recorded so nobody reads the move as a retirement.
+	OldBusHasOtherClients bool
+}
+
+// NotReady is every reason this mesh cannot move its bus yet, in the order somebody would fix them.
+//
+// Empty means ready. **Each entry names one thing and what to do about it**, because a readiness check
+// that says "not ready" is a check nobody can act on — and this is read at the point where the next
+// step is irreversible.
+func NotReady(r Readiness) []string {
+	var why []string
+
+	if strings.TrimSpace(r.TheBus) == "" {
+		why = append(why, "nothing names the bus to move to: set "+NATSVar+" on the control node "+
+			"to the address the new server answers on")
+	}
+	if !r.ServerStanding {
+		why = append(why, "no bus is answering at that address. Step 2 of the change raises it beside "+
+			"the one the mesh is on, carrying nothing — assign the module that holds "+
+			"mesh-broker and push the machine that runs it")
+	}
+	if r.Holder == "" {
+		why = append(why, "no machine holds mesh-broker, so nothing would compose the bus's user "+
+			"list. Assign the module that claims it")
+	} else if !r.AccountsComposed {
+		why = append(why, fmt.Sprintf(
+			"%s holds mesh-broker and has not been sent the composed user list, so the bus would "+
+				"refuse every connection. `push %s`", r.Holder, r.Holder))
+	}
+
+	// A node with no credential cannot come back after the move, and a node that cannot come back is
+	// a machine the mesh has lost until somebody goes to it.
+	var missing []string
+	for _, n := range r.Nodes {
+		if !r.Credentialled[n] {
+			missing = append(missing, n)
+		}
+	}
+	sort.Strings(missing)
+	if len(missing) > 0 {
+		why = append(why, fmt.Sprintf(
+			"%d machine(s) have no credential for the new bus and would not come back: %s. Each needs "+
+				"one minted before the move, not after — after, there is no bus to ask over",
+			len(missing), strings.Join(missing, ", ")))
+	}
+
+	// A module without one keeps running and stops being reachable, which is a smaller fault and still
+	// one somebody should choose rather than discover.
+	var quiet []string
+	for _, m := range r.Modules {
+		if !r.ModuleCredentialled[m] {
+			quiet = append(quiet, m)
+		}
+	}
+	sort.Strings(quiet)
+	if len(quiet) > 0 {
+		why = append(why, fmt.Sprintf(
+			"%d module(s) have no credential for the new bus: %s. Each keeps serving and stops "+
+				"answering tools and hearing events until it is issued one",
+			len(quiet), strings.Join(quiet, ", ")))
+	}
+
+	return why
+}
+
+// WhatMoves is what the rollout would do, in order, for somebody reading before they commit.
+//
+// **Written out rather than summarised.** This is the one step with nothing to inspect afterwards, so
+// the last useful moment to disagree with it is while reading this.
+func WhatMoves(r Readiness) []string {
+	out := []string{
+		fmt.Sprintf("compose the bus's user list and send it to %s", holderOr(r.Holder)),
+		fmt.Sprintf("move this control plane to %s, and confirm it is heard", busOr(r.TheBus)),
+	}
+	nodes := append([]string(nil), r.Nodes...)
+	sort.Strings(nodes)
+	for _, n := range nodes {
+		out = append(out, fmt.Sprintf("move %s, and confirm it reports", n))
+	}
+	if len(r.Modules) > 0 {
+		out = append(out, fmt.Sprintf("move %d module runtime(s), and confirm each answers",
+			len(r.Modules)))
+	}
+	if r.OldBusHasOtherClients {
+		out = append(out, "leave the old broker running: it stays an ordinary provider of `amqp` for "+
+			"whatever else uses it (ADR 0119), and this move is not its retirement")
+	}
+	return out
+}
+
+func holderOr(node string) string {
+	if node == "" {
+		return "whichever machine holds mesh-broker"
+	}
+	return node
+}
+
+func busOr(address string) string {
+	if address == "" {
+		return "the new bus"
+	}
+	return address
+}

internal/broker/readiness_test.go

@@ -0,0 +1,97 @@
+package broker
+
+import (
+	"strings"
+	"testing"
+)
+
+// Whether a mesh could move its bus.
+//
+// Every case here is a way of moving that leaves something behind, and the one that matters most is a
+// machine with no credential: after the move there is no bus to ask it over, so it is lost until
+// somebody walks to it.
+
+func aMeshReadyToMove() Readiness {
+	return Readiness{
+		TheBus: "nats://127.0.0.1:5671", ServerStanding: true,
+		Holder: "anchor", AccountsComposed: true,
+		Nodes:               []string{"anchor", "laptop"},
+		Credentialled:       map[string]bool{"anchor": true, "laptop": true},
+		Modules:             []string{"anchor/gitea"},
+		ModuleCredentialled: map[string]bool{"anchor/gitea": true},
+	}
+}
+
+func TestAMeshWithEverythingInPlaceIsReady(t *testing.T) {
+	if why := NotReady(aMeshReadyToMove()); len(why) != 0 {
+		t.Fatalf("a mesh with everything in place was refused: %v", why)
+	}
+}
+
+// **A machine with no credential is the one that must stop this.** It keeps running and cannot come
+// back, and there is no bus left to tell it anything over — so the remedy has to happen before, and
+// the message says so.
+func TestAMachineWithNoCredentialStopsTheMove(t *testing.T) {
+	r := aMeshReadyToMove()
+	r.Credentialled = map[string]bool{"anchor": true}
+
+	why := NotReady(r)
+	if len(why) == 0 {
+		t.Fatal("a machine that could not come back did not stop the move")
+	}
+	said := strings.Join(why, "\n")
+	if !strings.Contains(said, "laptop") {
+		t.Errorf("the refusal does not name the machine: %s", said)
+	}
+	if !strings.Contains(said, "before the move") {
+		t.Errorf("the refusal does not say the remedy comes first: %s", said)
+	}
+}
+
+// A bus nobody has raised, a seat nobody holds, and a user list nobody has been sent: each stops it,
+// and each names its own next step, because "not ready" that cannot be acted on is not an answer.
+func TestEachThingMissingNamesItsOwnRemedy(t *testing.T) {
+	for _, c := range []struct {
+		what   string
+		break_ func(*Readiness)
+		says   string
+	}{
+		{"no address", func(r *Readiness) { r.TheBus = "" }, NATSVar},
+		{"no server", func(r *Readiness) { r.ServerStanding = false }, "carrying nothing"},
+		{"no holder", func(r *Readiness) { r.Holder = "" }, "mesh-broker"},
+		{"no user list", func(r *Readiness) { r.AccountsComposed = false }, "push anchor"},
+		{"a module with none", func(r *Readiness) {
+			r.ModuleCredentialled = map[string]bool{}
+		}, "anchor/gitea"},
+	} {
+		r := aMeshReadyToMove()
+		c.break_(&r)
+		why := NotReady(r)
+		if len(why) == 0 {
+			t.Errorf("%s did not stop the move", c.what)
+			continue
+		}
+		if !strings.Contains(strings.Join(why, "\n"), c.says) {
+			t.Errorf("%s: the refusal does not mention %q: %v", c.what, c.says, why)
+		}
+	}
+}
+
+// What the move would do is written out rather than summarised, because this is the one step with
+// nothing to inspect afterwards — so reading it is the last chance to disagree.
+func TestWhatMovesNamesEveryMachineAndSaysTheOldBrokerStays(t *testing.T) {
+	r := aMeshReadyToMove()
+	r.OldBusHasOtherClients = true
+	steps := strings.Join(WhatMoves(r), "\n")
+
+	for _, want := range []string{"anchor", "laptop", "user list", "module runtime"} {
+		if !strings.Contains(steps, want) {
+			t.Errorf("the plan does not mention %q:\n%s", want, steps)
+		}
+	}
+	// Said explicitly, so nobody reads the move as switching the old broker off — it stays serving
+	// whatever else uses it, and that is a decision already taken.
+	if !strings.Contains(steps, "not its retirement") {
+		t.Errorf("the plan does not say the old broker stays:\n%s", steps)
+	}
+}