rollout check: whether this mesh could move its bus, and what is missing

The rollout moves every node at once, so there is nothing to inspect afterwards and no
half to roll back — either the mesh was ready or it was not. That makes the readiness
question the valuable half: it costs nothing, it can be asked of a mesh that is serving as
many times as you like, and every answer is a thing somebody can go and fix.

It reads from records and dials once. Is a bus answering, does a machine hold the seat, has
that machine been sent the composed user list, does every machine have a credential for the
new bus, does every module that speaks. Each missing thing names its own next step, because
"not ready" that cannot be acted on is not an answer — and this is read at the point where
the next step is irreversible.

**A machine with no credential is the one that must stop it.** It keeps running and cannot
come back, and afterwards there is no bus to tell it anything over, so the remedy has to
happen first. The message says so.

A module that never reaches the bus is not counted as missing a credential. A third of the
catalogue never speaks, and listing those would bury the ones that matter.

`rollout --confirm` refuses and says why: the move is not being written before its check has
been run against a real mesh. And the plan it prints says the old broker stays — it remains
an ordinary provider of `amqp` for whatever else uses it, which on this installation is a
whole automation layer that has nothing to do with the mesh. This move is not its retirement,
and that is why it is survivable: what breaks if it goes wrong is the mesh's ability to
change things, not the services its modules serve.

cmd/mesh-controller/rollout_test.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// Whether a mesh could move its bus, read from a real store.
+//
+// The readiness reasoning has its own tests; this is about the gathering — that the question is
+// answered from what the mesh actually holds, on a store with machines and modules in it, without
+// writing anything.
+
+func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) {
+	inv := inventory.ForTest(t)
+	ctx := context.Background()
+
+	// A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a
+	// module that never does.
+	for _, m := range []catalogue.Manifest{
+		{Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf",
+			Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}},
+		{Module: "gitea", Version: "1", Tools: []string{"repo_create"}},
+		{Module: "wallpaper", Version: "1"},
+	} {
+		if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, n := range []string{"anchor", "laptop"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} {
+		if err := inv.Assign(ctx, a[0], a[1]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	state, err := readinessOf(ctx, inv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if state.Holder != "anchor" {
+		t.Errorf("the machine holding the bus reads as %q", state.Holder)
+	}
+	if len(state.Nodes) != 2 {
+		t.Errorf("machines read as %v", state.Nodes)
+	}
+	// **A module that never speaks is not counted as missing a credential.** A third of the catalogue
+	// never reaches the bus, and listing those would bury the ones that matter.
+	for _, m := range state.Modules {
+		if strings.HasSuffix(m, "/wallpaper") {
+			t.Errorf("a module that never speaks was counted: %v", state.Modules)
+		}
+	}
+	if len(state.Modules) != 2 {
+		t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool",
+			state.Modules)
+	}
+
+	// Nothing has been minted, so it is not ready — and it says so about each machine by name.
+	why := notReadyOf(state)
+	if len(why) == 0 {
+		t.Fatal("a mesh where nothing has a credential was reported ready to move")
+	}
+	said := strings.Join(why, "\n")
+	for _, name := range []string{"anchor", "laptop"} {
+		if !strings.Contains(said, name) {
+			t.Errorf("the refusal does not name %s: %s", name, said)
+		}
+	}
+
+	// Mint for one machine and it drops out of the complaint, which is how somebody works through it.
+	if _, err := inv.MintBusPassword(ctx, inventory.BusUser{
+		Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	state, err = readinessOf(ctx, inv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !state.Credentialled["laptop"] {
+		t.Error("a machine that was minted a credential still reads as having none")
+	}
+}