From dded086b548b86354f1bc6b875c08964a187427c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 17:59:01 +0200 Subject: [PATCH] `rollout check`: whether this mesh could move its bus, and what is missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rollout moves every node at once, so there is nothing to inspect afterwards and no half to roll back — either the mesh was ready or it was not. That makes the readiness question the valuable half: it costs nothing, it can be asked of a mesh that is serving as many times as you like, and every answer is a thing somebody can go and fix. It reads from records and dials once. Is a bus answering, does a machine hold the seat, has that machine been sent the composed user list, does every machine have a credential for the new bus, does every module that speaks. Each missing thing names its own next step, because "not ready" that cannot be acted on is not an answer — and this is read at the point where the next step is irreversible. **A machine with no credential is the one that must stop it.** It keeps running and cannot come back, and afterwards there is no bus to tell it anything over, so the remedy has to happen first. The message says so. A module that never reaches the bus is not counted as missing a credential. A third of the catalogue never speaks, and listing those would bury the ones that matter. `rollout --confirm` refuses and says why: the move is not being written before its check has been run against a real mesh. And the plan it prints says the old broker stays — it remains an ordinary provider of `amqp` for whatever else uses it, which on this installation is a whole automation layer that has nothing to do with the mesh. This move is not its retirement, and that is why it is survivable: what breaks if it goes wrong is the mesh's ability to change things, not the services its modules serve. --- cmd/mesh-controller/main.go | 2 + cmd/mesh-controller/rollout.go | 193 ++++++++++++++++++++++++++++ cmd/mesh-controller/rollout_test.go | 93 ++++++++++++++ internal/broker/onnats.go | 13 ++ internal/broker/readiness.go | 142 ++++++++++++++++++++ internal/broker/readiness_test.go | 97 ++++++++++++++ 6 files changed, 540 insertions(+) create mode 100644 cmd/mesh-controller/rollout.go create mode 100644 cmd/mesh-controller/rollout_test.go create mode 100644 internal/broker/readiness.go create mode 100644 internal/broker/readiness_test.go diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 3bcf2c1..0727b34 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -114,6 +114,8 @@ func run() error { return planCommand(ctx, args[1:]) case "push": return pushCommand(ctx, args[1:]) + case "rollout": + return rolloutCommand(ctx, args[1:]) case "seats": return seatsCommand(ctx, args[1:]) case "status": diff --git a/cmd/mesh-controller/rollout.go b/cmd/mesh-controller/rollout.go new file mode 100644 index 0000000..4e4eb82 --- /dev/null +++ b/cmd/mesh-controller/rollout.go @@ -0,0 +1,193 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5). +// +// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both +// transports and every one of them chooses by a single fact; this is the step that flips it. That +// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays +// bounded until here — and it means the useful work is almost all in the checking. +// +// So `rollout check` is the command that matters and the one that can be run any number of times +// against a mesh that is serving. It answers from records: what is missing, and what would happen. +// `rollout` itself refuses unless the check is clean. +// +// **The old broker is not switched off by this.** It stays an ordinary provider of `amqp` for whatever +// else uses it — on this installation, a whole automation layer that has nothing to do with the mesh +// ([ADR 0119](../../02-DECISIONS/0119-amqp-is-a-provision-not-the-bus.md)). Only the mesh's own +// traffic moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's +// ability to change things, not the services its modules are serving. + +const rolloutUsage = "rollout check | rollout --confirm" + +func rolloutCommand(ctx context.Context, args []string) error { + switch { + case len(args) == 1 && args[0] == "check": + return rolloutCheck(ctx) + case len(args) == 1 && args[0] == "--confirm": + return errors.New( + "the rollout itself is not built yet: `rollout check` answers whether it could run, and " + + "what is missing. Moving every node at once is the one step with nothing to inspect " + + "afterwards, so it is not being written before the check it depends on has been run " + + "against a real mesh") + default: + return errors.New(rolloutUsage) + } +} + +// rolloutCheck says whether the mesh could move, and what would happen if it did. +func rolloutCheck(ctx context.Context) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + state, err := readinessOf(ctx, inv) + if err != nil { + return err + } + + fmt.Println("the bus this mesh would move to") + if state.TheBus == "" { + fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar) + } else { + standing := "not answering" + if state.ServerStanding { + standing = "answering" + } + fmt.Printf(" %s — %s\n", state.TheBus, standing) + } + fmt.Println() + + fmt.Println("what would move") + for _, step := range broker.WhatMoves(state) { + fmt.Printf(" %s\n", step) + } + fmt.Println() + + why := notReadyOf(state) + if len(why) == 0 { + fmt.Println("nothing is missing: this mesh could move its bus.") + fmt.Println() + fmt.Println("Read `what would move` above once more before running it. Every node moves at the") + fmt.Println("same moment and there is no half-moved state to look at afterwards.") + return nil + } + fmt.Printf("not ready — %d thing(s) to do first:\n", len(why)) + for i, w := range why { + fmt.Printf(" %d. %s\n", i+1, w) + } + return nil +} + +// readinessOf gathers what the mesh knows about its own ability to move. +// +// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the +// point: the answer is only useful if it can be had without committing to anything. +func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) { + state := broker.Readiness{ + Credentialled: map[string]bool{}, + ModuleCredentialled: map[string]bool{}, + // The old broker keeps its other clients on this installation, and saying so is how the plan + // stops reading as a retirement. + OldBusHasOtherClients: true, + } + + address, _, err := broker.OnNATS() + if err != nil { + return state, err + } + state.TheBus = address + if address != "" { + // One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about + // to move onto a server that is not there should hear it here rather than afterwards. + if conn, err := nats.Connect(broker.BareAddress(address), nats.Timeout(5*time.Second)); err == nil { + state.ServerStanding = true + conn.Close() + } + } + + nodes, err := inv.Nodes(ctx) + if err != nil { + return state, err + } + kept, err := inv.BusUsers(ctx) + if err != nil { + return state, err + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + return state, err + } + + for _, n := range nodes { + state.Nodes = append(state.Nodes, n.Name) + _, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()] + state.Credentialled[n.Name] = has + + assigned, err := inv.Assigned(ctx, n.Name) + if err != nil { + return state, err + } + for _, module := range assigned { + m, known := shelf[module] + if !known { + continue + } + // The machine that holds the bus seat is the one that would be sent the user list. + if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") { + state.Holder = n.Name + state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name) + } + // A module that never speaks needs no credential, so it is not counted as missing one. + if !speaksOnTheBus(m) { + continue + } + named := n.Name + "/" + module + state.Modules = append(state.Modules, named) + _, hasOne := kept[broker.Principal{ + Kind: broker.KindModule, Node: n.Name, Module: module, + }.Username()] + state.ModuleCredentialled[named] = hasOne + } + } + return state, nil +} + +// speaksOnTheBus says whether a module reaches the bus at all. +// +// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential +// would bury the ones that matter under a list nobody can act on. +func speaksOnTheBus(m catalogue.Manifest) bool { + return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 || + len(m.Seats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0 +} + +// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user +// list became part of one. +// +// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away +// has still been sent it, and this question is about whether the mesh did its part. +func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool { + digest, err := inv.Outstanding(ctx, node) + return err == nil && strings.TrimSpace(digest) != "" +} + +// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's +// printing. The reasoning itself is the broker package's, where it is pure. +func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) } diff --git a/cmd/mesh-controller/rollout_test.go b/cmd/mesh-controller/rollout_test.go new file mode 100644 index 0000000..d5a1264 --- /dev/null +++ b/cmd/mesh-controller/rollout_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Whether a mesh could move its bus, read from a real store. +// +// The readiness reasoning has its own tests; this is about the gathering — that the question is +// answered from what the mesh actually holds, on a store with machines and modules in it, without +// writing anything. + +func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) { + inv := inventory.ForTest(t) + ctx := context.Background() + + // A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a + // module that never does. + for _, m := range []catalogue.Manifest{ + {Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf", + Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}, + {Module: "gitea", Version: "1", Tools: []string{"repo_create"}}, + {Module: "wallpaper", Version: "1"}, + } { + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil { + t.Fatal(err) + } + } + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} { + if err := inv.Assign(ctx, a[0], a[1]); err != nil { + t.Fatal(err) + } + } + + state, err := readinessOf(ctx, inv) + if err != nil { + t.Fatal(err) + } + + if state.Holder != "anchor" { + t.Errorf("the machine holding the bus reads as %q", state.Holder) + } + if len(state.Nodes) != 2 { + t.Errorf("machines read as %v", state.Nodes) + } + // **A module that never speaks is not counted as missing a credential.** A third of the catalogue + // never reaches the bus, and listing those would bury the ones that matter. + for _, m := range state.Modules { + if strings.HasSuffix(m, "/wallpaper") { + t.Errorf("a module that never speaks was counted: %v", state.Modules) + } + } + if len(state.Modules) != 2 { + t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool", + state.Modules) + } + + // Nothing has been minted, so it is not ready — and it says so about each machine by name. + why := notReadyOf(state) + if len(why) == 0 { + t.Fatal("a mesh where nothing has a credential was reported ready to move") + } + said := strings.Join(why, "\n") + for _, name := range []string{"anchor", "laptop"} { + if !strings.Contains(said, name) { + t.Errorf("the refusal does not name %s: %s", name, said) + } + } + + // Mint for one machine and it drops out of the complaint, which is how somebody works through it. + if _, err := inv.MintBusPassword(ctx, inventory.BusUser{ + Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop", + }); err != nil { + t.Fatal(err) + } + state, err = readinessOf(ctx, inv) + if err != nil { + t.Fatal(err) + } + if !state.Credentialled["laptop"] { + t.Error("a machine that was minted a credential still reads as having none") + } +} diff --git a/internal/broker/onnats.go b/internal/broker/onnats.go index 12d81ed..245f429 100644 --- a/internal/broker/onnats.go +++ b/internal/broker/onnats.go @@ -55,6 +55,19 @@ func CredentialIn(address string) (user, password, bare string) { return user, password, scheme + address[at+1:] } +// BareAddress is a bus address with any credential stripped, for something that only needs to know +// whether a server is answering there. +func BareAddress(address string) string { + _, _, bare := CredentialIn(address) + if bare == "" { + return address + } + if strings.Contains(bare, "://") { + return bare + } + return "nats://" + bare +} + // MustBeOneBus refuses a configuration that names both buses for the mesh's own traffic. // // **Both clients ship and that is the point; both being live is not.** The rollout moves every node diff --git a/internal/broker/readiness.go b/internal/broker/readiness.go new file mode 100644 index 0000000..a1302d3 --- /dev/null +++ b/internal/broker/readiness.go @@ -0,0 +1,142 @@ +package broker + +import ( + "fmt" + "sort" + "strings" +) + +// Whether a mesh could move its bus, and what is missing if not. +// +// **Asked before anything moves, and answerable from records alone.** The rollout moves every node at +// once (novox/hq ADR 0116 step 5), so there is no partial state to inspect afterwards and no half to +// roll back: either the mesh was ready or it was not. That makes a readiness question the most +// valuable thing here — it costs nothing, it can be asked of a running mesh any number of times, and +// every answer is a thing somebody can go and fix. +// +// Deliberately pure. It is handed what the mesh knows and returns sentences; nothing here connects to +// anything, so it can be asked on a workstation about a mesh it has never reached. + +// Readiness is what the mesh knows about its own ability to move. +type Readiness struct { + // TheBus is the address the mesh's own traffic would move to, empty when nothing names one. + TheBus string + // ServerStanding is whether a bus is reachable at that address, as somebody checked. + ServerStanding bool + // Holder is the node running the module that holds the bus seat, empty when nothing does. + Holder string + // AccountsComposed is whether that node has been sent the composed user list. + AccountsComposed bool + // Nodes is every machine the mesh knows. + Nodes []string + // Credentialled is which of them has a credential for the new bus. + Credentialled map[string]bool + // Modules is every assigned module, as `/`. + Modules []string + // ModuleCredentialled is which of those has one. + ModuleCredentialled map[string]bool + // StillOnTheOldBus is whether anything of the mesh's own still needs the bus it is leaving — + // which is not a reason to stop, because that broker stays as an ordinary provider of `amqp` + // (ADR 0119). Recorded so nobody reads the move as a retirement. + OldBusHasOtherClients bool +} + +// NotReady is every reason this mesh cannot move its bus yet, in the order somebody would fix them. +// +// Empty means ready. **Each entry names one thing and what to do about it**, because a readiness check +// that says "not ready" is a check nobody can act on — and this is read at the point where the next +// step is irreversible. +func NotReady(r Readiness) []string { + var why []string + + if strings.TrimSpace(r.TheBus) == "" { + why = append(why, "nothing names the bus to move to: set "+NATSVar+" on the control node "+ + "to the address the new server answers on") + } + if !r.ServerStanding { + why = append(why, "no bus is answering at that address. Step 2 of the change raises it beside "+ + "the one the mesh is on, carrying nothing — assign the module that holds "+ + "mesh-broker and push the machine that runs it") + } + if r.Holder == "" { + why = append(why, "no machine holds mesh-broker, so nothing would compose the bus's user "+ + "list. Assign the module that claims it") + } else if !r.AccountsComposed { + why = append(why, fmt.Sprintf( + "%s holds mesh-broker and has not been sent the composed user list, so the bus would "+ + "refuse every connection. `push %s`", r.Holder, r.Holder)) + } + + // A node with no credential cannot come back after the move, and a node that cannot come back is + // a machine the mesh has lost until somebody goes to it. + var missing []string + for _, n := range r.Nodes { + if !r.Credentialled[n] { + missing = append(missing, n) + } + } + sort.Strings(missing) + if len(missing) > 0 { + why = append(why, fmt.Sprintf( + "%d machine(s) have no credential for the new bus and would not come back: %s. Each needs "+ + "one minted before the move, not after — after, there is no bus to ask over", + len(missing), strings.Join(missing, ", "))) + } + + // A module without one keeps running and stops being reachable, which is a smaller fault and still + // one somebody should choose rather than discover. + var quiet []string + for _, m := range r.Modules { + if !r.ModuleCredentialled[m] { + quiet = append(quiet, m) + } + } + sort.Strings(quiet) + if len(quiet) > 0 { + why = append(why, fmt.Sprintf( + "%d module(s) have no credential for the new bus: %s. Each keeps serving and stops "+ + "answering tools and hearing events until it is issued one", + len(quiet), strings.Join(quiet, ", "))) + } + + return why +} + +// WhatMoves is what the rollout would do, in order, for somebody reading before they commit. +// +// **Written out rather than summarised.** This is the one step with nothing to inspect afterwards, so +// the last useful moment to disagree with it is while reading this. +func WhatMoves(r Readiness) []string { + out := []string{ + fmt.Sprintf("compose the bus's user list and send it to %s", holderOr(r.Holder)), + fmt.Sprintf("move this control plane to %s, and confirm it is heard", busOr(r.TheBus)), + } + nodes := append([]string(nil), r.Nodes...) + sort.Strings(nodes) + for _, n := range nodes { + out = append(out, fmt.Sprintf("move %s, and confirm it reports", n)) + } + if len(r.Modules) > 0 { + out = append(out, fmt.Sprintf("move %d module runtime(s), and confirm each answers", + len(r.Modules))) + } + if r.OldBusHasOtherClients { + out = append(out, "leave the old broker running: it stays an ordinary provider of `amqp` for "+ + "whatever else uses it (ADR 0119), and this move is not its retirement") + } + return out +} + +func holderOr(node string) string { + if node == "" { + return "whichever machine holds mesh-broker" + } + return node +} + +func busOr(address string) string { + if address == "" { + return "the new bus" + } + return address +} diff --git a/internal/broker/readiness_test.go b/internal/broker/readiness_test.go new file mode 100644 index 0000000..827b786 --- /dev/null +++ b/internal/broker/readiness_test.go @@ -0,0 +1,97 @@ +package broker + +import ( + "strings" + "testing" +) + +// Whether a mesh could move its bus. +// +// Every case here is a way of moving that leaves something behind, and the one that matters most is a +// machine with no credential: after the move there is no bus to ask it over, so it is lost until +// somebody walks to it. + +func aMeshReadyToMove() Readiness { + return Readiness{ + TheBus: "nats://127.0.0.1:5671", ServerStanding: true, + Holder: "anchor", AccountsComposed: true, + Nodes: []string{"anchor", "laptop"}, + Credentialled: map[string]bool{"anchor": true, "laptop": true}, + Modules: []string{"anchor/gitea"}, + ModuleCredentialled: map[string]bool{"anchor/gitea": true}, + } +} + +func TestAMeshWithEverythingInPlaceIsReady(t *testing.T) { + if why := NotReady(aMeshReadyToMove()); len(why) != 0 { + t.Fatalf("a mesh with everything in place was refused: %v", why) + } +} + +// **A machine with no credential is the one that must stop this.** It keeps running and cannot come +// back, and there is no bus left to tell it anything over — so the remedy has to happen before, and +// the message says so. +func TestAMachineWithNoCredentialStopsTheMove(t *testing.T) { + r := aMeshReadyToMove() + r.Credentialled = map[string]bool{"anchor": true} + + why := NotReady(r) + if len(why) == 0 { + t.Fatal("a machine that could not come back did not stop the move") + } + said := strings.Join(why, "\n") + if !strings.Contains(said, "laptop") { + t.Errorf("the refusal does not name the machine: %s", said) + } + if !strings.Contains(said, "before the move") { + t.Errorf("the refusal does not say the remedy comes first: %s", said) + } +} + +// A bus nobody has raised, a seat nobody holds, and a user list nobody has been sent: each stops it, +// and each names its own next step, because "not ready" that cannot be acted on is not an answer. +func TestEachThingMissingNamesItsOwnRemedy(t *testing.T) { + for _, c := range []struct { + what string + break_ func(*Readiness) + says string + }{ + {"no address", func(r *Readiness) { r.TheBus = "" }, NATSVar}, + {"no server", func(r *Readiness) { r.ServerStanding = false }, "carrying nothing"}, + {"no holder", func(r *Readiness) { r.Holder = "" }, "mesh-broker"}, + {"no user list", func(r *Readiness) { r.AccountsComposed = false }, "push anchor"}, + {"a module with none", func(r *Readiness) { + r.ModuleCredentialled = map[string]bool{} + }, "anchor/gitea"}, + } { + r := aMeshReadyToMove() + c.break_(&r) + why := NotReady(r) + if len(why) == 0 { + t.Errorf("%s did not stop the move", c.what) + continue + } + if !strings.Contains(strings.Join(why, "\n"), c.says) { + t.Errorf("%s: the refusal does not mention %q: %v", c.what, c.says, why) + } + } +} + +// What the move would do is written out rather than summarised, because this is the one step with +// nothing to inspect afterwards — so reading it is the last chance to disagree. +func TestWhatMovesNamesEveryMachineAndSaysTheOldBrokerStays(t *testing.T) { + r := aMeshReadyToMove() + r.OldBusHasOtherClients = true + steps := strings.Join(WhatMoves(r), "\n") + + for _, want := range []string{"anchor", "laptop", "user list", "module runtime"} { + if !strings.Contains(steps, want) { + t.Errorf("the plan does not mention %q:\n%s", want, steps) + } + } + // Said explicitly, so nobody reads the move as switching the old broker off — it stays serving + // whatever else uses it, and that is a decision already taken. + if !strings.Contains(steps, "not its retirement") { + t.Errorf("the plan does not say the old broker stays:\n%s", steps) + } +}