Adopt genesis's bus through the bus's planned step, its streams snapshotted first (hq issue 590, ADR 0333 §3)

The broker step sent the nats module with a plain push, and the controller refuses a push that would
replace the bus: that is a planned step, bus upgrade, which snapshots the streams first and checks
them after (ADR 0236). Genesis stopped at step 18.

The step now snapshots every stream with the module's own snapshot program (no backup holder exists
yet to be asked), asks bus upgrade with that snapshot and why, says the one restart, and waits until
the bus runs the module's image, answers, and the controller ends the step done. The bundle's bus runs
the release the module's image is built on (2.11.17, by the same digest, where it ran 2.10.29), so the
step changes no server release and is said reversible.

internal/bootstrap/bus_adoption.go

@@ -0,0 +1,179 @@
+package bootstrap
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// Adopting the bus is the mesh's planned step for replacing it (novox/hq ADR 0333 §3, ADR 0236).
+//
+// The adoption replaces the bus's image once: the module's built image takes the place of the upstream
+// one the bundle raised, on the same streams, users and certificate. The controller refuses a push that
+// would replace the bus — "a planned step: `bus upgrade --why …` snapshots its streams first and checks
+// them after" — so genesis stopped at step 18 (issue 590). The installer takes that step: it snapshots the
+// streams itself, because at genesis no backup holder exists to be asked, and hands the snapshot to
+// `bus upgrade`, which sends the machine, opens `bus-maintenance`, and ends the step done only once the
+// bus answers healthy again — every stream, every durable consumer, a round trip (H-bus). The installer
+// waits for that end.
+
+// adoptionWhy is what the step is recorded as, for the record of hand acts and the bus's step.
+const adoptionWhy = "genesis adopts its bus as the nats module (ADR 0333)"
+
+// Where the installer keeps the streams' snapshot before the adoption, and the credential the snapshot
+// program reads while it runs. Beside genesis's own credentials; variables so a test writes elsewhere.
+var (
+	busSnapshotFile       = "/var/lib/mesh-genesis/bus-before-adoption.tar"
+	busSnapshotCredential = "/var/lib/mesh-genesis/bus-snapshot.credential"
+	busCertificateFile    = "/var/lib/mesh-broker-tls/tls.crt"
+	busControllerSecret   = BusControllerFile
+	// busStepWithin bounds the wait for the controller to say the step ended: its own bound (15 min,
+	// mesh-controller busStepBound), and its doctor's next look after that.
+	busStepWithin = 17 * time.Minute
+)
+
+// adoptTheBus sends the nats module to this machine through the bus's planned step. A bus already on the
+// module's image is the module's: it is sent like any module, and nothing is replaced.
+func adoptTheBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container,
+	say func(string)) error {
+
+	if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image {
+		say("  already adopted   " + bus.Name + " — it runs the module's image")
+		_, err := pushNode(ctx, o, control, say)
+		return err
+	}
+	image, err := theModulesBusImage(ctx, o, control)
+	if err != nil {
+		return err
+	}
+	if err := snapshotTheBus(ctx, o, control, image, say); err != nil {
+		return err
+	}
+	say("  restarting        " + bus.Name + " — once, onto the " + BusModule + " module's image, as the bus's " +
+		"planned step; its streams, users and certificate stay where they are (novox/hq ADR 0333 §3, ADR 0236)")
+	if _, err := control.within(2*o.Wait).tell(ctx, "bus", "upgrade", "--why", adoptionWhy,
+		"--reversible", "--snapshot-taken", busSnapshotFile); err != nil {
+		return fmt.Errorf("the bus's planned step was refused, so the bus is not replaced: %w", err)
+	}
+	return nil
+}
+
+// meshBuiltImage is an image the mesh built, as it records it: `artifact-store://<module>/<artifact>@sha256:…`.
+var meshBuiltImage = regexp.MustCompile(`artifact-store://([a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9-]*@sha256:[0-9a-f]{64})`)
+
+// theModulesBusImage is the nats module's server image, as the build just made it, pulled from this
+// mesh's registry: the image that carries the snapshot program.
+func theModulesBusImage(ctx context.Context, o Options, control controlPlane) (string, error) {
+	said, err := control.tell(ctx, "builds", BusModule, "-n", "1")
+	if err != nil {
+		return "", err
+	}
+	for _, m := range meshBuiltImage.FindAllStringSubmatch(said, -1) {
+		if strings.HasPrefix(m[1], BusModule+"/") {
+			return strings.TrimSuffix(o.Registry, "/") + "/" + m[1], nil
+		}
+	}
+	return "", fmt.Errorf("the mesh says no image it built for the %s module, so its snapshot program cannot "+
+		"be run before the bus is replaced:\n%s", BusModule, indent(strings.TrimSpace(said)))
+}
+
+// snapshotTheBus writes every stream of the running bus to busSnapshotFile with the module's own snapshot
+// program (mesh-catalog modules/nats, ADR 0235), as the controller's account, pinned to the bus's
+// certificate. The credential is written for the program's one run and removed after it.
+func snapshotTheBus(ctx context.Context, o Options, control controlPlane, image string, say func(string)) error {
+	password, err := readCredentialFile(busControllerSecret)
+	if err != nil {
+		return fmt.Errorf("the controller's bus credential is not at %s, so the streams cannot be snapshotted "+
+			"before the bus is replaced: %w", busControllerSecret, err)
+	}
+	fingerprint, err := certificateFingerprint(busCertificateFile)
+	if err != nil {
+		return err
+	}
+	port := o.Ports.orDefaults().Bus
+	server := "tls://127.0.0.1:" + strconv.Itoa(port)
+	credential, err := json.Marshal(map[string]string{"url": server, "fingerprint": fingerprint,
+		"user": busControllerUser, "password": password})
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(busSnapshotCredential, credential, 0o600); err != nil {
+		return fmt.Errorf("the snapshot's credential could not be written: %w", err)
+	}
+	defer os.Remove(busSnapshotCredential)
+
+	script := "docker run --rm -i --network host --entrypoint /usr/local/bin/mesh-nats-snapshot " + image +
+		" snapshot --server " + server + " --credential - < " + busSnapshotCredential + " > " + busSnapshotFile +
+		".partial && mv " + busSnapshotFile + ".partial " + busSnapshotFile
+	running, cancel := context.WithTimeout(ctx, busStepWithin)
+	defer cancel()
+	if out, err := control.run(running, "sh", "-c", script); err != nil {
+		_, _ = control.run(ctx, "rm", "-f", busSnapshotFile+".partial")
+		return fmt.Errorf("the bus's streams could not be snapshotted, so the bus is not replaced: %w\n%s",
+			err, indent(strings.TrimSpace(out)))
+	}
+	say("  snapshotted       every stream of the bus, at " + busSnapshotFile + " (its module's snapshot program)")
+	return nil
+}
+
+// certificateFingerprint is the pin of the bus's certificate: sha256 of its DER, as every machine pins it.
+func certificateFingerprint(path string) (string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("the bus's certificate is not at %s, so it cannot be pinned for the snapshot: %w", path, err)
+	}
+	block, _ := pem.Decode(raw)
+	if block == nil {
+		return "", fmt.Errorf("%s holds no PEM certificate", path)
+	}
+	sum := sha256.Sum256(block.Bytes)
+	return "sha256:" + hex.EncodeToString(sum[:]), nil
+}
+
+// waitForTheBusStep waits until the controller says the bus's planned step ended: done once the bus
+// answered healthy — every stream, every durable consumer, a round trip — or failed, with its snapshot as
+// the way back. The step's own bound is the controller's; this waits a little past it.
+func waitForTheBusStep(ctx context.Context, o Options, control controlPlane, say func(string)) error {
+	deadline := time.Now().Add(busStepWithin)
+	var last string
+	for {
+		said, err := control.tell(ctx, "bus")
+		if err == nil {
+			for _, line := range strings.Split(said, "\n") {
+				if strings.HasPrefix(strings.TrimSpace(line), "last step ") {
+					last = strings.TrimSpace(line)
+				}
+			}
+			switch {
+			case strings.Contains(last, ": done at "):
+				say("  bus step done     " + last)
+				return nil
+			case strings.Contains(last, ": failed at "):
+				return fmt.Errorf("the bus's planned step ended failed: %s\n%s", last, indent(strings.TrimSpace(said)))
+			}
+		}
+		if !time.Now().Before(deadline) {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(answerEvery):
+		}
+	}
+	if last == "" {
+		return errors.New("the controller says no bus step after the adoption was sent; `mesh-controller bus` says how it stands")
+	}
+	return fmt.Errorf("the bus's planned step did not end within %s: %s", busStepWithin, last)
+}

internal/bootstrap/bus_adoption_test.go

@@ -0,0 +1,135 @@
+package bootstrap
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Defends novox/hq issue 590 at its broker step (ADR 0333 §3, ADR 0236): the adoption replaces the bus's
+// image, and the mesh replaces its bus only as a planned step. A plain push was refused — "sending anchor
+// would replace the bus … a planned step: `bus upgrade --why …`" — and genesis stopped at step 18.
+
+func TestTheBrokerStepAdoptsTheBusThroughItsPlannedStep(t *testing.T) {
+	catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
+	rewritten, err := Rewrite(theRealBundle(t), held)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image
+	upgraded := false
+	var credential []byte
+	runtime := &asked{}
+	agrees := aMeshThatAgrees(nil)
+	runtime.answer = func(name string, args []string) (string, error) {
+		joined := strings.Join(args, " ")
+		switch {
+		case name == "sh" && strings.Contains(joined, "mesh-nats-snapshot"):
+			credential, _ = os.ReadFile(busSnapshotCredential) // as the program reads it
+		case strings.Contains(joined, "bus upgrade --why"):
+			upgraded = true
+		case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"):
+			if upgraded {
+				return theModulesImage + "\n", nil
+			}
+			return bundleImage + "\n", nil
+		}
+		return agrees(name, args)
+	}
+	_, fetch := aBusThatAdopts()
+	o := installing(t, catalogue)
+	o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
+	o.Wait = 5 * time.Second
+	said, err := busStepWith(t, o, runtime, fetch)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	snapshot, upgrade := -1, -1
+	for i, c := range runtime.commands {
+		switch {
+		case strings.HasPrefix(c, "sh -c") && strings.Contains(c, "mesh-nats-snapshot"):
+			snapshot = i
+			for _, want := range []string{"127.0.0.1:5000/nats/server@sha256:" + strings.Repeat("4", 64),
+				"snapshot --server tls://127.0.0.1:5671 --credential - < " + busSnapshotCredential,
+				"> " + busSnapshotFile + ".partial && mv " + busSnapshotFile + ".partial " + busSnapshotFile} {
+				if !strings.Contains(c, want) {
+					t.Errorf("the snapshot is not taken with %q: %s", want, c)
+				}
+			}
+		case strings.Contains(c, "/mesh-controller bus upgrade"):
+			upgrade = i
+			if want := "bus upgrade --why " + adoptionWhy + " --reversible --snapshot-taken " + busSnapshotFile; !strings.HasSuffix(c, want) {
+				t.Errorf("the planned step is asked as %q; want it to end %q", c, want)
+			}
+		case strings.HasSuffix(c, "/mesh-controller push anchor"):
+			t.Errorf("the bus's module was sent by a plain push, which the mesh refuses for the bus: %v", runtime.commands)
+		}
+	}
+	if snapshot < 0 || upgrade < 0 || snapshot > upgrade {
+		t.Fatalf("the streams were not snapshotted before the bus's planned step (snapshot %d, upgrade %d): %v",
+			snapshot, upgrade, runtime.commands)
+	}
+	var cred map[string]string
+	if err := json.Unmarshal(credential, &cred); err != nil {
+		t.Fatalf("the snapshot program was given no credential: %v", err)
+	}
+	fingerprint, _ := certificateFingerprint(busCertificateFile)
+	if cred["user"] != "controller" || cred["password"] != "BUS-PW" || cred["fingerprint"] != fingerprint ||
+		cred["url"] != "tls://127.0.0.1:5671" {
+		t.Errorf("the snapshot's credential is %v", cred)
+	}
+	if _, err := os.Stat(busSnapshotCredential); !os.IsNotExist(err) {
+		t.Error("the snapshot's credential was left on the machine")
+	}
+	text := strings.Join(said, "\n")
+	for _, want := range []string{"snapshotted       every stream", "restarting        mesh-broker-nats — once",
+		"bus answers       mesh-broker-nats", "bus step done     last step 1", "adopted           mesh-broker-nats"} {
+		if !strings.Contains(text, want) {
+			t.Errorf("the step did not say %q:\n%s", want, text)
+		}
+	}
+}
+
+// A planned step the controller ends failed is the broker step failing, with the controller's words.
+func TestABusStepThatEndsFailedIsRefused(t *testing.T) {
+	aGenesisBusOnDisk(t)
+	runtime := &asked{answer: func(name string, args []string) (string, error) {
+		return "last step 1: not known → 53e51864 on anchor by installer (x): failed at 2026-10-13 11:59; snapshot s\n" +
+			"  stream KV_x has no messages\n", nil
+	}}
+	control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
+	err := waitForTheBusStep(context.Background(), Options{}, control, quietly)
+	if err == nil || !strings.Contains(err.Error(), "failed at") || !strings.Contains(err.Error(), "KV_x") {
+		t.Fatalf("a failed bus step was not refused with its words: %v", err)
+	}
+}
+
+// The bundle's bus runs the release the module's image is built on, pinned by the same digest, so the
+// adoption changes no server release and is reversible as the step says (ADR 0333 §3).
+func TestTheBundlesBusRunsTheReleaseTheModulesImageIsBuiltOn(t *testing.T) {
+	var m struct {
+		Build struct {
+			On []struct{ Arg, Image string } `json:"on"`
+		} `json:"build"`
+	}
+	if err := json.Unmarshal(theCataloguesModule(t, BusModule), &m); err != nil {
+		t.Fatal(err)
+	}
+	base := ""
+	for _, on := range m.Build.On {
+		if on.Arg == "NATS_BASE" {
+			base = on.Image
+		}
+	}
+	rewritten, err := Rewrite(theRealBundle(t), held)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image; got != base {
+		t.Errorf("the bundle raises the bus on %s; the nats module's image is built on %s", got, base)
+	}
+}

internal/bootstrap/phase3.go

@@ -93,7 +93,8 @@ func InstallStore(ctx context.Context, o Options, control controlPlane,
 // be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker
 // and the vault each need it or need the shape, and three copies of it drifted.
 func installProvider(ctx context.Context, o Options, control controlPlane, module string,
-	manifest []byte, buildNote string, beforePush func() error, say func(string)) error {
+	manifest []byte, buildNote string, beforePush func() error, say func(string),
+	send ...func() error) error {
 
 	remote := "/" + module + "-module.json"
 	if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil {
@@ -132,6 +133,11 @@ func installProvider(ctx context.Context, o Options, control controlPlane, modul
 			return err
 		}
 	}
+	// A module whose first send is not a plain push says how it is sent: the bus, whose replacement is a
+	// planned step (ADR 0236), is sent by that step (adoptTheBus).
+	if len(send) > 0 && send[0] != nil {
+		return send[0]()
+	}
 	_, err := pushNode(ctx, o, control, say)
 	return err
 }
@@ -176,9 +182,10 @@ const BusModule = "nats"
 // **The image is the one thing that changes, once** (ADR 0333 §3). The store's module pins the upstream
 // postgres the foundation runs; the bus's module builds its server image (it carries the mesh's snapshot
 // tool), so no pinned reference can be equal to the foundation's. The applier recreates the container on
-// the module's image, on the same streams, users and certificate. This says so before the push and waits
-// after it until the container runs the module's image and the bus answers, so nothing after this step
-// dials a bus that is restarting.
+// the module's image, on the same streams, users and certificate. The mesh replaces its bus only as a
+// planned step (ADR 0236), so the module is sent by that step: its streams snapshotted first, the restart
+// said, and the installer waits until the container runs the module's image, the bus answers, and the
+// controller ends the step done (bus_adoption.go), so nothing after this step dials a bus that is restarting.
 func InstallBroker(ctx context.Context, o Options, control controlPlane,
 	foundation *declaration.Declaration, fetch func(context.Context, string) (int, string, error),
 	say func(string)) error {
@@ -197,20 +204,15 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane,
 	}
 	say("  adopting          " + bus.Name + " — the bus the foundation raised, its streams kept")
 	if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)",
-		func() error {
-			if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image {
-				say("  already restarted " + bus.Name + " — it runs the module's image")
-				return nil
-			}
-			say("  restarting        " + bus.Name + " — once, onto the " + module + " module's image; its " +
-				"streams, users and certificate stay where they are (novox/hq ADR 0333 §3)")
-			return nil
-		}, say); err != nil {
+		nil, say, func() error { return adoptTheBus(ctx, o, control, bus, say) }); err != nil {
 		return err
 	}
 	if err := waitForTheAdoptedBus(ctx, o, control, bus, fetch, say); err != nil {
 		return err
 	}
+	if err := waitForTheBusStep(ctx, o, control, say); err != nil {
+		return err
+	}
 	say("  adopted           " + bus.Name + " — the foundation's bus is now the " + module +
 		" module, and its user list is the module's from here on")
 	return nil

internal/bootstrap/phase3_test.go

@@ -69,6 +69,7 @@ func aBusThatAdopts() (*asked, func(context.Context, string) (int, string, error
 func busStepWith(t *testing.T, o Options, runtime *asked,
 	fetch func(context.Context, string) (int, string, error)) ([]string, error) {
 	t.Helper()
+	aGenesisBusOnDisk(t)
 	rewritten, err := Rewrite(theRealBundle(t), held)
 	if err != nil {
 		t.Fatal(err)
@@ -222,7 +223,7 @@ func TestTheBrokerStepSaysTheRestartAndWaitsForTheBusToAnswer(t *testing.T) {
 	runtime.answer = func(name string, args []string) (string, error) {
 		joined := strings.Join(args, " ")
 		switch {
-		case strings.Contains(joined, "push anchor"):
+		case strings.Contains(joined, "bus upgrade --why"):
 			pushed = true
 		case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"):
 			if !pushed {
@@ -304,7 +305,7 @@ func TestARerunAfterTheAdoptionSaysTheBusIsTheModulesAlready(t *testing.T) {
 		t.Fatal(err)
 	}
 	text := strings.Join(said, "\n")
-	if strings.Contains(text, "restarting") || !strings.Contains(text, "already restarted mesh-broker-nats") {
+	if strings.Contains(text, "restarting") || !strings.Contains(text, "already adopted   mesh-broker-nats") {
 		t.Errorf("a re-run announced a restart, or did not say the bus is the module's already:\n%s", text)
 	}
 }
@@ -417,3 +418,30 @@ func TestTheBundlesBusFilesAreSeeds(t *testing.T) {
 		}
 	}
 }
+
+// aGenesisBusOnDisk puts what the adoption reads on the machine in a test's own directory: the bus's
+// certificate, the controller's bus credential, and where the snapshot goes. The step's wait is short.
+func aGenesisBusOnDisk(t *testing.T) {
+	t.Helper()
+	dir := t.TempDir()
+	cert := filepath.Join(dir, "tls.crt")
+	if err := os.WriteFile(cert, []byte(aCertificatePEM), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	secret := filepath.Join(dir, "bus-controller.secret")
+	if err := os.WriteFile(secret, []byte("BUS-PW"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	was := []string{busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret}
+	wasWithin := busStepWithin
+	busSnapshotFile, busSnapshotCredential = filepath.Join(dir, "bus.tar"), filepath.Join(dir, "bus.credential")
+	busCertificateFile, busControllerSecret = cert, secret
+	busStepWithin = 50 * time.Millisecond
+	t.Cleanup(func() {
+		busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret = was[0], was[1], was[2], was[3]
+		busStepWithin = wasWithin
+	})
+}
+
+// aCertificatePEM is a PEM block standing for the bus's certificate; its pin is the sha256 of these bytes.
+const aCertificatePEM = "-----BEGIN CERTIFICATE-----\nAAECAwQFBgcICQ==\n-----END CERTIFICATE-----\n"

internal/bootstrap/registry_test.go

@@ -60,6 +60,14 @@ func aMeshThatAgrees(answers map[string]string) func(string, []string) (string,
 		switch {
 		case name == "sh" && len(args) == 2 && strings.Contains(args[1], "accounts.conf"):
 			return "", nil // the bus's users placed after a push (issue 608)
+		case name == "sh" && len(args) == 2 && strings.Contains(args[1], "mesh-nats-snapshot"):
+			return "", nil // the streams snapshotted before the bus's planned step (issue 590)
+		case name == "docker" && strings.HasSuffix(joined, "/mesh-controller bus"):
+			return "the bus is nats, built 53e51864, on anchor\nlast step 1: not known → 53e51864 on anchor by installer (" +
+				adoptionWhy + "): done at 2026-10-13 11:40; snapshot " + busSnapshotFile + "\n", nil
+		case name == "docker" && strings.Contains(joined, "/mesh-controller builds nats -n 1"):
+			return "nats  built 53e51864 anchor\n    image      artifact-store://nats/server@sha256:" +
+				strings.Repeat("4", 64) + "\n    bundle     artifact-store://nats/tools/blobs/sha256:" + strings.Repeat("5", 64) + "\n", nil
 		case name != "docker":
 			return "", fmt.Errorf("unexpected program %q", name)
 		case strings.HasSuffix(joined, "broker accounts"):