diff --git a/examples/foundation-first-node.lock b/examples/foundation-first-node.lock index d58c038..9f59df6 100644 --- a/examples/foundation-first-node.lock +++ b/examples/foundation-first-node.lock @@ -224,14 +224,17 @@ "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" }, // The upstream server, until the broker step adopts it: the `nats` module's own image, built by the - // mesh, takes its place there once, on these same directories (ADR 0333 §3). The published port is the + // mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned + // step (ADR 0236). The release is the one the module's image is built on (its NATS_BASE, 2.11.17), + // pinned upstream by that same digest: the adoption changes no server release, so it can be undone + // by putting the old image back and is said reversible. The published port is the // node's bus port, 5671 unless given, which every enrolment token names; the installer gives the // module the setting that maps its declared 4222 to it (ADR 0333 §2). { "id": "broker", "type": "container", "name": "mesh-broker-nats", - "image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927", + "image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00", "ports": ["5671:4222", "127.0.0.1:8222:8222"], "volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"], "args": ["-c", "/etc/nats/nats.conf", "-js"] @@ -239,8 +242,8 @@ { "id": "broker-ready", "type": "action", - "command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"], - "verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"] + "command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"], + "verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671'"] }, { "id": "control-plane", diff --git a/internal/bootstrap/bus_adoption.go b/internal/bootstrap/bus_adoption.go new file mode 100644 index 0000000..4f26ea0 --- /dev/null +++ b/internal/bootstrap/bus_adoption.go @@ -0,0 +1,179 @@ +package bootstrap + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Adopting the bus is the mesh's planned step for replacing it (novox/hq ADR 0333 §3, ADR 0236). +// +// The adoption replaces the bus's image once: the module's built image takes the place of the upstream +// one the bundle raised, on the same streams, users and certificate. The controller refuses a push that +// would replace the bus — "a planned step: `bus upgrade --why …` snapshots its streams first and checks +// them after" — so genesis stopped at step 18 (issue 590). The installer takes that step: it snapshots the +// streams itself, because at genesis no backup holder exists to be asked, and hands the snapshot to +// `bus upgrade`, which sends the machine, opens `bus-maintenance`, and ends the step done only once the +// bus answers healthy again — every stream, every durable consumer, a round trip (H-bus). The installer +// waits for that end. + +// adoptionWhy is what the step is recorded as, for the record of hand acts and the bus's step. +const adoptionWhy = "genesis adopts its bus as the nats module (ADR 0333)" + +// Where the installer keeps the streams' snapshot before the adoption, and the credential the snapshot +// program reads while it runs. Beside genesis's own credentials; variables so a test writes elsewhere. +var ( + busSnapshotFile = "/var/lib/mesh-genesis/bus-before-adoption.tar" + busSnapshotCredential = "/var/lib/mesh-genesis/bus-snapshot.credential" + busCertificateFile = "/var/lib/mesh-broker-tls/tls.crt" + busControllerSecret = BusControllerFile + // busStepWithin bounds the wait for the controller to say the step ended: its own bound (15 min, + // mesh-controller busStepBound), and its doctor's next look after that. + busStepWithin = 17 * time.Minute +) + +// adoptTheBus sends the nats module to this machine through the bus's planned step. A bus already on the +// module's image is the module's: it is sent like any module, and nothing is replaced. +func adoptTheBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container, + say func(string)) error { + + if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image { + say(" already adopted " + bus.Name + " — it runs the module's image") + _, err := pushNode(ctx, o, control, say) + return err + } + image, err := theModulesBusImage(ctx, o, control) + if err != nil { + return err + } + if err := snapshotTheBus(ctx, o, control, image, say); err != nil { + return err + } + say(" restarting " + bus.Name + " — once, onto the " + BusModule + " module's image, as the bus's " + + "planned step; its streams, users and certificate stay where they are (novox/hq ADR 0333 §3, ADR 0236)") + if _, err := control.within(2*o.Wait).tell(ctx, "bus", "upgrade", "--why", adoptionWhy, + "--reversible", "--snapshot-taken", busSnapshotFile); err != nil { + return fmt.Errorf("the bus's planned step was refused, so the bus is not replaced: %w", err) + } + return nil +} + +// meshBuiltImage is an image the mesh built, as it records it: `artifact-store:///@sha256:…`. +var meshBuiltImage = regexp.MustCompile(`artifact-store://([a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9-]*@sha256:[0-9a-f]{64})`) + +// theModulesBusImage is the nats module's server image, as the build just made it, pulled from this +// mesh's registry: the image that carries the snapshot program. +func theModulesBusImage(ctx context.Context, o Options, control controlPlane) (string, error) { + said, err := control.tell(ctx, "builds", BusModule, "-n", "1") + if err != nil { + return "", err + } + for _, m := range meshBuiltImage.FindAllStringSubmatch(said, -1) { + if strings.HasPrefix(m[1], BusModule+"/") { + return strings.TrimSuffix(o.Registry, "/") + "/" + m[1], nil + } + } + return "", fmt.Errorf("the mesh says no image it built for the %s module, so its snapshot program cannot "+ + "be run before the bus is replaced:\n%s", BusModule, indent(strings.TrimSpace(said))) +} + +// snapshotTheBus writes every stream of the running bus to busSnapshotFile with the module's own snapshot +// program (mesh-catalog modules/nats, ADR 0235), as the controller's account, pinned to the bus's +// certificate. The credential is written for the program's one run and removed after it. +func snapshotTheBus(ctx context.Context, o Options, control controlPlane, image string, say func(string)) error { + password, err := readCredentialFile(busControllerSecret) + if err != nil { + return fmt.Errorf("the controller's bus credential is not at %s, so the streams cannot be snapshotted "+ + "before the bus is replaced: %w", busControllerSecret, err) + } + fingerprint, err := certificateFingerprint(busCertificateFile) + if err != nil { + return err + } + port := o.Ports.orDefaults().Bus + server := "tls://127.0.0.1:" + strconv.Itoa(port) + credential, err := json.Marshal(map[string]string{"url": server, "fingerprint": fingerprint, + "user": busControllerUser, "password": password}) + if err != nil { + return err + } + if err := os.WriteFile(busSnapshotCredential, credential, 0o600); err != nil { + return fmt.Errorf("the snapshot's credential could not be written: %w", err) + } + defer os.Remove(busSnapshotCredential) + + script := "docker run --rm -i --network host --entrypoint /usr/local/bin/mesh-nats-snapshot " + image + + " snapshot --server " + server + " --credential - < " + busSnapshotCredential + " > " + busSnapshotFile + + ".partial && mv " + busSnapshotFile + ".partial " + busSnapshotFile + running, cancel := context.WithTimeout(ctx, busStepWithin) + defer cancel() + if out, err := control.run(running, "sh", "-c", script); err != nil { + _, _ = control.run(ctx, "rm", "-f", busSnapshotFile+".partial") + return fmt.Errorf("the bus's streams could not be snapshotted, so the bus is not replaced: %w\n%s", + err, indent(strings.TrimSpace(out))) + } + say(" snapshotted every stream of the bus, at " + busSnapshotFile + " (its module's snapshot program)") + return nil +} + +// certificateFingerprint is the pin of the bus's certificate: sha256 of its DER, as every machine pins it. +func certificateFingerprint(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("the bus's certificate is not at %s, so it cannot be pinned for the snapshot: %w", path, err) + } + block, _ := pem.Decode(raw) + if block == nil { + return "", fmt.Errorf("%s holds no PEM certificate", path) + } + sum := sha256.Sum256(block.Bytes) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +// waitForTheBusStep waits until the controller says the bus's planned step ended: done once the bus +// answered healthy — every stream, every durable consumer, a round trip — or failed, with its snapshot as +// the way back. The step's own bound is the controller's; this waits a little past it. +func waitForTheBusStep(ctx context.Context, o Options, control controlPlane, say func(string)) error { + deadline := time.Now().Add(busStepWithin) + var last string + for { + said, err := control.tell(ctx, "bus") + if err == nil { + for _, line := range strings.Split(said, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "last step ") { + last = strings.TrimSpace(line) + } + } + switch { + case strings.Contains(last, ": done at "): + say(" bus step done " + last) + return nil + case strings.Contains(last, ": failed at "): + return fmt.Errorf("the bus's planned step ended failed: %s\n%s", last, indent(strings.TrimSpace(said))) + } + } + if !time.Now().Before(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(answerEvery): + } + } + if last == "" { + return errors.New("the controller says no bus step after the adoption was sent; `mesh-controller bus` says how it stands") + } + return fmt.Errorf("the bus's planned step did not end within %s: %s", busStepWithin, last) +} diff --git a/internal/bootstrap/bus_adoption_test.go b/internal/bootstrap/bus_adoption_test.go new file mode 100644 index 0000000..cbb52e9 --- /dev/null +++ b/internal/bootstrap/bus_adoption_test.go @@ -0,0 +1,135 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" +) + +// Defends novox/hq issue 590 at its broker step (ADR 0333 §3, ADR 0236): the adoption replaces the bus's +// image, and the mesh replaces its bus only as a planned step. A plain push was refused — "sending anchor +// would replace the bus … a planned step: `bus upgrade --why …`" — and genesis stopped at step 18. + +func TestTheBrokerStepAdoptsTheBusThroughItsPlannedStep(t *testing.T) { + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image + upgraded := false + var credential []byte + runtime := &asked{} + agrees := aMeshThatAgrees(nil) + runtime.answer = func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case name == "sh" && strings.Contains(joined, "mesh-nats-snapshot"): + credential, _ = os.ReadFile(busSnapshotCredential) // as the program reads it + case strings.Contains(joined, "bus upgrade --why"): + upgraded = true + case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"): + if upgraded { + return theModulesImage + "\n", nil + } + return bundleImage + "\n", nil + } + return agrees(name, args) + } + _, fetch := aBusThatAdopts() + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + o.Wait = 5 * time.Second + said, err := busStepWith(t, o, runtime, fetch) + if err != nil { + t.Fatal(err) + } + + snapshot, upgrade := -1, -1 + for i, c := range runtime.commands { + switch { + case strings.HasPrefix(c, "sh -c") && strings.Contains(c, "mesh-nats-snapshot"): + snapshot = i + for _, want := range []string{"127.0.0.1:5000/nats/server@sha256:" + strings.Repeat("4", 64), + "snapshot --server tls://127.0.0.1:5671 --credential - < " + busSnapshotCredential, + "> " + busSnapshotFile + ".partial && mv " + busSnapshotFile + ".partial " + busSnapshotFile} { + if !strings.Contains(c, want) { + t.Errorf("the snapshot is not taken with %q: %s", want, c) + } + } + case strings.Contains(c, "/mesh-controller bus upgrade"): + upgrade = i + if want := "bus upgrade --why " + adoptionWhy + " --reversible --snapshot-taken " + busSnapshotFile; !strings.HasSuffix(c, want) { + t.Errorf("the planned step is asked as %q; want it to end %q", c, want) + } + case strings.HasSuffix(c, "/mesh-controller push anchor"): + t.Errorf("the bus's module was sent by a plain push, which the mesh refuses for the bus: %v", runtime.commands) + } + } + if snapshot < 0 || upgrade < 0 || snapshot > upgrade { + t.Fatalf("the streams were not snapshotted before the bus's planned step (snapshot %d, upgrade %d): %v", + snapshot, upgrade, runtime.commands) + } + var cred map[string]string + if err := json.Unmarshal(credential, &cred); err != nil { + t.Fatalf("the snapshot program was given no credential: %v", err) + } + fingerprint, _ := certificateFingerprint(busCertificateFile) + if cred["user"] != "controller" || cred["password"] != "BUS-PW" || cred["fingerprint"] != fingerprint || + cred["url"] != "tls://127.0.0.1:5671" { + t.Errorf("the snapshot's credential is %v", cred) + } + if _, err := os.Stat(busSnapshotCredential); !os.IsNotExist(err) { + t.Error("the snapshot's credential was left on the machine") + } + text := strings.Join(said, "\n") + for _, want := range []string{"snapshotted every stream", "restarting mesh-broker-nats — once", + "bus answers mesh-broker-nats", "bus step done last step 1", "adopted mesh-broker-nats"} { + if !strings.Contains(text, want) { + t.Errorf("the step did not say %q:\n%s", want, text) + } + } +} + +// A planned step the controller ends failed is the broker step failing, with the controller's words. +func TestABusStepThatEndsFailedIsRefused(t *testing.T) { + aGenesisBusOnDisk(t) + runtime := &asked{answer: func(name string, args []string) (string, error) { + return "last step 1: not known → 53e51864 on anchor by installer (x): failed at 2026-10-13 11:59; snapshot s\n" + + " stream KV_x has no messages\n", nil + }} + control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second} + err := waitForTheBusStep(context.Background(), Options{}, control, quietly) + if err == nil || !strings.Contains(err.Error(), "failed at") || !strings.Contains(err.Error(), "KV_x") { + t.Fatalf("a failed bus step was not refused with its words: %v", err) + } +} + +// The bundle's bus runs the release the module's image is built on, pinned by the same digest, so the +// adoption changes no server release and is reversible as the step says (ADR 0333 §3). +func TestTheBundlesBusRunsTheReleaseTheModulesImageIsBuiltOn(t *testing.T) { + var m struct { + Build struct { + On []struct{ Arg, Image string } `json:"on"` + } `json:"build"` + } + if err := json.Unmarshal(theCataloguesModule(t, BusModule), &m); err != nil { + t.Fatal(err) + } + base := "" + for _, on := range m.Build.On { + if on.Arg == "NATS_BASE" { + base = on.Image + } + } + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if got := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image; got != base { + t.Errorf("the bundle raises the bus on %s; the nats module's image is built on %s", got, base) + } +} diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index 12217fb..ae45e01 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -93,7 +93,8 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, // be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker // and the vault each need it or need the shape, and three copies of it drifted. func installProvider(ctx context.Context, o Options, control controlPlane, module string, - manifest []byte, buildNote string, beforePush func() error, say func(string)) error { + manifest []byte, buildNote string, beforePush func() error, say func(string), + send ...func() error) error { remote := "/" + module + "-module.json" if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { @@ -132,6 +133,11 @@ func installProvider(ctx context.Context, o Options, control controlPlane, modul return err } } + // A module whose first send is not a plain push says how it is sent: the bus, whose replacement is a + // planned step (ADR 0236), is sent by that step (adoptTheBus). + if len(send) > 0 && send[0] != nil { + return send[0]() + } _, err := pushNode(ctx, o, control, say) return err } @@ -176,9 +182,10 @@ const BusModule = "nats" // **The image is the one thing that changes, once** (ADR 0333 §3). The store's module pins the upstream // postgres the foundation runs; the bus's module builds its server image (it carries the mesh's snapshot // tool), so no pinned reference can be equal to the foundation's. The applier recreates the container on -// the module's image, on the same streams, users and certificate. This says so before the push and waits -// after it until the container runs the module's image and the bus answers, so nothing after this step -// dials a bus that is restarting. +// the module's image, on the same streams, users and certificate. The mesh replaces its bus only as a +// planned step (ADR 0236), so the module is sent by that step: its streams snapshotted first, the restart +// said, and the installer waits until the container runs the module's image, the bus answers, and the +// controller ends the step done (bus_adoption.go), so nothing after this step dials a bus that is restarting. func InstallBroker(ctx context.Context, o Options, control controlPlane, foundation *declaration.Declaration, fetch func(context.Context, string) (int, string, error), say func(string)) error { @@ -197,20 +204,15 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane, } say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept") if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)", - func() error { - if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image { - say(" already restarted " + bus.Name + " — it runs the module's image") - return nil - } - say(" restarting " + bus.Name + " — once, onto the " + module + " module's image; its " + - "streams, users and certificate stay where they are (novox/hq ADR 0333 §3)") - return nil - }, say); err != nil { + nil, say, func() error { return adoptTheBus(ctx, o, control, bus, say) }); err != nil { return err } if err := waitForTheAdoptedBus(ctx, o, control, bus, fetch, say); err != nil { return err } + if err := waitForTheBusStep(ctx, o, control, say); err != nil { + return err + } say(" adopted " + bus.Name + " — the foundation's bus is now the " + module + " module, and its user list is the module's from here on") return nil diff --git a/internal/bootstrap/phase3_test.go b/internal/bootstrap/phase3_test.go index 2b88575..d0eafc1 100644 --- a/internal/bootstrap/phase3_test.go +++ b/internal/bootstrap/phase3_test.go @@ -69,6 +69,7 @@ func aBusThatAdopts() (*asked, func(context.Context, string) (int, string, error func busStepWith(t *testing.T, o Options, runtime *asked, fetch func(context.Context, string) (int, string, error)) ([]string, error) { t.Helper() + aGenesisBusOnDisk(t) rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) @@ -222,7 +223,7 @@ func TestTheBrokerStepSaysTheRestartAndWaitsForTheBusToAnswer(t *testing.T) { runtime.answer = func(name string, args []string) (string, error) { joined := strings.Join(args, " ") switch { - case strings.Contains(joined, "push anchor"): + case strings.Contains(joined, "bus upgrade --why"): pushed = true case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"): if !pushed { @@ -304,7 +305,7 @@ func TestARerunAfterTheAdoptionSaysTheBusIsTheModulesAlready(t *testing.T) { t.Fatal(err) } text := strings.Join(said, "\n") - if strings.Contains(text, "restarting") || !strings.Contains(text, "already restarted mesh-broker-nats") { + if strings.Contains(text, "restarting") || !strings.Contains(text, "already adopted mesh-broker-nats") { t.Errorf("a re-run announced a restart, or did not say the bus is the module's already:\n%s", text) } } @@ -417,3 +418,30 @@ func TestTheBundlesBusFilesAreSeeds(t *testing.T) { } } } + +// aGenesisBusOnDisk puts what the adoption reads on the machine in a test's own directory: the bus's +// certificate, the controller's bus credential, and where the snapshot goes. The step's wait is short. +func aGenesisBusOnDisk(t *testing.T) { + t.Helper() + dir := t.TempDir() + cert := filepath.Join(dir, "tls.crt") + if err := os.WriteFile(cert, []byte(aCertificatePEM), 0o644); err != nil { + t.Fatal(err) + } + secret := filepath.Join(dir, "bus-controller.secret") + if err := os.WriteFile(secret, []byte("BUS-PW"), 0o600); err != nil { + t.Fatal(err) + } + was := []string{busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret} + wasWithin := busStepWithin + busSnapshotFile, busSnapshotCredential = filepath.Join(dir, "bus.tar"), filepath.Join(dir, "bus.credential") + busCertificateFile, busControllerSecret = cert, secret + busStepWithin = 50 * time.Millisecond + t.Cleanup(func() { + busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret = was[0], was[1], was[2], was[3] + busStepWithin = wasWithin + }) +} + +// aCertificatePEM is a PEM block standing for the bus's certificate; its pin is the sha256 of these bytes. +const aCertificatePEM = "-----BEGIN CERTIFICATE-----\nAAECAwQFBgcICQ==\n-----END CERTIFICATE-----\n" diff --git a/internal/bootstrap/registry_test.go b/internal/bootstrap/registry_test.go index f9a0280..0fbc676 100644 --- a/internal/bootstrap/registry_test.go +++ b/internal/bootstrap/registry_test.go @@ -60,6 +60,14 @@ func aMeshThatAgrees(answers map[string]string) func(string, []string) (string, switch { case name == "sh" && len(args) == 2 && strings.Contains(args[1], "accounts.conf"): return "", nil // the bus's users placed after a push (issue 608) + case name == "sh" && len(args) == 2 && strings.Contains(args[1], "mesh-nats-snapshot"): + return "", nil // the streams snapshotted before the bus's planned step (issue 590) + case name == "docker" && strings.HasSuffix(joined, "/mesh-controller bus"): + return "the bus is nats, built 53e51864, on anchor\nlast step 1: not known → 53e51864 on anchor by installer (" + + adoptionWhy + "): done at 2026-10-13 11:40; snapshot " + busSnapshotFile + "\n", nil + case name == "docker" && strings.Contains(joined, "/mesh-controller builds nats -n 1"): + return "nats built 53e51864 anchor\n image artifact-store://nats/server@sha256:" + + strings.Repeat("4", 64) + "\n bundle artifact-store://nats/tools/blobs/sha256:" + strings.Repeat("5", 64) + "\n", nil case name != "docker": return "", fmt.Errorf("unexpected program %q", name) case strings.HasSuffix(joined, "broker accounts"):