From c7c7268db08c05722c22f34210317066f7e4af6c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 02:13:35 +0200 Subject: [PATCH] =?UTF-8?q?Raise=20genesis=20past=20its=20broker=20step=20?= =?UTF-8?q?and=20hold=20that=20the=20bus=20is=20kept=20(hq=20issue=20590,?= =?UTF-8?q?=20ADR=200333=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bed runs the installer from bare twice: first with a catalogue missing the nats module, so it stops at the broker step by name and the bed reads the bus there; then with the whole catalogue, through the step. It passes only when exactly one bus container runs, it is mesh-broker-nats on the module's image, every stream that held something before the step is still the same stream, and the mesh still hears from the first node. A first run that never reaches the step skips with the reason; a second run that stops at the step fails. The judgement is a pure function, test/integration/bus-kept.ts, held by unit tests without a lab. --- scenarios/genesis-keeps-its-bus.yml | 41 +++ test/bus-kept.test.ts | 76 +++++ test/integration/bus-kept.ts | 112 ++++++++ .../integration/genesis-keeps-its-bus.test.ts | 269 ++++++++++++++++++ 4 files changed, 498 insertions(+) create mode 100644 scenarios/genesis-keeps-its-bus.yml create mode 100644 test/bus-kept.test.ts create mode 100644 test/integration/bus-kept.ts create mode 100644 test/integration/genesis-keeps-its-bus.test.ts diff --git a/scenarios/genesis-keeps-its-bus.yml b/scenarios/genesis-keeps-its-bus.yml new file mode 100644 index 0000000..1b5b0ef --- /dev/null +++ b/scenarios/genesis-keeps-its-bus.yml @@ -0,0 +1,41 @@ +# GENESIS PAST ITS BROKER STEP: the bed of novox/hq ADR 0333 §5 (issue 590). +# +# The installer raises the bus from its bundle, then adopts it as the `nats` module at its broker step (step +# 18 of 24). Before ADR 0333 the bundle raised the bus under another name and on other paths than the module +# keeps it, so adopting it would have raised a second bus beside the first, and genesis stopped there. The +# bed runs the installer from bare to that step, looks at the bus, runs it on past the step, and looks again. +# +# Two machines on one public segment, both with a way out — the installer pulls the store, the bus and the +# toolchains over it, as a bare machine does: +# - anchor: the machine genesis raises; +# - forge: serves the control plane's, the base's, the SDK's and the catalogue's repositories at the +# commits under test over git's own protocol, so the installer builds exactly those and nothing on the +# workstation listens for the lab. On the anchor itself it would be a listener, and genesis refuses a +# machine something already listens on. +scenario: genesis-keeps-its-bus + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + # The foundation, the registry, the forge genesis raises, the base and the modules it builds. + memory: 8GiB + cpus: 4 + disk: 60GiB + + forge: + at: { segment: hosting, address: [192.0.2.30] } + egress: true + inbound: allow + memory: 1GiB + cpus: 1 + disk: 10GiB + +place: + all: [host, runtime] diff --git a/test/bus-kept.test.ts b/test/bus-kept.test.ts new file mode 100644 index 0000000..89a4391 --- /dev/null +++ b/test/bus-kept.test.ts @@ -0,0 +1,76 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { busNotKept, runningFrom, streamsFrom, written, type Seen } from "./integration/bus-kept.ts"; + +/** + * The judgement of the bed that raises genesis past its broker step (novox/hq ADR 0333 §5, issue 590): + * exactly one bus container, it is mesh-broker-nats, a stream written before the step is still there, and + * the first node is still connected. Held here without a lab, so the bed's verdict is the judgement's and + * not whatever a bed happened to compare. + */ + +const UPSTREAM = "nats@sha256:" + "b".repeat(64); +const MODULES = "192.0.2.10:5000/nats@sha256:" + "c".repeat(64); +const PORTS = "127.0.0.1:8222->8222/tcp, 0.0.0.0:5671->4222/tcp"; + +function jsz(streams: Array<[string, string, number, number]>): string { + return JSON.stringify({ account_details: [{ name: "MESH", stream_detail: streams.map(([name, created, messages, last]) => + ({ name, created, state: { messages, last_seq: last } })) }] }); +} + +const before: Seen = { + running: runningFrom(`mesh-store\tpostgres@sha256:aa\t0.0.0.0:5432->5432/tcp\nmesh-broker-nats\t${UPSTREAM}\t${PORTS}\n` + + "mesh-controller\t192.0.2.10:5000/mesh-controller@sha256:dd\t\n"), + streams: streamsFrom(jsz([["KV_mesh-controller_lease", "2026-10-13T00:10:00Z", 3, 7], ["EMPTY", "2026-10-13T00:10:01Z", 0, 0]])), +}; + +const kept: Seen = { + running: runningFrom(`mesh-store\tpostgres@sha256:aa\t0.0.0.0:5432->5432/tcp\nmesh-broker-nats\t${MODULES}\t${PORTS}\n` + + "mesh-controller\t192.0.2.10:5000/mesh-controller@sha256:dd\t\n"), + streams: streamsFrom(jsz([["KV_mesh-controller_lease", "2026-10-13T00:10:00Z", 3, 9]])), +}; + +const SAID = "broker — the foundation's bus, adopted as the nats module\n restarting mesh-broker-nats — once, onto the nats module's image"; +const HERE = "anchor here last 3s ago"; + +test("a bus adopted in place is kept: one bus, the module's image, its streams, the node connected", () => { + assert.deepEqual(busNotKept(before, kept, HERE, SAID), []); +}); + +test("only a stream holding something counts as written before the step", () => { + assert.deepEqual(written(before).map((s) => s.name), ["KV_mesh-controller_lease"]); +}); + +test("a second bus beside the first is not kept: the fault genesis had before ADR 0333", () => { + const two: Seen = { ...kept, running: [...kept.running, + { name: "mesh-broker", image: UPSTREAM, ports: "0.0.0.0:5671->4222/tcp" }] }; + const wrong = busNotKept(before, two, HERE, SAID); + assert.equal(wrong.length, 1); + assert.match(wrong[0] as string, /2 bus containers.*mesh-broker \(/); +}); + +test("a bus under another name, or none, is not kept", () => { + const renamed: Seen = { ...kept, running: kept.running.map((r) => r.name === "mesh-broker-nats" ? { ...r, name: "nats-server" } : r) }; + assert.ok(busNotKept(before, renamed, HERE, SAID).some((w) => /mesh-broker-nats does not run/.test(w))); + const none: Seen = { ...kept, running: kept.running.filter((r) => r.name !== "mesh-broker-nats") }; + assert.ok(busNotKept(before, none, HERE, SAID).some((w) => /0 bus containers/.test(w))); +}); + +test("a bus still on the bundle's image was not adopted", () => { + const same: Seen = { ...kept, running: before.running }; + assert.ok(busNotKept(before, same, HERE, SAID).some((w) => /still runs .* the image the bundle raised/.test(w))); + assert.ok(busNotKept(before, kept, HERE, "broker — adopted").some((w) => /did not say the bus restarts/.test(w))); +}); + +test("a stream gone or made again is not kept", () => { + const gone: Seen = { ...kept, streams: [] }; + assert.ok(busNotKept(before, gone, HERE, SAID).some((w) => /KV_mesh-controller_lease .* is gone/.test(w))); + const again: Seen = { ...kept, streams: streamsFrom(jsz([["KV_mesh-controller_lease", "2026-10-13T00:40:00Z", 1, 1]])) }; + assert.ok(busNotKept(before, again, HERE, SAID).some((w) => /was made again/.test(w))); +}); + +test("a node the mesh has not heard from since the step is not connected", () => { + assert.ok(busNotKept(before, kept, "anchor out of touch 4m", SAID).some((w) => /has not heard from the first node/.test(w))); + assert.ok(busNotKept(before, kept, "", SAID).some((w) => /no record/.test(w))); +}); diff --git a/test/integration/bus-kept.ts b/test/integration/bus-kept.ts new file mode 100644 index 0000000..7c95d42 --- /dev/null +++ b/test/integration/bus-kept.ts @@ -0,0 +1,112 @@ +/** + * Whether genesis kept its bus through the broker step (novox/hq ADR 0333 §5, issue 590). + * + * Pure, so the judgement is tested without a lab (test/bus-kept.test.ts) and the bed only gathers what it + * judges: the containers the anchor runs, the streams its bus holds, and what the mesh says of the node, + * once before the broker step and once after it. + */ + +/** One running container, as `docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'` says it. */ +export interface Running { + name: string; + image: string; + ports: string; +} + +/** One stream on the bus, as the server's monitor says it (`/jsz?accounts=true&streams=true`). */ +export interface Stream { + account: string; + name: string; + created: string; + messages: number; + lastSeq: number; +} + +/** What the anchor showed at one moment. */ +export interface Seen { + running: Running[]; + streams: Stream[]; +} + +/** The container the bundle raises the bus as, and the nats module keeps it as (ADR 0333 §1). */ +export const BUS = "mesh-broker-nats"; + +/** `docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'`, read. */ +export function runningFrom(ps: string): Running[] { + return ps.split("\n").map((l) => l.trim()).filter(Boolean).map((l) => { + const [name = "", image = "", ports = ""] = l.split("\t"); + return { name, image, ports }; + }); +} + +/** The server's `/jsz?accounts=true&streams=true`, read: every stream of every account. */ +export function streamsFrom(jsz: string): Stream[] { + const parsed = JSON.parse(jsz) as { + account_details?: Array<{ + name?: string; + stream_detail?: Array<{ name?: string; created?: string; state?: { messages?: number; last_seq?: number } }>; + }>; + }; + const out: Stream[] = []; + for (const account of parsed.account_details ?? []) { + for (const s of account.stream_detail ?? []) { + out.push({ + account: account.name ?? "", name: s.name ?? "", created: s.created ?? "", + messages: s.state?.messages ?? 0, lastSeq: s.state?.last_seq ?? 0, + }); + } + } + return out; +} + +/** + * A bus container: one that serves the bus's port inside it, or carries a bus's name. Both, because a + * second bus raised by a module under another name is exactly what this looks for, and so is one raised on + * another port under the bus's name. + */ +export function busContainers(running: Running[]): Running[] { + return running.filter((r) => /->4222\/tcp/.test(r.ports) || r.name.startsWith("mesh-broker")); +} + +/** The streams written before the step: the ones holding anything. */ +export function written(seen: Seen): Stream[] { + return seen.streams.filter((s) => s.messages > 0 || s.lastSeq > 0); +} + +/** + * Every way the bus was not kept, in sentences; empty when it was. `node` is the mesh's line for the first + * node from `node list`; `said` is what the installer printed on the run that crossed the step. + */ +export function busNotKept(before: Seen, after: Seen, node: string, said: string): string[] { + const wrong: string[] = []; + const buses = busContainers(after.running); + if (buses.length !== 1) { + wrong.push(`${buses.length} bus containers run after the broker step, not one: ` + + (buses.map((b) => `${b.name} (${b.image}, ${b.ports || "nothing published"})`).join("; ") || "none")); + } + const bus = after.running.find((r) => r.name === BUS); + if (!bus) { + wrong.push(`${BUS} does not run after the broker step`); + } + const was = before.running.find((r) => r.name === BUS); + if (bus && was && bus.image === was.image) { + wrong.push(`${BUS} still runs ${bus.image}, the image the bundle raised it from: the module's image never ` + + `took its place (ADR 0333 §3)`); + } + if (!/restarting\s+mesh-broker-nats/.test(said)) { + wrong.push("the installer did not say the bus restarts onto the module's image (ADR 0333 §3)"); + } + for (const s of written(before)) { + const now = after.streams.find((a) => a.account === s.account && a.name === s.name); + if (!now) { + wrong.push(`the stream ${s.name} (${s.account}), holding ${s.messages} message(s) before the step, is gone`); + } else if (now.created !== s.created || now.lastSeq < s.lastSeq) { + wrong.push(`the stream ${s.name} (${s.account}) was made again: created ${s.created}, last ${s.lastSeq} ` + + `before the step; created ${now.created}, last ${now.lastSeq} after it`); + } + } + if (!/^\S+\s+here\b/.test(node.trim())) { + wrong.push(`the mesh has not heard from the first node since the step: ${node.trim() || "it has no record of it"}`); + } + return wrong; +} diff --git a/test/integration/genesis-keeps-its-bus.test.ts b/test/integration/genesis-keeps-its-bus.test.ts new file mode 100644 index 0000000..7541cc4 --- /dev/null +++ b/test/integration/genesis-keeps-its-bus.test.ts @@ -0,0 +1,269 @@ +/** + * GENESIS KEEPS ITS BUS through the broker step (novox/hq ADR 0333 §5, issue 590). + * + * The installer raises the bus from its bundle and, at its broker step (step 18 of 24), adopts it as the + * catalogue's `nats` module. Until ADR 0333 the bundle raised the bus as `mesh-broker` on docker volumes, + * and the module keeps it as `mesh-broker-nats` on directories of the machine: adopting it would have raised + * a second bus beside the first, so the installer refused there and genesis stopped. ADR 0333 has the bundle + * raise the bus where the module keeps it, so the adoption replaces the server's image once and keeps + * everything else. + * + * The bed runs the installer from bare twice. The first run is given a catalogue without the `nats` module, + * so it stops at the broker step by name, before anything is registered — the moment just before the step, + * with everything genesis wrote on the bus by then. The bed reads the bus there: its containers, and every + * stream holding anything. The second run, given the whole catalogue, finds the steps before done and goes + * on through the broker step. The bed reads the bus again, and passes only when (ADR 0333 §5): + * + * - exactly one bus container runs, and it is mesh-broker-nats, on the module's image now; + * - every stream that held something before the step is still there, the same stream, as long or longer; + * - the mesh has heard from the first node since. + * + * The judgement is test/integration/bus-kept.ts, held without a lab by test/bus-kept.test.ts. + * + * **A bed that could not reach the step says so and does not fail**: a first run that stopped earlier than + * the broker step — a pull the internet refused, a build that broke — proves nothing about the step, and the + * test is skipped with the reason. A second run that stops AT the broker step is the fault, and fails. One + * that stops after it has crossed it: what it stopped on is reported, and the bus is judged. + * + * MESH_LAB_HOST_BINARY=/mesh-host MESH_LAB_BOOTSTRAP_BINARY=/mesh-bootstrap + * MESH_LAB_BUNDLE=/examples/foundation-first-node.lock + * MESH_LAB_CATALOG=/modules + * MESH_LAB_REPOS= + * — served by the forge machine at each checkout's HEAD; the catalogue's is the one built from + * MESH_LAB_KEEP=1 leave it standing + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec, push } from "../../src/lifecycle/operate.ts"; +import { bootstrapBinaryPath, hostBinaryPath } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts"; +import { genesis, type GenesisResult } from "./genesis.ts"; +import { BUS, busContainers, busNotKept, runningFrom, streamsFrom, written, type Seen } from "./bus-kept.ts"; + +const SCENARIO = "genesis-keeps-its-bus"; +const ANCHOR = "anchor"; +const FORGE = "forge"; +const FORGE_ADDRESS = "192.0.2.30"; +const REGISTRY = "192.0.2.10:5000"; +/** The repositories the installer clones, served by the forge at each checkout's HEAD. */ +const REPOS = ["mesh-controller", "mesh-tools", "mesh-sdk", "mesh-catalog"] as const; +/** The step this bed is about, as the installer names it. */ +const BROKER = "broker"; +/** The installer's steps before it, in its order: a first run stopped at one of these never reached it. */ +const BEFORE_BROKER = ["preflight", "load", "build", "bundle", "apply", "verify", "enrol", "operator", "registry", + "publish", "control-plane", "retire", "builder", "packages", "sdk", "base", "store"]; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; +const repos = process.env["MESH_LAB_REPOS"] ?? ""; +const KEEP = !!process.env["MESH_LAB_KEEP"]; +const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "genesis-keeps-its-bus-live" : undefined); + +const missingRepo = REPOS.find((r) => !repos || !existsSync(resolve(repos, r, ".git"))); +const skip = + !capability.usable ? capability.why : + !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : + !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" : + !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to the genesis bundle" : + !catalogDir || !existsSync(join(catalogDir, "nats", "module.json")) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" : + missingRepo ? `MESH_LAB_REPOS holds no ${missingRepo} checkout` : + false; + +let instanceId = ""; +/** Why the bed never reached the step, or "" when it did. */ +let unreached = ""; +let first: GenesisResult | undefined; +let second: GenesisResult | undefined; +let seenBefore: Seen | undefined; +let seenAfter: Seen | undefined; +let nodeLine = ""; +const heads: Record = {}; +const report: string[] = []; + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} + +/** Each repository at its checkout's HEAD, served from the forge machine over git's protocol as `main`. */ +async function raiseForge(): Promise { + const dir = mkdtempSync(join(tmpdir(), "mesh-lab-590-forge-")); + try { + for (const repo of REPOS) { + const checkout = resolve(repos, repo); + heads[repo] = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + const bare = join(dir, `${repo}.git`); + execFileSync("git", ["init", "-q", "--bare", bare]); + // No collection after the push: one run in the background rewrites the objects while they are carried. + execFileSync("git", ["-C", bare, "config", "receive.autogc", "false"]); + execFileSync("git", ["-C", checkout, "push", "-q", "--force", bare, "HEAD:refs/heads/main"], { stdio: "pipe" }); + } + const tar = join(tmpdir(), `mesh-lab-590-forge-${process.pid}.tar`); + execFileSync("tar", ["-cf", tar, "-C", dir, "."]); + await push(instanceId, FORGE, tar, "/tmp/forge.tar"); + rmSync(tar, { force: true }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + // Owned by the daemon's user, or git refuses to serve it ("dubious ownership") and says nothing. + await must(FORGE, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` + + `git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE_ADDRESS} --pid-file=/run/git-daemon.pid`); + for (let i = 0; ; i++) { + if ((await on(ANCHOR, `git ls-remote git://${FORGE_ADDRESS}/mesh-catalog.git main`)).ok) break; + if (i >= 30) throw new Error(`the anchor cannot reach the forge at git://${FORGE_ADDRESS}/`); + await new Promise((r) => setTimeout(r, 2_000)); + } +} + +/** What the anchor shows of its bus: the containers it runs, and every stream the bus holds. */ +async function look(): Promise { + const ps = await must(ANCHOR, `docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'`); + // The monitor, published on the anchor's loopback by the bundle and by the module alike. + let jsz = ""; + for (let i = 0; ; i++) { + const asked = await on(ANCHOR, `curl -sf --max-time 10 'http://127.0.0.1:8222/jsz?accounts=true&streams=true'`); + if (asked.ok) { jsz = asked.out; break; } + if (i >= 30) throw new Error(`the bus's monitor does not answer on the anchor: ${asked.out}`); + await new Promise((r) => setTimeout(r, 2_000)); + } + return { running: runningFrom(ps), streams: streamsFrom(jsz) }; +} + +function stoppedAt(r: GenesisResult): string { + return r.step.replace(/^step \d+ of \d+, /, ""); +} + +before(async () => { + if (skip) return; + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), + }); + instanceId = raised.instanceId; + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); + + try { + await raiseForge(); + report.push(` forge ${REPOS.map((r) => `${r} ${heads[r]?.slice(0, 8)}`).join(", ")}`); + } catch (err) { + unreached = `the forge was not raised, so the installer never ran: ${(err as Error).message}`; + return; + } + + const forge = (repo: string) => `git://${FORGE_ADDRESS}/${repo}.git`; + const run = (catalogue: string) => genesis({ + instanceId, + node: ANCHOR, + installer: installer as string, + catalogDir: catalogue, + catalogueModules: ["distribution", "builder", "postgres"], + bundleTemplate: foundationBundle(bundle, []), + registry: REGISTRY, + source: forge("mesh-controller"), + sourceRef: heads["mesh-controller"] as string, + toolsSource: forge("mesh-tools"), + sdkSource: forge("mesh-sdk"), + catalogSource: forge("mesh-catalog"), + // A refusal is the answer, and a re-run is the second half of this bed, not a retry. + attempts: 1, + verify: false, + log: (m) => console.log(m), + }); + + // 1. From bare to the broker step, and no further: the catalogue the installer reads has no nats module. + const withoutTheBus = mkdtempSync(join(tmpdir(), "mesh-lab-590-catalogue-")); + try { + const nats = join(catalogDir, "nats"); + cpSync(catalogDir, withoutTheBus, { recursive: true, filter: (src) => src !== nats && !src.startsWith(nats + "/") }); + first = await run(withoutTheBus); + } catch (err) { + unreached = `the installer could not be run: ${(err as Error).message}`; + return; + } finally { + rmSync(withoutTheBus, { recursive: true, force: true }); + } + report.push(...first.report); + if (first.ok || stoppedAt(first) !== BROKER) { + unreached = first.ok + ? "the first run finished without the nats module, so the broker step never stopped it" + : `the first run stopped at ${first.step || "no step it named"}, before the broker step:\n${first.why}`; + return; + } + if (!/nats module's manifest could not be read/.test(first.said)) { + unreached = `the first run stopped at the broker step for another reason than the missing module:\n${first.why}`; + return; + } + seenBefore = await look(); + report.push(` before the step ${busContainers(seenBefore.running).map((b) => `${b.name} on ${b.image}`).join("; ")}`); + report.push(` ${written(seenBefore).length} stream(s) holding something: ` + + written(seenBefore).map((s) => `${s.name} (last ${s.lastSeq})`).join(", ")); + if (written(seenBefore).length === 0) { + unreached = "nothing was written on the bus before the broker step, so whether its streams survive it says nothing"; + return; + } + + // 2. The whole catalogue: the steps before are done already, and the broker step adopts the bus. + try { + second = await run(catalogDir); + } catch (err) { + unreached = `the installer could not be run again: ${(err as Error).message}`; + return; + } + report.push(...second.report); + if (!second.ok && BEFORE_BROKER.includes(stoppedAt(second))) { + unreached = `the second run stopped at ${second.step}, before the broker step it was run again for:\n${second.why}`; + return; + } + seenAfter = await look(); + report.push(` after the step ${busContainers(seenAfter.running).map((b) => `${b.name} on ${b.image}`).join("; ") || "no bus"}`); + const nodes = await on(ANCHOR, `docker exec mesh-controller /mesh-controller node list`, 60_000); + nodeLine = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${ANCHOR} `)) ?? ""; + report.push(` node list ${nodeLine || nodes.out.trim()}`); +}, { timeout: 7_200_000 }); + +after(async () => { + if (KEEP) { + console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`); + return; + } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("genesis adopts its bus as the nats module in place: one bus, its streams kept, the node connected", { + skip, timeout: 7_200_000, +}, async (t) => { + console.log(report.join("\n")); + if (unreached) { + t.skip(`ADR 0333 §5 NOT REACHED: ${unreached}`); + return; + } + const ran = second as GenesisResult; + // Stopped AT the broker step: the adoption itself refused or failed — the fault, not a missed bed. + assert.ok(ran.ok || stoppedAt(ran) !== BROKER, + `genesis stopped at its broker step and did not adopt the bus:\n${ran.why}`); + if (!ran.ok) { + console.log(`\nthe second run crossed the broker step and stopped later, at ${ran.step} — not this bed's ` + + `question:\n${ran.why}`); + } + const wrong = busNotKept(seenBefore as Seen, seenAfter as Seen, nodeLine, ran.said); + assert.deepEqual(wrong, [], `genesis did not keep its bus through the broker step:\n - ${wrong.join("\n - ")}`); + console.log(`\nVERDICT: one bus, ${BUS}, on the module's image; ${written(seenBefore as Seen).length} stream(s) ` + + `kept; ${ANCHOR} connected.`); +});