diff --git a/scenarios/built-store-cross-node.yml b/scenarios/built-store-cross-node.yml new file mode 100644 index 0000000..794ea3d --- /dev/null +++ b/scenarios/built-store-cross-node.yml @@ -0,0 +1,54 @@ +# TWO MACHINES, AND NOTHING FAKED. The no-fake multi-node gate. +# +# The one-node scenario proves a machine given nothing but a container runtime ends up with a mesh +# that BUILT everything it runs. This is its two-machine sibling, and the multi-node claim it adds +# is the one the rewrite-based beds could not honestly make: every image on either machine was +# pulled from the internet or built by the mesh's own builder — the bed rewrites nothing, pins +# nothing, stocks nothing. The builder is the only thing that ever turns a manifest's placeholder +# into a digest, exactly as in production. +# +# anchor is raised into a mesh of one by the installer, adopts the foundation store and broker as +# the postgres and lavinmq modules (built by the mesh), and node2 joins and runs the consumers — +# amqp-ping against the one broker, letta against the one store — over the overlay. +# +# **There is no `images:` key, and that is the whole point of this file.** +# +# EGRESS IS NOT OPTIONAL: with nothing loaded, a sealed machine stops at the installer's first pull. +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap +# MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock +# MESH_LAB_CATALOG=.../mesh-catalog/modules +# MESH_LAB_SOURCE= MESH_LAB_SOURCE_REF= +scenario: built-store-cross-node + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + # The address matters: the foundation template names the broker at 192.0.2.10:5671, and a token + # carries that verbatim as the endpoint an enrolling node dials. + # + # Sized like the one-node anchor: store, broker, registry, two control planes during the pivot, + # the builder with a Node toolchain and npm cache, and the built module images on top. + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 12GiB + cpus: 6 + disk: 60GiB + # The joined consumer node. It builds nothing — it pulls what the mesh's registry serves and what + # its modules name upstream (letta's app image comes from the internet, ~2GB), so it needs egress + # and room for images, not build horsepower. + node2: + at: { segment: hosting, address: [192.0.2.20] } + egress: true + inbound: allow + memory: 6GiB + cpus: 4 + disk: 40GiB + +place: + all: [host, runtime] diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts new file mode 100644 index 0000000..9f6cac7 --- /dev/null +++ b/test/integration/built-store-cross-node.test.ts @@ -0,0 +1,324 @@ +/** + * THE NO-FAKE MULTI-NODE GATE: two machines, everything built by the mesh itself. + * + * The rewrite-based multi-node beds pre-stock runtime images and rewrite each manifest's + * placeholder digest to whatever they stocked — bed code doing the builder's job, which means the + * builder's job was never under test. This bed removes the shortcut. The scenario names NO images; + * the committed manifests are registered VERBATIM; the only thing that ever turns + * `mesh-runtime-@sha256:0000…` (or an `artifact:` reference) into a real digest is the mesh's + * own builder, exactly as in production. + * + * What it proves, end to end: + * 1. The installer raises `anchor` into a mesh of one — building the control plane (ADR 0073), + * standing up the registry and the builder. + * 2. The mesh BUILDS the shared base, then postgres and lavinmq, from the forge — and adopts the + * foundation's own store and broker as those modules (ADR 0078), the store's genesis superuser + * carried in through `secret accept` (the same act as hq phase3 deliverSuperuser). + * 3. `node2` joins, and its consumers are BUILT and delivered the same way: `amqp-ping` opens the + * one broker and `letta` gets a database on the one store — both across the overlay, admitted + * by the firewall the nftables module derives (issues 055/056/057 in one bed). + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap + * MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock (the TEMPLATE) + * MESH_LAB_CATALOG=.../mesh-catalog/modules + * MESH_LAB_SOURCE=git:///mesh-controller.git MESH_LAB_SOURCE_REF= + * MESH_LAB_BUILD_REF= + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec, push } from "../../src/lifecycle/operate.ts"; +import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts"; +import { genesis, type GenesisResult } from "./genesis.ts"; +import { keep, ready, returnTo } from "../../src/warm.ts"; + +const SCENARIO = "built-store-cross-node"; +const CONTROL = "anchor"; +const NODE = "node2"; +const ANCHOR = "192.0.2.10"; +const REGISTRY = `${ANCHOR}:5000`; +const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" }; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; +const source = process.env["MESH_LAB_SOURCE"] ?? ""; +const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? ""; +const KEEP = !!process.env["MESH_LAB_KEEP"]; +/** + * MESH_LAB_WARM=1: restore the post-genesis, both-nodes-enrolled state from a snapshot instead of + * re-running the installer (minutes -> seconds), refused — not silently rebuilt — when the commits + * it was built from have moved (src/warm.ts). Fresh stays the default: a run that must MEAN + * something raises from nothing. + */ +const warming = process.env["MESH_LAB_WARM"] === "1"; + +function forgeUrl(repo: string): string { + const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`]; + if (override) return override; + return source.replace(/[^/]+\.git$/, `${repo}.git`); +} +function refFor(repo: string): string { + const override = process.env[`MESH_LAB_BUILD_REF_${repo.toUpperCase().replaceAll("-", "_")}`]; + return override ?? process.env["MESH_LAB_BUILD_REF"] ?? "main"; +} +const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ?? + resolve(catalogDir, "..", "..", BASE.repo, "module.json"); + +const skip = + !capability.usable ? capability.why : + !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : + !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" : + !source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" : + !sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" : + !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" : + !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" : + false; + +let instanceId = ""; +let raised: GenesisResult; + +function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} +/** The control plane — retried through the brief recreate window a spec change causes. */ +async function mesh(command: string, timeoutMs?: number): Promise { + const deadline = Date.now() + (timeoutMs ?? 120_000); + for (;;) { + const got = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); + if (got.ok) return got.out; + if (!/is not running|No such container/.test(got.out) || Date.now() > deadline) { + throw new Error(`${CONTROL}: mesh ${command}\n${got.out}`); + } + await new Promise((r) => setTimeout(r, 5_000)); + } +} +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +/** Register a committed manifest VERBATIM — the point of this bed: no rewriting, ever. */ +async function registerModule(module: string, manifest: string): Promise { + assert.ok(existsSync(manifest), `no manifest for ${module} at ${manifest}`); + const onMachine = `/tmp/${module}.json`; + await push(instanceId, CONTROL, manifest, onMachine); + await must(CONTROL, `docker cp ${onMachine} mesh-controller:/${module}.json`); + return mesh(`module add /${module}.json`); +} + +/** Build a module with the mesh's own builder, issue its account, and assign it to a node. */ +async function buildAndAssign(module: string, node: string, opts?: { build?: boolean }): Promise { + await registerModule(module, resolve(catalogDir, module, "module.json")); + if (opts?.build !== false) { + const built = await mesh( + `build ${forgeUrl("mesh-catalog")} --path modules/${module} --ref ${refFor("mesh-catalog")} --wait 1200s`, + 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + } + const manifest = readFileSync(resolve(catalogDir, module, "module.json"), "utf8"); + if (manifest.includes("MESH_BROKER_FILE")) { + const issued = await mesh(`module issue ${module} --node ${node}`); + assert.match(issued, /scoped to what it (emits and consumes|consumes and emits)/, + `the broker account for ${module} was not issued:\n${issued}`); + } + await mesh(`assign ${node} ${module}`); + await mesh(`push ${node}`, 600_000); +} + +async function waitForContainer(node: string, container: string, seconds = 300): Promise { + const deadline = Date.now() + seconds * 1_000; + let last = ""; + while (Date.now() < deadline) { + const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + last = ps; + const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container); + if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps; + await new Promise((r) => setTimeout(r, 5_000)); + } + const logs = (await on(node, `docker logs --tail 20 ${container} 2>&1`)).out; + throw new Error(`${container} never came up on ${node}:\n${last}\n--- logs ---\n${logs}`); +} + +before(async () => { + if (skip) return; + if (warming) { + const said = await ready(SCENARIO); + if (said.use === "restore") { + instanceId = said.instanceId; + const seconds = await returnTo(instanceId); + // A snapshot captures disk, not memory: the restore reboots. anchor's host is a systemd + // service (genesis hostService) and comes back on its own; node2's was hand-started. + await must(NODE, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`); + const back = (await on(CONTROL, `docker ps --format '{{.Names}}'`)).out; + assert.match(back, /mesh-controller/, `the control plane did not come back after restore:\n${back}`); + console.log(`warm: returned ${instanceId} to its post-genesis state in ${seconds.toFixed(1)}s`); + return; + } + console.log(`warm: raising fresh — ${said.why}`); + } + const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = bed.instanceId; + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP)" : ""}`); + console.log("NOTHING WAS LOADED: this scenario names no images; the mesh builds or pulls everything."); + + // Genesis: the installer raises anchor into a mesh of one, BUILDING the control plane, and + // leaves the anchor enrolled with an agent running (hostService). + raised = await genesis({ + instanceId, + node: CONTROL, + installer: installer as string, + catalogDir, + bundleTemplate: foundationBundle(bundle, []), + registry: REGISTRY, + source, + sourceRef, + toolsSource: forgeUrl(BASE.repo), + toolsRef: refFor(BASE.repo), + catalogSource: forgeUrl("mesh-catalog"), + catalogRef: refFor("mesh-catalog"), + sdkSource: forgeUrl("mesh-sdk"), + sdkRef: refFor("mesh-sdk"), + site: "hosting", + hostService: true, + ...(binary ? { hostBinary: binary } : {}), + log: (m) => console.log(m), + }); + if (!raised.ok) throw new Error(`${raised.step || "genesis"}: ${raised.why}\n\n${raised.report.join("\n")}`); + + // node2 joins the mesh: a token against the anchor's public broker endpoint, then an agent. + await mesh(`node add ${NODE}`); + const token = tokenFrom(await mesh(`token issue --node ${NODE}`)); + const said = await must(NODE, `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, new RegExp(`enrolled as ${NODE}`), said); + await must(NODE, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + + // The expensive, deterministic part is done: genesis ran and both machines are enrolled. Keep it. + if (warming) { + const warm = await keep(SCENARIO, instanceId); + console.log(`warm: ${warm.instanceId} kept, against ` + + Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", ")); + } +}, { timeout: 7_200_000 }); + +after(async () => { + if (warming) { console.log(`warm — ${instanceId} stays; \`mesh-lab warm cool\` retires it`); return; } + if (KEEP) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("a joined node's consumers open the store and broker the mesh built and adopted, over the overlay", { + skip, timeout: 3_600_000, +}, async () => { + // The overlay, so bindings carry `.internal` names; the firewall, so from:mesh is what admits them. + await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site hosting`); + await mesh(`overlay place ${NODE} --site hosting`); + await mesh(`assign ${CONTROL} networking`); + await mesh(`assign ${NODE} networking`); + + // The networking module carries the registry trust (ADR 0082): a merged daemon.json naming the + // store's internal name, and a docker restart when it first lands. It must be ON both machines + // before anything builds or pulls — the builder pushes to anchor.internal:5000 the moment the + // first build finishes. Pushed and WAITED for, because the restart bounces the runtime and an + // apply in flight retries. + for (const machine of [CONTROL, NODE]) { + console.log(await mesh(`push ${machine}`, 600_000)); + const deadline = Date.now() + 300_000; + let trusted = false; + while (Date.now() < deadline) { + const got = await on(machine, `grep -s "anchor.internal:5000" /etc/docker/daemon.json && docker info --format '{{json .RegistryConfig.IndexConfigs}}' 2>/dev/null | grep -q "anchor.internal:5000" && echo TRUSTED`); + if (/TRUSTED/.test(got.out)) { trusted = true; break; } + await new Promise((r) => setTimeout(r, 5_000)); + } + // A failure here has two distinguishable shapes, so the dump carries both: a declaration that + // never named the trust (the controller composed without it — issues 042/048 as a race), and + // one that named it and was never applied (delivery or apply). mesh-host's log says which. + assert.ok(trusted, `${machine}'s runtime never learned the registry trust:\n` + + (await on(machine, `cat /etc/docker/daemon.json 2>&1; docker info 2>&1 | tail -20`)).out + + `\n--- ${machine} mesh-host.log ---\n` + + (await on(machine, `tail -60 /var/log/mesh-host.log 2>&1`)).out + + `\n--- ${machine} declared registry-trust? ---\n` + + (await on(machine, `base64 -d < /var/lib/mesh-host/declared.json 2>/dev/null | grep -c registry-trust; python3 -c "import json,base64; d=json.load(open('/var/lib/mesh-host/declared.json')); print('registry-trust' in base64.b64decode(d['declaration']).decode())" 2>&1`)).out); + } + + // The shared base first — every module with code of its own stands on it. + await registerModule(BASE.module, baseManifest); + const base = await mesh(`build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000); + assert.doesNotMatch(base, /failed/i, base); + + // Genesis already adopted the store as the postgres module (superuser accepted), and installed + // nftables and the catalogue — verified rather than redone. The broker is the one foundation + // half genesis leaves to the mesh proper: adopt it here, BUILT by the mesh's own builder. + await waitForContainer(CONTROL, "mesh-postgres", 120); + await buildAndAssign("lavinmq", CONTROL); + await waitForContainer(CONTROL, "mesh-lavinmq", 600); + + // The consumer on the joined node — built by the mesh, delivered from its registry. One consumer, + // amqp-ping, because it is one of the eight modules the mesh can actually build; the catalogue's + // DB consumers all lack a build section (hq issue 060), so the store's cross-node proof stays with + // the stocked-image bed until one of them gains one. NOTE: this delivery is the path hq issues + // 042 (a node has no registry account) and 048 (nothing makes a machine trust the registry) leave + // open — this bed is their honest gate, red until they are fixed. + await buildAndAssign("amqp-ping", NODE); + + // 057: the provider node is composed again so its provisioners learn of the remote consumers. + await mesh(`push ${CONTROL}`, 600_000); + + // THE BROKER, cross-node: amqp-ping's binding names the control-node's overlay name, its vhost is + // minted on the one broker, and it holds the connection. + try { + await waitForContainer(NODE, "amqp-ping", 600); + } catch (err) { + const hostLog = (await on(NODE, `tail -40 /var/log/mesh-host.log`)).out; + throw new Error(`${(err as Error).message}\n--- ${NODE} mesh-host.log (a pull that failed ` + + `never reaches container logs; hq issues 042/048) ---\n${hostLog}`); + } + const amqpBound = (await on(NODE, `cat /var/lib/amqp-ping/amqp.json 2>&1`)).out; + assert.match(amqpBound, new RegExp(`${CONTROL}\\.internal`), + `the amqp grant does not point at the control-node over the overlay:\n${amqpBound}`); + { + const deadline = Date.now() + 240_000; + let vhosts = ""; + while (Date.now() < deadline) { + vhosts = (await on(CONTROL, `docker exec mesh-broker lavinmqctl list_vhosts 2>&1`)).out; + if (new RegExp(`${NODE}|amqp-ping`).test(vhosts)) break; + await new Promise((r) => setTimeout(r, 5_000)); + } + assert.match(vhosts, new RegExp(`${NODE}|amqp-ping`), + `no vhost was minted on the one broker for the joined consumer:\n${vhosts}\n` + + `--- provisioner ---\n${(await on(CONTROL, `docker logs mesh-lavinmq 2>&1 | tail -20`)).out}\n` + + `--- consumer ---\n${(await on(NODE, `docker logs amqp-ping 2>&1 | tail -20`)).out}`); + } + + // Adoption did not cost the foundation: the containers genesis raised are still the ones running. + const up = await must(CONTROL, `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { + assert.match(up, new RegExp(`(^|\\n)${c}(\\n|$)`), `${c} did not survive adoption:\n${up}`); + } + // Steady a moment, then confirm the consumer did not crash-loop after connecting. + await new Promise((r) => setTimeout(r, 15_000)); + assert.match((await on(NODE, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out, + /amqp-ping\tUp/, `amqp-ping did not stay up on ${NODE}`); + + console.log(`amqp: ${amqpBound.trim().slice(0, 160)}`); +});