From bfd70e9e57797209f579ca00be7989c8649b3144 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 22:15:34 +0200 Subject: [PATCH 01/10] The no-fake multi-node bed: two machines, everything built by the mesh itself The scenario names no images and the bed rewrites nothing: the installer raises anchor (building the control plane), the mesh's own builder builds base, postgres, lavinmq and the joined node's consumers from the forge and pins every digest itself, the committed manifests are registered verbatim, and node2 proves both foundation halves cross-node. Being iterated toward green (run 3 in flight); banked so nothing is lost. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- scenarios/built-store-cross-node.yml | 54 ++++ .../built-store-cross-node.test.ts | 287 ++++++++++++++++++ 2 files changed, 341 insertions(+) create mode 100644 scenarios/built-store-cross-node.yml create mode 100644 test/integration/built-store-cross-node.test.ts diff --git a/scenarios/built-store-cross-node.yml b/scenarios/built-store-cross-node.yml new file mode 100644 index 0000000..794ea3d --- /dev/null +++ b/scenarios/built-store-cross-node.yml @@ -0,0 +1,54 @@ +# TWO MACHINES, AND NOTHING FAKED. The no-fake multi-node gate. +# +# The one-node scenario proves a machine given nothing but a container runtime ends up with a mesh +# that BUILT everything it runs. This is its two-machine sibling, and the multi-node claim it adds +# is the one the rewrite-based beds could not honestly make: every image on either machine was +# pulled from the internet or built by the mesh's own builder — the bed rewrites nothing, pins +# nothing, stocks nothing. The builder is the only thing that ever turns a manifest's placeholder +# into a digest, exactly as in production. +# +# anchor is raised into a mesh of one by the installer, adopts the foundation store and broker as +# the postgres and lavinmq modules (built by the mesh), and node2 joins and runs the consumers — +# amqp-ping against the one broker, letta against the one store — over the overlay. +# +# **There is no `images:` key, and that is the whole point of this file.** +# +# EGRESS IS NOT OPTIONAL: with nothing loaded, a sealed machine stops at the installer's first pull. +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap +# MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock +# MESH_LAB_CATALOG=.../mesh-catalog/modules +# MESH_LAB_SOURCE= MESH_LAB_SOURCE_REF= +scenario: built-store-cross-node + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + # The address matters: the foundation template names the broker at 192.0.2.10:5671, and a token + # carries that verbatim as the endpoint an enrolling node dials. + # + # Sized like the one-node anchor: store, broker, registry, two control planes during the pivot, + # the builder with a Node toolchain and npm cache, and the built module images on top. + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 12GiB + cpus: 6 + disk: 60GiB + # The joined consumer node. It builds nothing — it pulls what the mesh's registry serves and what + # its modules name upstream (letta's app image comes from the internet, ~2GB), so it needs egress + # and room for images, not build horsepower. + node2: + at: { segment: hosting, address: [192.0.2.20] } + egress: true + inbound: allow + memory: 6GiB + cpus: 4 + disk: 40GiB + +place: + all: [host, runtime] diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts new file mode 100644 index 0000000..6029ce8 --- /dev/null +++ b/test/integration/built-store-cross-node.test.ts @@ -0,0 +1,287 @@ +/** + * THE NO-FAKE MULTI-NODE GATE: two machines, everything built by the mesh itself. + * + * The rewrite-based multi-node beds pre-stock runtime images and rewrite each manifest's + * placeholder digest to whatever they stocked — bed code doing the builder's job, which means the + * builder's job was never under test. This bed removes the shortcut. The scenario names NO images; + * the committed manifests are registered VERBATIM; the only thing that ever turns + * `mesh-runtime-@sha256:0000…` (or an `artifact:` reference) into a real digest is the mesh's + * own builder, exactly as in production. + * + * What it proves, end to end: + * 1. The installer raises `anchor` into a mesh of one — building the control plane (ADR 0073), + * standing up the registry and the builder. + * 2. The mesh BUILDS the shared base, then postgres and lavinmq, from the forge — and adopts the + * foundation's own store and broker as those modules (ADR 0078), the store's genesis superuser + * carried in through `secret accept` (the same act as hq phase3 deliverSuperuser). + * 3. `node2` joins, and its consumers are BUILT and delivered the same way: `amqp-ping` opens the + * one broker and `letta` gets a database on the one store — both across the overlay, admitted + * by the firewall the nftables module derives (issues 055/056/057 in one bed). + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap + * MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock (the TEMPLATE) + * MESH_LAB_CATALOG=.../mesh-catalog/modules + * MESH_LAB_SOURCE=git:///mesh-controller.git MESH_LAB_SOURCE_REF= + * MESH_LAB_BUILD_REF= + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec, push } from "../../src/lifecycle/operate.ts"; +import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts"; +import { genesis, type GenesisResult } from "./genesis.ts"; + +const SCENARIO = "built-store-cross-node"; +const CONTROL = "anchor"; +const NODE = "node2"; +const ANCHOR = "192.0.2.10"; +const REGISTRY = `${ANCHOR}:5000`; +const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" }; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; +const source = process.env["MESH_LAB_SOURCE"] ?? ""; +const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? ""; +const KEEP = !!process.env["MESH_LAB_KEEP"]; + +function forgeUrl(repo: string): string { + const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`]; + if (override) return override; + return source.replace(/[^/]+\.git$/, `${repo}.git`); +} +function refFor(repo: string): string { + const override = process.env[`MESH_LAB_BUILD_REF_${repo.toUpperCase().replaceAll("-", "_")}`]; + return override ?? process.env["MESH_LAB_BUILD_REF"] ?? "main"; +} +const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ?? + resolve(catalogDir, "..", "..", BASE.repo, "module.json"); + +const skip = + !capability.usable ? capability.why : + !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : + !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" : + !source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" : + !sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" : + !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" : + !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" : + false; + +let instanceId = ""; +let raised: GenesisResult; + +function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} +/** The control plane — retried through the brief recreate window a spec change causes. */ +async function mesh(command: string, timeoutMs?: number): Promise { + const deadline = Date.now() + (timeoutMs ?? 120_000); + for (;;) { + const got = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); + if (got.ok) return got.out; + if (!/is not running|No such container/.test(got.out) || Date.now() > deadline) { + throw new Error(`${CONTROL}: mesh ${command}\n${got.out}`); + } + await new Promise((r) => setTimeout(r, 5_000)); + } +} +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +/** Register a committed manifest VERBATIM — the point of this bed: no rewriting, ever. */ +async function registerModule(module: string, manifest: string): Promise { + assert.ok(existsSync(manifest), `no manifest for ${module} at ${manifest}`); + const onMachine = `/tmp/${module}.json`; + await push(instanceId, CONTROL, manifest, onMachine); + await must(CONTROL, `docker cp ${onMachine} mesh-controller:/${module}.json`); + return mesh(`module add /${module}.json`); +} + +/** Build a module with the mesh's own builder, issue its account, and assign it to a node. */ +async function buildAndAssign(module: string, node: string, opts?: { build?: boolean }): Promise { + await registerModule(module, resolve(catalogDir, module, "module.json")); + if (opts?.build !== false) { + const built = await mesh( + `build ${forgeUrl("mesh-catalog")} --path modules/${module} --ref ${refFor("mesh-catalog")} --wait 1200s`, + 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + } + const manifest = (await on(CONTROL, `docker exec mesh-controller cat /${module}.json`)).out; + if (manifest.includes("MESH_BROKER_FILE")) await mesh(`module issue ${module} --node ${node}`); + await mesh(`assign ${node} ${module}`); +} + +async function waitForContainer(node: string, container: string, seconds = 300): Promise { + const deadline = Date.now() + seconds * 1_000; + let last = ""; + while (Date.now() < deadline) { + const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + last = ps; + const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container); + if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps; + await new Promise((r) => setTimeout(r, 5_000)); + } + const logs = (await on(node, `docker logs --tail 20 ${container} 2>&1`)).out; + throw new Error(`${container} never came up on ${node}:\n${last}\n--- logs ---\n${logs}`); +} + +before(async () => { + if (skip) return; + const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = bed.instanceId; + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP)" : ""}`); + console.log("NOTHING WAS LOADED: this scenario names no images; the mesh builds or pulls everything."); + + // Genesis: the installer raises anchor into a mesh of one, BUILDING the control plane, and + // leaves the anchor enrolled with an agent running (hostService). + raised = await genesis({ + instanceId, + node: CONTROL, + installer: installer as string, + catalogDir, + bundleTemplate: foundationBundle(bundle, []), + registry: REGISTRY, + source, + sourceRef, + toolsSource: forgeUrl(BASE.repo), + toolsRef: refFor(BASE.repo), + catalogSource: forgeUrl("mesh-catalog"), + catalogRef: refFor("mesh-catalog"), + sdkSource: forgeUrl("mesh-sdk"), + sdkRef: refFor("mesh-sdk"), + site: "hosting", + hostService: true, + hostBinary: binary, + log: (m) => console.log(m), + }); + if (!raised.ok) throw new Error(`${raised.step || "genesis"}: ${raised.why}\n\n${raised.report.join("\n")}`); + + // node2 joins the mesh: a token against the anchor's public broker endpoint, then an agent. + await mesh(`node add ${NODE}`); + const token = tokenFrom(await mesh(`token issue --node ${NODE}`)); + const said = await must(NODE, `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, new RegExp(`enrolled as ${NODE}`), said); + await must(NODE, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); +}, { timeout: 3_000_000 }); + +after(async () => { + if (KEEP) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("a joined node's consumers open the store and broker the mesh built and adopted, over the overlay", { + skip, timeout: 3_600_000, +}, async () => { + // The overlay, so bindings carry `.internal` names; the firewall, so from:mesh is what admits them. + await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site lab`); + await mesh(`overlay place ${NODE} --site lab`); + await mesh(`assign ${CONTROL} networking`); + await mesh(`assign ${NODE} networking`); + + // The shared base first — every module with code of its own stands on it. + await registerModule(BASE.module, baseManifest); + const base = await mesh(`build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000); + assert.doesNotMatch(base, /failed/i, base); + + // Adopt the foundation store and broker as the postgres and lavinmq modules, BUILT by the mesh. + // The store's superuser is the foundation's, made at genesis — carried in through `secret accept` + // before the push, or the module mints one the running store does not know. + await buildAndAssign("nftables", CONTROL, { build: false }); + await buildAndAssign("postgres", CONTROL); + const superPw = (await must(CONTROL, + `docker inspect mesh-store --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^POSTGRES_PASSWORD=//p'`)).trim(); + await must(CONTROL, `printf %s ${quote(superPw)} > /tmp/superuser && docker cp /tmp/superuser mesh-controller:/superuser`); + await mesh(`secret accept ${CONTROL} postgres superuser --from /superuser`); + await buildAndAssign("lavinmq", CONTROL); + await mesh(`push ${CONTROL}`, 600_000); + await waitForContainer(CONTROL, "mesh-postgres", 600); + await waitForContainer(CONTROL, "mesh-lavinmq", 600); + + // The consumers on the joined node — built by the mesh, delivered from its registry. + await buildAndAssign("amqp-ping", NODE); + await buildAndAssign("letta", NODE); + await mesh(`push ${NODE}`, 900_000); + + // 057: the provider node is composed again so its provisioners learn of the remote consumers. + await mesh(`push ${CONTROL}`, 600_000); + + // THE BROKER, cross-node: amqp-ping's binding names the control-node's overlay name, its vhost is + // minted on the one broker, and it holds the connection. + await waitForContainer(NODE, "amqp-ping", 600); + const amqpBound = (await on(NODE, `cat /var/lib/amqp-ping/amqp.json 2>&1`)).out; + assert.match(amqpBound, new RegExp(`${CONTROL}\\.internal`), + `the amqp grant does not point at the control-node over the overlay:\n${amqpBound}`); + { + const deadline = Date.now() + 240_000; + let vhosts = ""; + while (Date.now() < deadline) { + vhosts = (await on(CONTROL, `docker exec mesh-broker lavinmqctl list_vhosts 2>&1`)).out; + if (new RegExp(`${NODE}|amqp-ping`).test(vhosts)) break; + await new Promise((r) => setTimeout(r, 5_000)); + } + assert.match(vhosts, new RegExp(`${NODE}|amqp-ping`), + `no vhost was minted on the one broker for the joined consumer:\n${vhosts}\n` + + `--- provisioner ---\n${(await on(CONTROL, `docker logs mesh-lavinmq 2>&1 | tail -20`)).out}\n` + + `--- consumer ---\n${(await on(NODE, `docker logs amqp-ping 2>&1 | tail -20`)).out}`); + } + + // THE STORE, cross-node: letta's binding names the control-node, and the login the mesh derived + // authenticates on the one store with the password the provisioner minted. (letta's own app needs + // pgvector and is not the claim here — the credential reaching a real database is.) + const bound = await (async () => { + const deadline = Date.now() + 240_000; + let raw = ""; + while (Date.now() < deadline) { + const got = await on(NODE, `cat /var/lib/letta/database.json 2>/dev/null`); + if (got.ok && /"as"/.test(got.out)) { raw = got.out; break; } + await new Promise((r) => setTimeout(r, 5_000)); + } + assert.match(raw, /"as"/, + `the mesh never wrote letta's database binding:\n${raw}\n` + + `--- store provisioner ---\n${(await on(CONTROL, `docker logs mesh-postgres 2>&1 | tail -20`)).out}`); + return JSON.parse(raw) as { as: string; at: string; provision: string }; + })(); + assert.equal(bound.provision, "postgres-database", `letta was bound the wrong provision: ${bound.provision}`); + assert.match(bound.at, new RegExp(`${CONTROL}\\.internal`), + `letta's database binding does not point at the control-node over the overlay: ${bound.at}`); + const pw = (await must(NODE, `cat /var/lib/letta/database.secret`)).trim(); + assert.ok(bound.as && pw, `letta's login or password was empty (as=${bound.as})`); + { + const conn = `postgresql://${bound.as}:${encodeURIComponent(pw)}@127.0.0.1:5432/${bound.as}?sslmode=disable`; + let pg = { out: "", ok: false }; + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + pg = await on(CONTROL, `docker exec mesh-postgres psql ${quote(conn)} -tAc 'select 1' 2>&1`); + if (pg.ok && /^1$/m.test(pg.out)) break; + if (/authentication failed/i.test(pg.out)) break; + await new Promise((r) => setTimeout(r, 5_000)); + } + assert.doesNotMatch(pg.out, /authentication failed/i, + `the store does not know the password the mesh delivered letta:\n${pg.out}`); + assert.match(pg.out, /^1$/m, `letta's login could not open its database on the one store:\n${pg.out}`); + } + + return `amqp: ${amqpBound.trim().slice(0, 120)}…\ndb: as=${bound.as} at=${bound.at}`; +}); From d3a6dc97843e81bea6ae5f15ccd1fff9f7033bd7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 22:23:13 +0200 Subject: [PATCH 02/10] The bed adopts only what genesis leaves it: the broker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 3 showed the Phase-3 installer already adopts postgres (superuser included), nftables and the catalogue at genesis — re-registering them was redundant. Only lavinmq and the joined node's consumers are the bed's to add. Issuance is now asserted per module and each module is pushed as it lands, mirroring the one-node bringUp. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- .../built-store-cross-node.test.ts | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts index 6029ce8..b323578 100644 --- a/test/integration/built-store-cross-node.test.ts +++ b/test/integration/built-store-cross-node.test.ts @@ -126,8 +126,13 @@ async function buildAndAssign(module: string, node: string, opts?: { build?: boo assert.doesNotMatch(built, /failed/i, built); } const manifest = (await on(CONTROL, `docker exec mesh-controller cat /${module}.json`)).out; - if (manifest.includes("MESH_BROKER_FILE")) await mesh(`module issue ${module} --node ${node}`); + if (manifest.includes("MESH_BROKER_FILE")) { + const issued = await mesh(`module issue ${module} --node ${node}`); + assert.match(issued, /scoped to what it (emits and consumes|consumes and emits)/, + `the broker account for ${module} was not issued:\n${issued}`); + } await mesh(`assign ${node} ${module}`); + await mesh(`push ${node}`, 600_000); } async function waitForContainer(node: string, container: string, seconds = 300): Promise { @@ -205,18 +210,11 @@ test("a joined node's consumers open the store and broker the mesh built and ado const base = await mesh(`build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000); assert.doesNotMatch(base, /failed/i, base); - // Adopt the foundation store and broker as the postgres and lavinmq modules, BUILT by the mesh. - // The store's superuser is the foundation's, made at genesis — carried in through `secret accept` - // before the push, or the module mints one the running store does not know. - await buildAndAssign("nftables", CONTROL, { build: false }); - await buildAndAssign("postgres", CONTROL); - const superPw = (await must(CONTROL, - `docker inspect mesh-store --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^POSTGRES_PASSWORD=//p'`)).trim(); - await must(CONTROL, `printf %s ${quote(superPw)} > /tmp/superuser && docker cp /tmp/superuser mesh-controller:/superuser`); - await mesh(`secret accept ${CONTROL} postgres superuser --from /superuser`); + // Genesis already adopted the store as the postgres module (superuser accepted), and installed + // nftables and the catalogue — verified rather than redone. The broker is the one foundation + // half genesis leaves to the mesh proper: adopt it here, BUILT by the mesh's own builder. + await waitForContainer(CONTROL, "mesh-postgres", 120); await buildAndAssign("lavinmq", CONTROL); - await mesh(`push ${CONTROL}`, 600_000); - await waitForContainer(CONTROL, "mesh-postgres", 600); await waitForContainer(CONTROL, "mesh-lavinmq", 600); // The consumers on the joined node — built by the mesh, delivered from its registry. From 428f13bfae2da95b8fd9f7ec6df85dae02840ab7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 22:39:48 +0200 Subject: [PATCH 03/10] The settle check records settling instead of inferring it from the clock The review found a race: a container that settled in the window's last seconds could be re-inspected past the deadline and failed as 'never stopped restarting'. A boolean now says what happened. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- test/integration/assigned-two-node-db.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/assigned-two-node-db.test.ts b/test/integration/assigned-two-node-db.test.ts index e826863..b3b0ffd 100644 --- a/test/integration/assigned-two-node-db.test.ts +++ b/test/integration/assigned-two-node-db.test.ts @@ -500,14 +500,14 @@ test("consumers on a joined node get their databases from the one foundation sto const stable = expected.filter((n) => n !== "letta"); for (const name of stable) { const deadline = Date.now() + 300_000; - let prev = -1, stableSince = 0, last = "?"; + let prev = -1, stableSince = 0, last = "?", settled = false; while (Date.now() < deadline) { const [running, count] = (await must(NODE, `docker inspect -f '{{.State.Running}} {{.RestartCount}}' ${name}`)).trim().split(" "); last = `running=${running} restarts=${count}`; const n = Number(count); if (running === "true" && n === prev) { if (stableSince === 0) stableSince = Date.now(); - if (Date.now() - stableSince >= 30_000) break; // up and unchanged for 30s — settled + if (Date.now() - stableSince >= 30_000) { settled = true; break; } // up and unchanged 30s } else { prev = n; stableSince = 0; } @@ -516,7 +516,7 @@ test("consumers on a joined node get their databases from the one foundation sto const [running, count] = (await must(NODE, `docker inspect -f '{{.State.Running}} {{.RestartCount}}' ${name}`)).trim().split(" "); assert.equal(running, "true", `${name} is not running after the push (${last}):\n${(await on(NODE, `docker logs ${name} 2>&1 | tail -40`)).out}`); - if (Date.now() >= deadline) + if (!settled) assert.fail(`${name} never stopped restarting within 300s (last ${last}) — a crash-loop, not startup churn:\n${(await on(NODE, `docker logs ${name} 2>&1 | tail -40`)).out}`); void count; } From 158fe5c3274f94418921e52522a0f273417e309d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 22:39:49 +0200 Subject: [PATCH 04/10] Apply the bed review: one buildable consumer, honest gates, tighter plumbing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit letta is not mesh-buildable (hq issue 060) — the store's cross-node proof stays with the stocked bed until a DB consumer gains a build section; amqp-ping carries the no-fake proof alone, and the node2 delivery is named as the honest red gate for issues 042/048 (no registry account, no registry trust). Also: overlay sites match genesis (hosting), the manifest is read locally instead of a swallowed docker-exec, before() gets the one-node budget, a node2 failure appends the host log (a failed pull never reaches container logs), and the foundation's survival plus the consumer's steadiness are asserted. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- .../built-store-cross-node.test.ts | 69 +++++++------------ 1 file changed, 26 insertions(+), 43 deletions(-) diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts index b323578..d69c605 100644 --- a/test/integration/built-store-cross-node.test.ts +++ b/test/integration/built-store-cross-node.test.ts @@ -125,7 +125,7 @@ async function buildAndAssign(module: string, node: string, opts?: { build?: boo 1_500_000); assert.doesNotMatch(built, /failed/i, built); } - const manifest = (await on(CONTROL, `docker exec mesh-controller cat /${module}.json`)).out; + const manifest = readFileSync(resolve(catalogDir, module, "module.json"), "utf8"); if (manifest.includes("MESH_BROKER_FILE")) { const issued = await mesh(`module issue ${module} --node ${node}`); assert.match(issued, /scoped to what it (emits and consumes|consumes and emits)/, @@ -188,7 +188,7 @@ before(async () => { const said = await must(NODE, `${HOST_PATH} enrol --token ${quote(token)}`); assert.match(said, new RegExp(`enrolled as ${NODE}`), said); await must(NODE, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); -}, { timeout: 3_000_000 }); +}, { timeout: 7_200_000 }); after(async () => { if (KEEP) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; } @@ -200,8 +200,8 @@ test("a joined node's consumers open the store and broker the mesh built and ado skip, timeout: 3_600_000, }, async () => { // The overlay, so bindings carry `.internal` names; the firewall, so from:mesh is what admits them. - await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site lab`); - await mesh(`overlay place ${NODE} --site lab`); + await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site hosting`); + await mesh(`overlay place ${NODE} --site hosting`); await mesh(`assign ${CONTROL} networking`); await mesh(`assign ${NODE} networking`); @@ -217,17 +217,26 @@ test("a joined node's consumers open the store and broker the mesh built and ado await buildAndAssign("lavinmq", CONTROL); await waitForContainer(CONTROL, "mesh-lavinmq", 600); - // The consumers on the joined node — built by the mesh, delivered from its registry. + // The consumer on the joined node — built by the mesh, delivered from its registry. One consumer, + // amqp-ping, because it is one of the eight modules the mesh can actually build; the catalogue's + // DB consumers all lack a build section (hq issue 060), so the store's cross-node proof stays with + // the stocked-image bed until one of them gains one. NOTE: this delivery is the path hq issues + // 042 (a node has no registry account) and 048 (nothing makes a machine trust the registry) leave + // open — this bed is their honest gate, red until they are fixed. await buildAndAssign("amqp-ping", NODE); - await buildAndAssign("letta", NODE); - await mesh(`push ${NODE}`, 900_000); // 057: the provider node is composed again so its provisioners learn of the remote consumers. await mesh(`push ${CONTROL}`, 600_000); // THE BROKER, cross-node: amqp-ping's binding names the control-node's overlay name, its vhost is // minted on the one broker, and it holds the connection. - await waitForContainer(NODE, "amqp-ping", 600); + try { + await waitForContainer(NODE, "amqp-ping", 600); + } catch (err) { + const hostLog = (await on(NODE, `tail -40 /var/log/mesh-host.log`)).out; + throw new Error(`${(err as Error).message}\n--- ${NODE} mesh-host.log (a pull that failed ` + + `never reaches container logs; hq issues 042/048) ---\n${hostLog}`); + } const amqpBound = (await on(NODE, `cat /var/lib/amqp-ping/amqp.json 2>&1`)).out; assert.match(amqpBound, new RegExp(`${CONTROL}\\.internal`), `the amqp grant does not point at the control-node over the overlay:\n${amqpBound}`); @@ -245,41 +254,15 @@ test("a joined node's consumers open the store and broker the mesh built and ado `--- consumer ---\n${(await on(NODE, `docker logs amqp-ping 2>&1 | tail -20`)).out}`); } - // THE STORE, cross-node: letta's binding names the control-node, and the login the mesh derived - // authenticates on the one store with the password the provisioner minted. (letta's own app needs - // pgvector and is not the claim here — the credential reaching a real database is.) - const bound = await (async () => { - const deadline = Date.now() + 240_000; - let raw = ""; - while (Date.now() < deadline) { - const got = await on(NODE, `cat /var/lib/letta/database.json 2>/dev/null`); - if (got.ok && /"as"/.test(got.out)) { raw = got.out; break; } - await new Promise((r) => setTimeout(r, 5_000)); - } - assert.match(raw, /"as"/, - `the mesh never wrote letta's database binding:\n${raw}\n` + - `--- store provisioner ---\n${(await on(CONTROL, `docker logs mesh-postgres 2>&1 | tail -20`)).out}`); - return JSON.parse(raw) as { as: string; at: string; provision: string }; - })(); - assert.equal(bound.provision, "postgres-database", `letta was bound the wrong provision: ${bound.provision}`); - assert.match(bound.at, new RegExp(`${CONTROL}\\.internal`), - `letta's database binding does not point at the control-node over the overlay: ${bound.at}`); - const pw = (await must(NODE, `cat /var/lib/letta/database.secret`)).trim(); - assert.ok(bound.as && pw, `letta's login or password was empty (as=${bound.as})`); - { - const conn = `postgresql://${bound.as}:${encodeURIComponent(pw)}@127.0.0.1:5432/${bound.as}?sslmode=disable`; - let pg = { out: "", ok: false }; - const deadline = Date.now() + 120_000; - while (Date.now() < deadline) { - pg = await on(CONTROL, `docker exec mesh-postgres psql ${quote(conn)} -tAc 'select 1' 2>&1`); - if (pg.ok && /^1$/m.test(pg.out)) break; - if (/authentication failed/i.test(pg.out)) break; - await new Promise((r) => setTimeout(r, 5_000)); - } - assert.doesNotMatch(pg.out, /authentication failed/i, - `the store does not know the password the mesh delivered letta:\n${pg.out}`); - assert.match(pg.out, /^1$/m, `letta's login could not open its database on the one store:\n${pg.out}`); + // Adoption did not cost the foundation: the containers genesis raised are still the ones running. + const up = await must(CONTROL, `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { + assert.match(up, new RegExp(`(^|\\n)${c}(\\n|$)`), `${c} did not survive adoption:\n${up}`); } + // Steady a moment, then confirm the consumer did not crash-loop after connecting. + await new Promise((r) => setTimeout(r, 15_000)); + assert.match((await on(NODE, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out, + /amqp-ping\tUp/, `amqp-ping did not stay up on ${NODE}`); - return `amqp: ${amqpBound.trim().slice(0, 120)}…\ndb: as=${bound.as} at=${bound.at}`; + return `amqp: ${amqpBound.trim().slice(0, 160)}`; }); From b77d03a1f842ad4509d387eb5a10f97ee03b84f8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 22:48:34 +0200 Subject: [PATCH 05/10] readFileSync is imported, not assumed https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- test/integration/built-store-cross-node.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts index d69c605..cbbbbf8 100644 --- a/test/integration/built-store-cross-node.test.ts +++ b/test/integration/built-store-cross-node.test.ts @@ -26,7 +26,7 @@ */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { resolve } from "node:path"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; From 27b5f8d092e139a79d39125cd71436889c77d69a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 22:52:31 +0200 Subject: [PATCH 06/10] The bed passes the typecheck gate https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- test/integration/built-store-cross-node.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts index cbbbbf8..3a4a3bc 100644 --- a/test/integration/built-store-cross-node.test.ts +++ b/test/integration/built-store-cross-node.test.ts @@ -177,7 +177,7 @@ before(async () => { sdkRef: refFor("mesh-sdk"), site: "hosting", hostService: true, - hostBinary: binary, + hostBinary: binary ?? undefined, log: (m) => console.log(m), }); if (!raised.ok) throw new Error(`${raised.step || "genesis"}: ${raised.why}\n\n${raised.report.join("\n")}`); @@ -264,5 +264,5 @@ test("a joined node's consumers open the store and broker the mesh built and ado assert.match((await on(NODE, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out, /amqp-ping\tUp/, `amqp-ping did not stay up on ${NODE}`); - return `amqp: ${amqpBound.trim().slice(0, 160)}`; + console.log(`amqp: ${amqpBound.trim().slice(0, 160)}`); }); From 832c287ccc60e94e65bcf3c52be0ef84a09d173b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 22:54:09 +0200 Subject: [PATCH 07/10] The bed passes the gate under exactOptionalPropertyTypes https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- test/integration/built-store-cross-node.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts index 3a4a3bc..72fb0fe 100644 --- a/test/integration/built-store-cross-node.test.ts +++ b/test/integration/built-store-cross-node.test.ts @@ -177,7 +177,7 @@ before(async () => { sdkRef: refFor("mesh-sdk"), site: "hosting", hostService: true, - hostBinary: binary ?? undefined, + ...(binary ? { hostBinary: binary } : {}), log: (m) => console.log(m), }); if (!raised.ok) throw new Error(`${raised.step || "genesis"}: ${raised.why}\n\n${raised.report.join("\n")}`); From 5ded8e2a8e8976906bfa99e761a3a1d392ae9bb1 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 22:55:00 +0200 Subject: [PATCH 08/10] The bed keeps its genesis warm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MESH_LAB_WARM=1 snapshots the post-genesis, both-nodes-enrolled state and restores it in seconds on later runs — refused, not silently rebuilt, when the commits have moved (src/warm.ts, the mechanism mesh.test.ts already uses and this session had ignored). Fresh stays the default. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- .../built-store-cross-node.test.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts index 72fb0fe..7f95ac1 100644 --- a/test/integration/built-store-cross-node.test.ts +++ b/test/integration/built-store-cross-node.test.ts @@ -34,6 +34,7 @@ import { destroy, exec, push } from "../../src/lifecycle/operate.ts"; import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts"; import { genesis, type GenesisResult } from "./genesis.ts"; +import { keep, ready, returnTo } from "../../src/warm.ts"; const SCENARIO = "built-store-cross-node"; const CONTROL = "anchor"; @@ -50,6 +51,13 @@ const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; const source = process.env["MESH_LAB_SOURCE"] ?? ""; const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? ""; const KEEP = !!process.env["MESH_LAB_KEEP"]; +/** + * MESH_LAB_WARM=1: restore the post-genesis, both-nodes-enrolled state from a snapshot instead of + * re-running the installer (minutes -> seconds), refused — not silently rebuilt — when the commits + * it was built from have moved (src/warm.ts). Fresh stays the default: a run that must MEAN + * something raises from nothing. + */ +const warming = process.env["MESH_LAB_WARM"] === "1"; function forgeUrl(repo: string): string { const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`]; @@ -151,6 +159,21 @@ async function waitForContainer(node: string, container: string, seconds = 300): before(async () => { if (skip) return; + if (warming) { + const said = await ready(SCENARIO); + if (said.use === "restore") { + instanceId = said.instanceId; + const seconds = await returnTo(instanceId); + // A snapshot captures disk, not memory: the restore reboots. anchor's host is a systemd + // service (genesis hostService) and comes back on its own; node2's was hand-started. + await must(NODE, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`); + const back = (await on(CONTROL, `docker ps --format '{{.Names}}'`)).out; + assert.match(back, /mesh-controller/, `the control plane did not come back after restore:\n${back}`); + console.log(`warm: returned ${instanceId} to its post-genesis state in ${seconds.toFixed(1)}s`); + return; + } + console.log(`warm: raising fresh — ${said.why}`); + } const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), }); @@ -188,9 +211,17 @@ before(async () => { const said = await must(NODE, `${HOST_PATH} enrol --token ${quote(token)}`); assert.match(said, new RegExp(`enrolled as ${NODE}`), said); await must(NODE, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + + // The expensive, deterministic part is done: genesis ran and both machines are enrolled. Keep it. + if (warming) { + const warm = await keep(SCENARIO, instanceId); + console.log(`warm: ${warm.instanceId} kept, against ` + + Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", ")); + } }, { timeout: 7_200_000 }); after(async () => { + if (warming) { console.log(`warm — ${instanceId} stays; \`mesh-lab warm cool\` retires it`); return; } if (KEEP) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; } if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); From ca263d2a8bdf2efe589327f110d230a9dc82a450 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 23:16:26 +0200 Subject: [PATCH 09/10] The trust lands before anything builds The networking module delivers the registry trust, so the bed pushes both machines after assigning it and waits for each runtime to actually hold the trust (file present AND the daemon reloaded) before the first build pushes to anchor.internal:5000. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- .../integration/built-store-cross-node.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts index 7f95ac1..b5f2235 100644 --- a/test/integration/built-store-cross-node.test.ts +++ b/test/integration/built-store-cross-node.test.ts @@ -236,6 +236,24 @@ test("a joined node's consumers open the store and broker the mesh built and ado await mesh(`assign ${CONTROL} networking`); await mesh(`assign ${NODE} networking`); + // The networking module carries the registry trust (ADR 0082): a merged daemon.json naming the + // store's internal name, and a docker restart when it first lands. It must be ON both machines + // before anything builds or pulls — the builder pushes to anchor.internal:5000 the moment the + // first build finishes. Pushed and WAITED for, because the restart bounces the runtime and an + // apply in flight retries. + for (const machine of [CONTROL, NODE]) { + await mesh(`push ${machine}`, 600_000); + const deadline = Date.now() + 300_000; + let trusted = false; + while (Date.now() < deadline) { + const got = await on(machine, `grep -s "anchor.internal:5000" /etc/docker/daemon.json && docker info --format '{{json .RegistryConfig.IndexConfigs}}' 2>/dev/null | grep -q "anchor.internal:5000" && echo TRUSTED`); + if (/TRUSTED/.test(got.out)) { trusted = true; break; } + await new Promise((r) => setTimeout(r, 5_000)); + } + assert.ok(trusted, `${machine}'s runtime never learned the registry trust:\n` + + (await on(machine, `cat /etc/docker/daemon.json 2>&1; docker info 2>&1 | tail -20`)).out); + } + // The shared base first — every module with code of its own stands on it. await registerModule(BASE.module, baseManifest); const base = await mesh(`build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000); From cb353f9881ea38e9ac88ac36c2cd701c340797c7 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 18 Sep 2026 00:15:16 +0200 Subject: [PATCH 10/10] The trust wait dumps mesh-host's log and the declaration on failure Run 11 failed with node2's daemon.json never written and nothing to say whether the declaration lacked the trust or never applied. The dump now answers that, and the push output is printed so a compose that refused is visible in the run log. --- test/integration/built-store-cross-node.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts index b5f2235..9f6cac7 100644 --- a/test/integration/built-store-cross-node.test.ts +++ b/test/integration/built-store-cross-node.test.ts @@ -242,7 +242,7 @@ test("a joined node's consumers open the store and broker the mesh built and ado // first build finishes. Pushed and WAITED for, because the restart bounces the runtime and an // apply in flight retries. for (const machine of [CONTROL, NODE]) { - await mesh(`push ${machine}`, 600_000); + console.log(await mesh(`push ${machine}`, 600_000)); const deadline = Date.now() + 300_000; let trusted = false; while (Date.now() < deadline) { @@ -250,8 +250,15 @@ test("a joined node's consumers open the store and broker the mesh built and ado if (/TRUSTED/.test(got.out)) { trusted = true; break; } await new Promise((r) => setTimeout(r, 5_000)); } + // A failure here has two distinguishable shapes, so the dump carries both: a declaration that + // never named the trust (the controller composed without it — issues 042/048 as a race), and + // one that named it and was never applied (delivery or apply). mesh-host's log says which. assert.ok(trusted, `${machine}'s runtime never learned the registry trust:\n` + - (await on(machine, `cat /etc/docker/daemon.json 2>&1; docker info 2>&1 | tail -20`)).out); + (await on(machine, `cat /etc/docker/daemon.json 2>&1; docker info 2>&1 | tail -20`)).out + + `\n--- ${machine} mesh-host.log ---\n` + + (await on(machine, `tail -60 /var/log/mesh-host.log 2>&1`)).out + + `\n--- ${machine} declared registry-trust? ---\n` + + (await on(machine, `base64 -d < /var/lib/mesh-host/declared.json 2>/dev/null | grep -c registry-trust; python3 -c "import json,base64; d=json.load(open('/var/lib/mesh-host/declared.json')); print('registry-trust' in base64.b64decode(d['declaration']).decode())" 2>&1`)).out); } // The shared base first — every module with code of its own stands on it.