From 155800bc9aef1a2c9ef582b5135b9487813b9e4d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 01:01:54 +0200 Subject: [PATCH 1/2] A two-node bed: a joined node opening the adopted broker over the overlay (055) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit anchor raises the foundation and adopts lavinmq; node2 joins and runs amqp-ping, which requires amqp and provides nothing. Asserts the grant names anchor.internal, a vhost is minted on the far broker, and the consumer stays up. Currently RED: it caught two real gaps — the broker's amqps port not in the firewall (fixed in mesh-catalog) and the module broker URL using the public address not the overlay (issue 055, needs a controller fix). Goes green when 055 is fixed. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- scenarios/adopted-store-cross-node.yml | 48 +++++ .../adopted-store-cross-node.test.ts | 200 ++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 scenarios/adopted-store-cross-node.yml create mode 100644 test/integration/adopted-store-cross-node.test.ts diff --git a/scenarios/adopted-store-cross-node.yml b/scenarios/adopted-store-cross-node.yml new file mode 100644 index 0000000..2229344 --- /dev/null +++ b/scenarios/adopted-store-cross-node.yml @@ -0,0 +1,48 @@ +# Cross-node reachability of the ADOPTED store/broker (novox/hq issue 055). +# +# The foundation (store, broker, control) lives on `anchor`, the control-node, and the `lavinmq` +# module is adopted THERE — so `mesh-broker` is the one shared broker. `node2` joins the mesh and +# runs ONLY `amqp-ping`, a consumer that requires `amqp`. Nothing on node2 provides amqp; the grant +# it gets must resolve to the broker on anchor, reached over the overlay by anchor's `.internal` +# name. This is the shape no other bed has: a consumer on a different node than the provider. +scenario: adopted-store-cross-node + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + # The control-node: foundation + the adopted broker's provisioner. mesh-controller:development is + # the foundation's control-plane image; mesh-runtime-lavinmq runs the lavinmq module's provisioner + # that mints the consumer's vhost on the adopted mesh-broker. + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 4GiB + cpus: 4 + disk: 20GiB + images: + - mesh-controller:development + - mesh-runtime-lavinmq:development + # The joined node: the amqp consumer, and nothing that provides amqp. + node2: + at: { segment: hosting, address: [192.0.2.20] } + egress: true + inbound: allow + memory: 4GiB + cpus: 4 + disk: 20GiB + images: + - mesh-runtime-amqp-ping:development + +# Every image the scenario stocks (validated against the per-machine lists above). +images: + - mesh-controller:development + - mesh-runtime-lavinmq:development + - mesh-runtime-amqp-ping:development + +# Place the host binary and the stocked runtimes on every machine (as whole-mesh-novox does). +place: + all: [host, runtime] diff --git a/test/integration/adopted-store-cross-node.test.ts b/test/integration/adopted-store-cross-node.test.ts new file mode 100644 index 0000000..8a23bbb --- /dev/null +++ b/test/integration/adopted-store-cross-node.test.ts @@ -0,0 +1,200 @@ +/** + * CROSS-NODE REACHABILITY OF THE ADOPTED BROKER (novox/hq issue 055). + * + * Phase 3 made the foundation's broker the ordinary `lavinmq` module, adopted in place on the + * control-node ([ADR 0078](../../../hq/02-DECISIONS/0078-the-store-and-broker-are-modules.md)) — one + * broker, bound mesh-wide. Every existing two-node bed co-locates a provider with its consumer, so + * the one thing none of them prove is the whole point of "one broker": a consumer on a DIFFERENT + * node opening the shared broker over the overlay. + * + * This bed does exactly that. `anchor` is the control-node: it raises the foundation (mesh-store, + * mesh-broker, mesh-controller) and adopts `lavinmq` there, so `mesh-broker` is the one broker. + * `node2` joins the mesh and runs ONLY `amqp-ping`, which `requires: [amqp]` and provides nothing. + * The grant it gets must resolve to the broker on anchor and reach it by anchor's `.internal` + * overlay name. The assertions are the proof: amqp-ping's bound file names `anchor.internal`, its + * vhost exists on anchor's `mesh-broker`, and its container stays up (it connected). + * + * MESH_LAB_INCUS='sudo -n incus' + * MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host + * MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock + * MESH_LAB_CATALOG=.../mesh-catalog/modules + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; +const skip = !capability.usable ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle" + : false; + +const SCENARIO = "adopted-store-cross-node"; +const NODE = "node2"; +const catalogDir = process.env["MESH_LAB_CATALOG"] + ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") + ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + +let instanceId = ""; +let held: HeldImage[] = []; + +function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} +/** The control plane, a container on anchor. */ +async function mesh(command: string, timeoutMs?: number): Promise { + return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); +} +function pinned(reference: string): string { return onTheMachine(reference, held); } +function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); } + +/** Load a committed module.json with its container images rewritten to the scenario's pinned digests. */ +function loadManifest(name: string): { manifest: string; broker: boolean } { + const path = resolve(catalogDir, name, "module.json"); + const m = JSON.parse(readFileSync(path, "utf8")) as { + resources?: { type: string; image?: string; artifact?: string }[]; + }; + for (const r of m.resources ?? []) { + if (r.type !== "container") continue; + if (typeof r.image === "string") { + // A placeholder image (mesh-runtime-@0…0) resolves to the stocked digest, as redis does. + r.image = pinned(r.image); + } else if (typeof r.artifact === "string") { + // The bundle-model bed does not build, so resolve a module's runtime ARTIFACT to its stocked + // image directly — postgres/lavinmq name their provisioner by artifact, not a placeholder. + r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`); + delete r.artifact; + } + } + const manifest = JSON.stringify(m); + return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; +} +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} +/** Register a catalog module, issue its broker account if it needs one, and assign it to a node. */ +async function install(name: string, node: string): Promise { + const { manifest, broker } = loadManifest(name); + await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`); + await mesh(`module add /${name}.json`); + if (broker) await mesh(`module issue ${name} --node ${node}`); + await mesh(`assign ${node} ${name}`); +} + +before(async () => { + if (skip) return; + assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`) }); + instanceId = raised.instanceId; + held = raised.images; + + // anchor raises the foundation from its bundle — store, broker and control plane. + await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); + await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000); + const up = await must("anchor", `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { + assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`); + } + + // Both machines join the one mesh and run a host so they apply what they are pushed. + for (const machine of ["anchor", NODE]) { + await mesh(`node add ${machine}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, new RegExp(`enrolled as ${machine}`), said); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + } +}, { timeout: 2_700_000 }); + +after(async () => { + if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("a consumer on a joined node opens the adopted broker on the control-node over the overlay", { + skip, timeout: 2_400_000, +}, async () => { + // The overlay, so the provider's binding address is its `.internal` overlay name, not loopback. + await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); + await mesh(`overlay place ${NODE} --site lab`); + await mesh("assign anchor networking"); + await mesh(`assign ${NODE} networking`); + // The packet filter on the control-node, so the from:mesh rule (not a bare 0.0.0.0 bind) is what + // admits node2 to the broker — the firewall half of issue 055. + await install("nftables", "anchor"); + + // Adopt the broker as the lavinmq module ON ANCHOR: same mesh-broker container, plus its + // provisioner that mints a vhost per consumer. + await install("lavinmq", "anchor"); + await mesh(`push anchor`, 600_000); + + // The consumer on the joined node — requires amqp, provides nothing. + await install("amqp-ping", NODE); + await mesh(`push ${NODE}`, 900_000); + + // amqp-ping's container comes up and stays up (a broker it could not reach would crash-loop it). + const psName = async () => (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + const deadline = Date.now() + 600_000; + let ps = ""; + while (Date.now() < deadline) { + ps = await psName(); + if (/amqp-ping\s+Up/.test(ps)) break; + await new Promise((r) => setTimeout(r, 8_000)); + } + assert.match(ps, /amqp-ping\s+Up/, `amqp-ping did not come up on ${NODE}:\n${ps}`); + + // THE PROOF (055): the binding written to the joined consumer names the control-node's overlay + // address, not a loopback or the consumer's own node. + const bound = (await on(NODE, `cat /var/lib/amqp-ping/amqp.json 2>&1`)).out; + assert.match(bound, /anchor\.internal/, + `the amqp grant does not point at the control-node over the overlay:\n${bound}`); + + // And it is real on the far side: a vhost for this consumer exists on anchor's mesh-broker. Poll, + // because the provider's provisioner reconciles the remote consumer's grant a moment after push. + const vhostDeadline = Date.now() + 120_000; + let vhosts = ""; + while (Date.now() < vhostDeadline) { + vhosts = (await on("anchor", `docker exec mesh-broker lavinmqctl list_vhosts 2>&1`)).out; + if (/amqp-ping|node2/.test(vhosts)) break; + await new Promise((r) => setTimeout(r, 5_000)); + } + if (!/amqp-ping|node2/.test(vhosts)) { + // Diagnostics for a cross-node provisioning gap: what the provider's provisioner was given, and + // what it and the consumer logged. + const grants = (await on("anchor", `cat /var/lib/lavinmq-module/grants/mesh.json 2>&1`)).out; + const provLog = (await on("anchor", `docker logs mesh-lavinmq 2>&1 | tail -30`)).out; + const pingLog = (await on(NODE, `docker logs amqp-ping 2>&1 | tail -30`)).out; + assert.fail(`no vhost minted on the control-node's broker for the joined consumer.\n` + + `vhosts:\n${vhosts}\n\nprovisioner grants (anchor):\n${grants}\n\n` + + `mesh-lavinmq log:\n${provLog}\n\namqp-ping log:\n${pingLog}`); + } + + // Steady a moment, then confirm it did not crash-loop after connecting. + await new Promise((r) => setTimeout(r, 15_000)); + assert.match(await psName(), /amqp-ping\s+Up/, `amqp-ping did not stay up on ${NODE}`); + + return `bound: ${bound.trim()}\nvhosts: ${vhosts.trim()}`; +}); -- 2.54.0 From b6bf31d4e6968223c31d4e443feefe26234b76c7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 01:34:22 +0200 Subject: [PATCH 2/2] The cross-node bed is green: push the provider node after adding the remote consumer A provision secret is minted as a side-effect of composing the CONSUMER's plan, and the provider's grant list is a pure read of secrets already issued from it. So a cross-node consumer's grant exists only after its node is pushed, and the provider's provisioner mints the vhost only when the provider node is composed again. Push anchor once more after node2, and the bed passes: amqp-ping on node2 reaches mesh-broker on anchor over the overlay, its binding names anchor.internal, and its vhost is minted. Proves both halves of issue 055. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- test/integration/adopted-store-cross-node.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/integration/adopted-store-cross-node.test.ts b/test/integration/adopted-store-cross-node.test.ts index 8a23bbb..522d6bf 100644 --- a/test/integration/adopted-store-cross-node.test.ts +++ b/test/integration/adopted-store-cross-node.test.ts @@ -155,6 +155,12 @@ test("a consumer on a joined node opens the adopted broker on the control-node o await install("amqp-ping", NODE); await mesh(`push ${NODE}`, 900_000); + // The consumer minted its grant against anchor's broker only when node2 was composed + // (a provision secret is a side-effect of composing the CONSUMER). anchor's provisioner + // learns of a cross-node consumer only when anchor is composed again, so push it once more to + // mint the vhost. Adding a cross-node consumer means pushing the provider node too (issue 055). + await mesh(`push anchor`, 600_000); + // amqp-ping's container comes up and stays up (a broker it could not reach would crash-loop it). const psName = async () => (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; const deadline = Date.now() + 600_000; -- 2.54.0