/** * **A machine joins through the tunnel, and the bus is never public** (novox/hq ADR 0169). * * The anchor raises the foundation from the bundle, joins over its own loopback — it runs the bus, * so it needs no tunnel to reach it — and is placed as the hub. The joiner then does what a new * machine does: makes its tunnel key and prints the public half, is issued a token for that key, * and enrols with the token alone. The bus is closed to the joiner's own address on the anchor * before any of that, so the enrolment can arrive only over the tunnel the token gave it. * * It asserts the ADR's last row: *a new machine joins from outside the hub's network with the bus * closed to it.* The rows before it are the controller's own tests. * * It needs a host binary, the foundation bundle and the control plane's image: * * MESH_LAB_HOST_BINARY=.../mesh-host * MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock * (mesh-controller:development, from mesh-controller's `make image`) */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { existsSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" : false; const SCENARIO = "joins-through-the-tunnel"; /** The hub's tunnel port, and the bus's port as the bundle publishes it on the anchor. */ const HUB_PORT = 51820; const BUS_PORT = 5671; let instanceId = ""; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { const { stdout } = await exec(instanceId, machine, [ "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, ], timeoutMs); const marker = stdout.lastIndexOf("__exit="); if (marker < 0) return { out: stdout, ok: false }; return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; } async function must(machine: string, command: string, timeoutMs?: number): Promise { const { out, ok } = await on(machine, command, timeoutMs); if (!ok) throw new Error(`${machine}: ${command}\n${out}`); return out; } async function mesh(command: string, timeoutMs?: number): Promise { return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); } function tokenFrom(said: string): string { const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); assert.ok(found, `no token in:\n${said}`); return found; } async function until(what: string, within: number, check: () => Promise, why: () => Promise): Promise { const end = Date.now() + within; while (Date.now() < end) { if (await check()) return; await new Promise((r) => setTimeout(r, 3000)); } assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`); } /** * Put the mesh's composed user list where the anchor's bus reads it, and make it re-read. * * **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146): * the bus here is the bundle's, so an account the mesh composes reaches it only if whoever raised it * places it — the enrolment's when a token is issued, the node's own once it enrols. */ async function placeTheBusUsers(): Promise { await must("anchor", `docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` + `docker kill -s HUP mesh-broker >/dev/null`); } before(async () => { if (skip) return; const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`); await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000); const up = await must("anchor", `docker ps --format '{{.Names}}'`); for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`); } }, { timeout: 1_800_000 }); after(async () => { if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; } if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 600_000 }); test("the machine that runs the bus joins over its own loopback and becomes the hub", { skip, timeout: 900_000 }, async () => { await mesh("node add anchor"); const token = tokenFrom(await mesh("token issue --node anchor")); await placeTheBusUsers(); const said = await must("anchor", `${HOST_PATH} enrol --token ${quote(token)}`); assert.match(said, /enrolled as anchor/, said); assert.doesNotMatch(said, /tunnel to the hub/, `the machine running the bus went through a tunnel:\n${said}`); await placeTheBusUsers(); await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`); await mesh(`overlay place anchor --hub --endpoint 192.0.2.10:${HUB_PORT} --site lab`); await mesh("assign anchor mesh-wireguard"); await mesh("push anchor", 600_000); await until("the hub's tunnel coming up", 300_000, async () => (await on("anchor", `wg show mesh0 listen-port`)).out.trim() === String(HUB_PORT), async () => `--- anchor host ---\n${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`); // The hub's port, opened as the derived filter opens it — scenery here: the filter module is the // two-node walk's to prove, and the bundle's own filter admits only ssh, the bus and the registry. await must("anchor", `nft insert rule inet mesh input udp dport ${HUB_PORT} accept`); }); test("a machine joins through the tunnel, with the bus closed to it", { skip, timeout: 900_000 }, async () => { // **The bus closed to the joiner**, at the anchor's very first hook — before the container // runtime's forwarding — so nothing from its own address reaches the bus, by any path. await must("anchor", `nft add table ip lab_bus_closed && ` + `nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` + `nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport ${BUS_PORT} drop`); const reached = await on("joiner", `timeout 5 bash -c '/dev/null 2>&1`, 300_000); const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(); assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`); // Asked again, the same key: a token may already have been issued for it. assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key); // The token is issued for the key; the hub is sent the machine as a peer before it is shown. const issued = await mesh(`token issue --new joiner --overlay-key ${key}`, 600_000); const token = tokenFrom(issued); const carried = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as { broker: string; tunnel?: { key: string; address: string; hub_endpoint: string }; }; assert.equal(carried.tunnel?.key, key, `the token was not issued for the machine's key: ${JSON.stringify(carried)}`); assert.equal(carried.tunnel?.hub_endpoint, `192.0.2.10:${HUB_PORT}`); assert.doesNotMatch(carried.broker, /^192\.0\.2\./, `the token sends the machine to the bus's public address: ${carried.broker}`); await placeTheBusUsers(); const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`, 300_000); assert.match(said, /the tunnel to the hub is up/, said); assert.match(said, /the hub answered the tunnel/, said); assert.match(said, /enrolled as joiner/, said); const shakes = await must("joiner", `wg show mesh0 latest-handshakes`); assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`); // And the bus is still closed to it: it joined through the tunnel, not around it. assert.ok(!(await on("joiner", `timeout 5 bash -c '