diff --git a/README.md b/README.md index fdbab51..38cbe33 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,10 @@ mesh-lab diagram --live draw what is actually standing correctly and snapshots roughly 76× slower — which does not make the lab slow, it makes it unused, and a warning about that is read once and ignored forever. +Behind a VPN client that routes every private range, incus finds no free range for the uplink a +machine fetches its images over, and nothing can be raised. Name one the host does not route: +`MESH_LAB_UPLINK_V4=192.168.231.1/24`. + If the incus socket is not reachable as your user — the group was granted to a session that already existed — set `MESH_LAB_INCUS="sudo -n incus"`. diff --git a/scenarios/joins-through-the-tunnel.yml b/scenarios/joins-through-the-tunnel.yml new file mode 100644 index 0000000..98ecc9f --- /dev/null +++ b/scenarios/joins-through-the-tunnel.yml @@ -0,0 +1,33 @@ +# A machine joins through the tunnel, with the bus closed to it (novox/hq ADR 0169). +# +# The anchor raises the foundation from the bundle, joins over its own loopback and becomes the hub. +# The joiner has a host and nothing else: it makes its tunnel key, a token is issued for that key, +# and it enrols. The bus is closed to the joiner's own address on the anchor, so the only way its +# enrolment can arrive is over the tunnel the token gave it. +scenario: joins-through-the-tunnel + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + joiner: + at: { segment: hosting, address: [192.0.2.30] } + egress: true + inbound: allow + memory: 1GiB + # It carries none of the images: it only has to join. + images: [] + +images: + - mesh-controller:development + +place: + all: [host, runtime] diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index cce110a..cc7d769 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -28,13 +28,6 @@ machines: egress: true inbound: allow memory: 2GiB - # A third machine that joins through the tunnel with the bus closed to it (novox/hq ADR 0169). It - # carries none of the images: it only has to join. - joiner: - at: { segment: hosting, address: [192.0.2.30] } - egress: true - inbound: allow - images: [] images: # The packet filter's seat runtime (novox/hq ADR 0170): the filter module now serves its verbs from diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index 908237f..5c53e1e 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -135,6 +135,18 @@ async function createNetwork( return name; } +/** + * The uplink's own address and prefix: incus picks a free one, unless `MESH_LAB_UPLINK_V4` names it. + * + * **A workstation behind a VPN client may have no free range left to pick.** A corporate client that + * routes all of 10.0.0.0/8 and 172.16.0.0/12 leaves incus refusing with "failed to automatically find + * an unused IPv4 subnet", and no scenario can be raised at all. Naming one the host does not route — + * `192.168.231.1/24` — is the way past it; it is the host's fact, never the scenario's. + */ +export function uplinkAddress(env: NodeJS.ProcessEnv = process.env): string { + return env["MESH_LAB_UPLINK_V4"]?.trim() || "auto"; +} + /** * The one network the lab supplies rather than the declaration. * @@ -153,7 +165,7 @@ async function createUplink(instanceId: string): Promise { if (await succeeds(["network", "show", name], 15_000)) return name; await incus([ "network", "create", name, - "ipv4.address=auto", + `ipv4.address=${uplinkAddress()}`, "ipv4.nat=true", "ipv6.address=none", `user.mesh-lab.instance=${instanceId}`, diff --git a/test/integration/joins-through-the-tunnel.test.ts b/test/integration/joins-through-the-tunnel.test.ts new file mode 100644 index 0000000..08e687f --- /dev/null +++ b/test/integration/joins-through-the-tunnel.test.ts @@ -0,0 +1,179 @@ +/** + * **A machine joins through the tunnel, and the bus is never public** (novox/hq ADR 0169). + * + * The anchor raises the foundation from the bundle, joins over its own loopback — it runs the bus, + * so it needs no tunnel to reach it — and is placed as the hub. The joiner then does what a new + * machine does: makes its tunnel key and prints the public half, is issued a token for that key, + * and enrols with the token alone. The bus is closed to the joiner's own address on the anchor + * before any of that, so the enrolment can arrive only over the tunnel the token gave it. + * + * It asserts the ADR's last row: *a new machine joins from outside the hub's network with the bus + * closed to it.* The rows before it are the controller's own tests. + * + * It needs a host binary, the foundation bundle and the control plane's image: + * + * MESH_LAB_HOST_BINARY=.../mesh-host + * MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock + * (mesh-controller:development, from mesh-controller's `make image`) + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" + : false; + +const SCENARIO = "joins-through-the-tunnel"; +/** The hub's tunnel port, and the bus's port as the bundle publishes it on the anchor. */ +const HUB_PORT = 51820; +const BUS_PORT = 5671; +let instanceId = ""; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} + +async function mesh(command: string, timeoutMs?: number): Promise { + return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +async function until(what: string, within: number, check: () => Promise, why: () => Promise): Promise { + const end = Date.now() + within; + while (Date.now() < end) { + if (await check()) return; + await new Promise((r) => setTimeout(r, 3000)); + } + assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`); +} + +/** + * Put the mesh's composed user list where the anchor's bus reads it, and make it re-read. + * + * **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146): + * the bus here is the bundle's, so an account the mesh composes reaches it only if whoever raised it + * places it — the enrolment's when a token is issued, the node's own once it enrols. + */ +async function placeTheBusUsers(): Promise { + await must("anchor", + `docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` + + `docker kill -s HUP mesh-broker >/dev/null`); +} + +before(async () => { + if (skip) return; + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + + await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`); + await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000); + const up = await must("anchor", `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { + assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`); + } +}, { timeout: 1_800_000 }); + +after(async () => { + if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 600_000 }); + +test("the machine that runs the bus joins over its own loopback and becomes the hub", { skip, timeout: 900_000 }, async () => { + await mesh("node add anchor"); + const token = tokenFrom(await mesh("token issue --node anchor")); + await placeTheBusUsers(); + const said = await must("anchor", `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, /enrolled as anchor/, said); + assert.doesNotMatch(said, /tunnel to the hub/, `the machine running the bus went through a tunnel:\n${said}`); + await placeTheBusUsers(); + await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`); + + await mesh(`overlay place anchor --hub --endpoint 192.0.2.10:${HUB_PORT} --site lab`); + await mesh("assign anchor mesh-wireguard"); + await mesh("push anchor", 600_000); + await until("the hub's tunnel coming up", 300_000, + async () => (await on("anchor", `wg show mesh0 listen-port`)).out.trim() === String(HUB_PORT), + async () => `--- anchor host ---\n${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`); + // The hub's port, opened as the derived filter opens it — scenery here: the filter module is the + // two-node walk's to prove, and the bundle's own filter admits only ssh, the bus and the registry. + await must("anchor", `nft insert rule inet mesh input udp dport ${HUB_PORT} accept`); +}); + +test("a machine joins through the tunnel, with the bus closed to it", { skip, timeout: 900_000 }, async () => { + // **The bus closed to the joiner**, at the anchor's very first hook — before the container + // runtime's forwarding — so nothing from its own address reaches the bus, by any path. + await must("anchor", `nft add table ip lab_bus_closed && ` + + `nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` + + `nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport ${BUS_PORT} drop`); + const reached = await on("joiner", `timeout 5 bash -c '/dev/null 2>&1`, 300_000); + const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(); + assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`); + // Asked again, the same key: a token may already have been issued for it. + assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key); + + // The token is issued for the key; the hub is sent the machine as a peer before it is shown. + const issued = await mesh(`token issue --new joiner --overlay-key ${key}`, 600_000); + const token = tokenFrom(issued); + const carried = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as { + broker: string; tunnel?: { key: string; address: string; hub_endpoint: string }; + }; + assert.equal(carried.tunnel?.key, key, `the token was not issued for the machine's key: ${JSON.stringify(carried)}`); + assert.equal(carried.tunnel?.hub_endpoint, `192.0.2.10:${HUB_PORT}`); + assert.doesNotMatch(carried.broker, /^192\.0\.2\./, `the token sends the machine to the bus's public address: ${carried.broker}`); + await placeTheBusUsers(); + + const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`, 300_000); + assert.match(said, /the tunnel to the hub is up/, said); + assert.match(said, /the hub answered the tunnel/, said); + assert.match(said, /enrolled as joiner/, said); + + const shakes = await must("joiner", `wg show mesh0 latest-handshakes`); + assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`); + // And the bus is still closed to it: it joined through the tunnel, not around it. + assert.ok(!(await on("joiner", `timeout 5 bash -c ' { - await must("anchor", `nft add table ip lab_bus_closed && ` + - `nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` + - `nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport 4222 drop`); - try { - await must("joiner", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`); - const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(); - assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`); - // Asked again, the same key: a token may already have been issued for it. - assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key); - - const token = tokenFrom(await mesh(`token issue --new joiner --overlay-key ${key}`)); - await composeTheBusUsers(); - const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`); - assert.match(said, /the tunnel to the hub is up/, said); - assert.match(said, /enrolled as joiner/, said); - - const shakes = await must("joiner", `wg show mesh0 latest-handshakes`); - assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`); - } finally { - await on("anchor", `nft delete table ip lab_bus_closed`); - } -}); - +// A machine joining through the tunnel with the bus closed to it is its own bed: +// joins-through-the-tunnel.test.ts (novox/hq ADR 0169). diff --git a/test/uplink.test.ts b/test/uplink.test.ts new file mode 100644 index 0000000..7d6976d --- /dev/null +++ b/test/uplink.test.ts @@ -0,0 +1,10 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { uplinkAddress } from "../src/lifecycle/raise.ts"; + +// A workstation behind a VPN client that routes every private range leaves incus nothing to pick, so +// the uplink's range may be named; otherwise incus picks it. +test("the uplink's range is incus's to pick unless the host names one", () => { + assert.equal(uplinkAddress({}), "auto"); + assert.equal(uplinkAddress({ MESH_LAB_UPLINK_V4: " 192.168.231.1/24 " }), "192.168.231.1/24"); +});