From d2c6e8fc5ad5e50483bd87153be0a521eddf8b3f Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 2 Oct 2026 15:11:18 +0200 Subject: [PATCH 1/6] The two-node bed proves a machine joins through the tunnel with the bus closed to it A third machine makes its tunnel key, is issued a token for it, and enrols while the anchor drops its packets to the bus at the first hook; it can only have arrived over the tunnel (novox/hq ADR 0169). --- scenarios/two-nodes.yml | 7 +++++++ test/integration/mesh.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index c046332..17055ee 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -28,6 +28,13 @@ machines: egress: true inbound: allow memory: 2GiB + # A third machine that joins through the tunnel with the bus closed to it (novox/hq ADR 0169). It + # carries none of the images: it only has to join. + joiner: + at: { segment: hosting, address: [192.0.2.30] } + egress: true + inbound: allow + images: [] images: - mesh-controller:development diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index 6287ba8..ea878bf 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -1662,3 +1662,30 @@ test("a third-party workload is adopted, with the credential it already had", { // a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy // assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into // mesh-grant-end-to-end, against the catalogue's redis. + +// **A machine joins through the tunnel, and needs the bus only through it** (novox/hq ADR 0169). +// The bus is closed to this machine at the anchor's very first hook, before the container runtime's +// forwarding, so the only way its enrolment can arrive is over the tunnel the token gave it. +test("a machine joins through the tunnel, with the bus closed to it", { skip, timeout: 900_000 }, async () => { + await must("anchor", `nft add table ip lab_bus_closed && ` + + `nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` + + `nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport 4222 drop`); + try { + await must("joiner", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`); + const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(); + assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`); + // Asked again, the same key: a token may already have been issued for it. + assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key); + + const token = tokenFrom(await mesh(`token issue --new joiner --overlay-key ${key}`)); + const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, /the tunnel to the hub is up/, said); + assert.match(said, /enrolled as joiner/, said); + + const shakes = await must("joiner", `wg show mesh0 latest-handshakes`); + assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`); + } finally { + await on("anchor", `nft delete table ip lab_bus_closed`); + } +}); + -- 2.54.0 From 960539066e2fd471a61a3820bac32e73af3a2aa1 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 2 Oct 2026 16:29:21 +0200 Subject: [PATCH 2/6] The two-node bed places the bus's users as genesis must This bed raises genesis by hand, so it places the composed user list after each token and each enrolment, as the trust bed does (novox/hq issue 146); without it the first join was refused. --- test/integration/mesh.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index ea878bf..73210ad 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -193,6 +193,20 @@ function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); } +/** + * Put the mesh's composed user list where the anchor's bus reads it, and make it re-read. + * + * **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146). + * The bus here is the bundle's, not a module the mesh delivers, so an account the mesh composes — + * the enrolment when a token is issued, the node's own when it enrols — reaches it only if whoever + * raised it places it. Without this the first join is refused with an authorisation violation. + */ +async function composeTheBusUsers(): Promise { + await must("anchor", + `docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` + + `docker kill -s HUP mesh-broker >/dev/null`); +} + /** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */ function tokenFrom(said: string): string { const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); @@ -311,9 +325,11 @@ test("both machines join it, and the token is all they need", { skip, timeout: 9 for (const [machine, node] of [["anchor", "anchor"], ["laptop", "laptop"]] as const) { await mesh(`node add ${node}`); const token = tokenFrom(await mesh(`token issue --node ${node}`)); + await composeTheBusUsers(); // No --name. The token says what the mesh calls the machine, which is the fault this walk // found the first time it was run. const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); + await composeTheBusUsers(); assert.match(said, new RegExp(`enrolled as ${node}`), said); assert.match(said, /sealing key/, "no sealing key was generated"); } @@ -1678,6 +1694,7 @@ test("a machine joins through the tunnel, with the bus closed to it", { skip, ti assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key); const token = tokenFrom(await mesh(`token issue --new joiner --overlay-key ${key}`)); + await composeTheBusUsers(); const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`); assert.match(said, /the tunnel to the hub is up/, said); assert.match(said, /enrolled as joiner/, said); -- 2.54.0 From db0069f262014092949a39afa70d8d16a670561a Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 2 Oct 2026 16:41:00 +0200 Subject: [PATCH 3/6] The two-node bed joins its second machine through the tunnel A token carries the bus's address, and at genesis that is the anchor's loopback, which no other machine reaches. The anchor joins locally and becomes the hub; the laptop makes its tunnel key, is issued a token for it and joins over the tunnel (novox/hq ADR 0169, issue 146). --- test/integration/mesh.test.ts | 42 ++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index 73210ad..c6dccbf 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -322,17 +322,37 @@ test("a bare machine becomes a mesh", { skip, timeout: 600_000 }, async () => { }); test("both machines join it, and the token is all they need", { skip, timeout: 900_000 }, async () => { - for (const [machine, node] of [["anchor", "anchor"], ["laptop", "laptop"]] as const) { - await mesh(`node add ${node}`); - const token = tokenFrom(await mesh(`token issue --node ${node}`)); - await composeTheBusUsers(); - // No --name. The token says what the mesh calls the machine, which is the fault this walk - // found the first time it was run. - const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); - await composeTheBusUsers(); - assert.match(said, new RegExp(`enrolled as ${node}`), said); - assert.match(said, /sealing key/, "no sealing key was generated"); - } + // **The anchor runs the bus, so it joins over its own loopback.** Every other machine joins + // through the tunnel (novox/hq ADR 0169): the anchor is the hub, and its tunnel comes up first. + await mesh(`node add anchor`); + const first = tokenFrom(await mesh(`token issue --node anchor`)); + await composeTheBusUsers(); + // No --name. The token says what the mesh calls the machine, which is the fault this walk + // found the first time it was run. + const anchorSaid = await must("anchor", `${HOST_PATH} enrol --token ${quote(first)}`); + await composeTheBusUsers(); + assert.match(anchorSaid, /enrolled as anchor/, anchorSaid); + assert.match(anchorSaid, /sealing key/, "no sealing key was generated"); + + await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); + await mesh("assign anchor networking"); + await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`); + await mesh("push anchor"); + await new Promise((r) => setTimeout(r, 8000)); + await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on }); + + // **The laptop makes its tunnel key, and the token is issued for it.** The hub is told the key + // before the token is shown, so the tunnel answers the first time the laptop knocks. + await must("laptop", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`); + const key = (await must("laptop", `${HOST_PATH} key 2>/dev/null`)).trim(); + await mesh(`node add laptop`); + const second = tokenFrom(await mesh(`token issue --node laptop --overlay-key ${key}`)); + await composeTheBusUsers(); + const laptopSaid = await must("laptop", `${HOST_PATH} enrol --token ${quote(second)}`); + await composeTheBusUsers(); + assert.match(laptopSaid, /the tunnel to the hub is up/, laptopSaid); + assert.match(laptopSaid, /enrolled as laptop/, laptopSaid); + assert.match(laptopSaid, /sealing key/, "no sealing key was generated"); const recorded = await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` + -- 2.54.0 From 8fdbf9ca90e060fcf772f1723998789d83557a51 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 2 Oct 2026 16:46:16 +0200 Subject: [PATCH 4/6] The two-node bed waits for the anchor's first apply before its filter --- test/integration/mesh.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index c6dccbf..351f88f 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -338,7 +338,14 @@ test("both machines join it, and the token is all they need", { skip, timeout: 9 await mesh("assign anchor networking"); await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`); await mesh("push anchor"); - await new Promise((r) => setTimeout(r, 8000)); + // The host reports which of its links face outside with every apply, and the filter is not + // composed for a machine that has not — so its first apply is waited for, not slept past. + const appliedBy = Date.now() + 180_000; + while (!(await on("anchor", `grep -q 'applied [0-9]' /var/log/mesh-host.log`)).ok) { + if (Date.now() > appliedBy) assert.fail(`the anchor never applied what it was sent:\n${(await on("anchor", "tail -30 /var/log/mesh-host.log")).out}`); + await new Promise((r) => setTimeout(r, 3000)); + } + await new Promise((r) => setTimeout(r, 5000)); await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on }); // **The laptop makes its tunnel key, and the token is issued for it.** The hub is told the key -- 2.54.0 From 4b7ad9a387c6b572e4e4df8a300e398c99c8a4fe Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 2 Oct 2026 18:08:04 +0200 Subject: [PATCH 5/6] The two-node bed stocks the packet filter's seat runtime The filter module now serves its verbs from a runtime the mesh builds (novox/hq ADR 0170), and a bed registered its raw manifest, which the mesh refuses as unbuilt. The bed stocks mesh-runtime-nftables and the filter helper registers the module through the stocked image. --- scenarios/two-nodes.yml | 3 +++ test/integration/harness.ts | 6 +++++- test/integration/mesh.test.ts | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index 17055ee..cce110a 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -37,6 +37,9 @@ machines: images: [] images: + # The packet filter's seat runtime (novox/hq ADR 0170): the filter module now serves its verbs from + # a runtime the mesh builds, so a bed that installs the filter stocks it. + - mesh-runtime-nftables:development - mesh-controller:development # And the builder, because it is a module the mesh assigns rather than a program somebody # starts by hand — which is the only way its credential can be one the mesh delivered. diff --git a/test/integration/harness.ts b/test/integration/harness.ts index d26d67e..da8e458 100644 --- a/test/integration/harness.ts +++ b/test/integration/harness.ts @@ -376,11 +376,15 @@ function shellQuote(s: string): string { */ export async function deriveTheFilterOn(o: { machine: string; node: string; hubPort: number; + /** The images the machines hold. Given, the filter module's runtime — the packet-filter seat's, + * which the mesh would build (novox/hq ADR 0170) — is the stocked one, as for any catalogue + * module a bed installs; the scenario must then stock `mesh-runtime-nftables:development`. */ + held?: HeldImage[]; must: (machine: string, command: string, timeoutMs?: number) => Promise; mesh: (command: string, timeoutMs?: number) => Promise; on: (machine: string, command: string, timeoutMs?: number) => Promise<{ out: string; ok: boolean }>; }): Promise { - const manifest = readFileSync(catalogueManifest(FILTER_MODULE), "utf8"); + const manifest = o.held ? catalogueModule(FILTER_MODULE, o.held) : readFileSync(catalogueManifest(FILTER_MODULE), "utf8"); await o.must(o.machine, `printf %s ${shellQuote(manifest)} > /tmp/${FILTER_MODULE}.json && docker cp /tmp/${FILTER_MODULE}.json mesh-controller:/${FILTER_MODULE}.json`); await o.mesh(`module add /${FILTER_MODULE}.json`); diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index 351f88f..863a272 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -346,7 +346,7 @@ test("both machines join it, and the token is all they need", { skip, timeout: 9 await new Promise((r) => setTimeout(r, 3000)); } await new Promise((r) => setTimeout(r, 5000)); - await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on }); + await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, held, must, mesh, on }); // **The laptop makes its tunnel key, and the token is issued for it.** The hub is told the key // before the token is shown, so the tunnel answers the first time the laptop knocks. @@ -409,7 +409,7 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou // The anchor's derived filter, admitting the hub's port — what genesis does on the control-node, // and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter // keeps the hub closed and nothing on the laptop reaches anchor over the private network. - await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on }); + await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, held, must, mesh, on }); const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim(); // Named after the machine *and* the module, because a consumer is both (novox/hq -- 2.54.0 From e80a4b1642bf01d587231fa417a36b788dc9052e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 01:52:58 +0200 Subject: [PATCH 6/6] Prove a machine joins through the tunnel in a bed of its own Two machines: the anchor raises the foundation, joins over its own loopback and becomes the hub; the joiner makes its tunnel key, is issued a token for it and enrols with the bus's port closed to its own address, so the enrolment can arrive only over the tunnel (novox/hq ADR 0169). The check that had been added to the two-node walk moves here, closing the port the bundle publishes the bus on rather than the bus's own. And the uplink's range may be named: behind a VPN client that routes every private range, incus had none left to pick and no scenario could be raised. --- README.md | 4 + scenarios/joins-through-the-tunnel.yml | 33 ++++ scenarios/two-nodes.yml | 7 - src/lifecycle/raise.ts | 14 +- .../joins-through-the-tunnel.test.ts | 179 ++++++++++++++++++ test/integration/mesh.test.ts | 29 +-- test/uplink.test.ts | 10 + 7 files changed, 241 insertions(+), 35 deletions(-) create mode 100644 scenarios/joins-through-the-tunnel.yml create mode 100644 test/integration/joins-through-the-tunnel.test.ts create mode 100644 test/uplink.test.ts diff --git a/README.md b/README.md index fdbab51..38cbe33 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,10 @@ mesh-lab diagram --live draw what is actually standing correctly and snapshots roughly 76× slower — which does not make the lab slow, it makes it unused, and a warning about that is read once and ignored forever. +Behind a VPN client that routes every private range, incus finds no free range for the uplink a +machine fetches its images over, and nothing can be raised. Name one the host does not route: +`MESH_LAB_UPLINK_V4=192.168.231.1/24`. + If the incus socket is not reachable as your user — the group was granted to a session that already existed — set `MESH_LAB_INCUS="sudo -n incus"`. diff --git a/scenarios/joins-through-the-tunnel.yml b/scenarios/joins-through-the-tunnel.yml new file mode 100644 index 0000000..98ecc9f --- /dev/null +++ b/scenarios/joins-through-the-tunnel.yml @@ -0,0 +1,33 @@ +# A machine joins through the tunnel, with the bus closed to it (novox/hq ADR 0169). +# +# The anchor raises the foundation from the bundle, joins over its own loopback and becomes the hub. +# The joiner has a host and nothing else: it makes its tunnel key, a token is issued for that key, +# and it enrols. The bus is closed to the joiner's own address on the anchor, so the only way its +# enrolment can arrive is over the tunnel the token gave it. +scenario: joins-through-the-tunnel + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + joiner: + at: { segment: hosting, address: [192.0.2.30] } + egress: true + inbound: allow + memory: 1GiB + # It carries none of the images: it only has to join. + images: [] + +images: + - mesh-controller:development + +place: + all: [host, runtime] diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index cce110a..cc7d769 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -28,13 +28,6 @@ machines: egress: true inbound: allow memory: 2GiB - # A third machine that joins through the tunnel with the bus closed to it (novox/hq ADR 0169). It - # carries none of the images: it only has to join. - joiner: - at: { segment: hosting, address: [192.0.2.30] } - egress: true - inbound: allow - images: [] images: # The packet filter's seat runtime (novox/hq ADR 0170): the filter module now serves its verbs from diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index 908237f..5c53e1e 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -135,6 +135,18 @@ async function createNetwork( return name; } +/** + * The uplink's own address and prefix: incus picks a free one, unless `MESH_LAB_UPLINK_V4` names it. + * + * **A workstation behind a VPN client may have no free range left to pick.** A corporate client that + * routes all of 10.0.0.0/8 and 172.16.0.0/12 leaves incus refusing with "failed to automatically find + * an unused IPv4 subnet", and no scenario can be raised at all. Naming one the host does not route — + * `192.168.231.1/24` — is the way past it; it is the host's fact, never the scenario's. + */ +export function uplinkAddress(env: NodeJS.ProcessEnv = process.env): string { + return env["MESH_LAB_UPLINK_V4"]?.trim() || "auto"; +} + /** * The one network the lab supplies rather than the declaration. * @@ -153,7 +165,7 @@ async function createUplink(instanceId: string): Promise { if (await succeeds(["network", "show", name], 15_000)) return name; await incus([ "network", "create", name, - "ipv4.address=auto", + `ipv4.address=${uplinkAddress()}`, "ipv4.nat=true", "ipv6.address=none", `user.mesh-lab.instance=${instanceId}`, diff --git a/test/integration/joins-through-the-tunnel.test.ts b/test/integration/joins-through-the-tunnel.test.ts new file mode 100644 index 0000000..08e687f --- /dev/null +++ b/test/integration/joins-through-the-tunnel.test.ts @@ -0,0 +1,179 @@ +/** + * **A machine joins through the tunnel, and the bus is never public** (novox/hq ADR 0169). + * + * The anchor raises the foundation from the bundle, joins over its own loopback — it runs the bus, + * so it needs no tunnel to reach it — and is placed as the hub. The joiner then does what a new + * machine does: makes its tunnel key and prints the public half, is issued a token for that key, + * and enrols with the token alone. The bus is closed to the joiner's own address on the anchor + * before any of that, so the enrolment can arrive only over the tunnel the token gave it. + * + * It asserts the ADR's last row: *a new machine joins from outside the hub's network with the bus + * closed to it.* The rows before it are the controller's own tests. + * + * It needs a host binary, the foundation bundle and the control plane's image: + * + * MESH_LAB_HOST_BINARY=.../mesh-host + * MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock + * (mesh-controller:development, from mesh-controller's `make image`) + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" + : false; + +const SCENARIO = "joins-through-the-tunnel"; +/** The hub's tunnel port, and the bus's port as the bundle publishes it on the anchor. */ +const HUB_PORT = 51820; +const BUS_PORT = 5671; +let instanceId = ""; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} + +async function mesh(command: string, timeoutMs?: number): Promise { + return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +async function until(what: string, within: number, check: () => Promise, why: () => Promise): Promise { + const end = Date.now() + within; + while (Date.now() < end) { + if (await check()) return; + await new Promise((r) => setTimeout(r, 3000)); + } + assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`); +} + +/** + * Put the mesh's composed user list where the anchor's bus reads it, and make it re-read. + * + * **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146): + * the bus here is the bundle's, so an account the mesh composes reaches it only if whoever raised it + * places it — the enrolment's when a token is issued, the node's own once it enrols. + */ +async function placeTheBusUsers(): Promise { + await must("anchor", + `docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` + + `docker kill -s HUP mesh-broker >/dev/null`); +} + +before(async () => { + if (skip) return; + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + + await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`); + await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000); + const up = await must("anchor", `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { + assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`); + } +}, { timeout: 1_800_000 }); + +after(async () => { + if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 600_000 }); + +test("the machine that runs the bus joins over its own loopback and becomes the hub", { skip, timeout: 900_000 }, async () => { + await mesh("node add anchor"); + const token = tokenFrom(await mesh("token issue --node anchor")); + await placeTheBusUsers(); + const said = await must("anchor", `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, /enrolled as anchor/, said); + assert.doesNotMatch(said, /tunnel to the hub/, `the machine running the bus went through a tunnel:\n${said}`); + await placeTheBusUsers(); + await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`); + + await mesh(`overlay place anchor --hub --endpoint 192.0.2.10:${HUB_PORT} --site lab`); + await mesh("assign anchor mesh-wireguard"); + await mesh("push anchor", 600_000); + await until("the hub's tunnel coming up", 300_000, + async () => (await on("anchor", `wg show mesh0 listen-port`)).out.trim() === String(HUB_PORT), + async () => `--- anchor host ---\n${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`); + // The hub's port, opened as the derived filter opens it — scenery here: the filter module is the + // two-node walk's to prove, and the bundle's own filter admits only ssh, the bus and the registry. + await must("anchor", `nft insert rule inet mesh input udp dport ${HUB_PORT} accept`); +}); + +test("a machine joins through the tunnel, with the bus closed to it", { skip, timeout: 900_000 }, async () => { + // **The bus closed to the joiner**, at the anchor's very first hook — before the container + // runtime's forwarding — so nothing from its own address reaches the bus, by any path. + await must("anchor", `nft add table ip lab_bus_closed && ` + + `nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` + + `nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport ${BUS_PORT} drop`); + const reached = await on("joiner", `timeout 5 bash -c '/dev/null 2>&1`, 300_000); + const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(); + assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`); + // Asked again, the same key: a token may already have been issued for it. + assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key); + + // The token is issued for the key; the hub is sent the machine as a peer before it is shown. + const issued = await mesh(`token issue --new joiner --overlay-key ${key}`, 600_000); + const token = tokenFrom(issued); + const carried = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as { + broker: string; tunnel?: { key: string; address: string; hub_endpoint: string }; + }; + assert.equal(carried.tunnel?.key, key, `the token was not issued for the machine's key: ${JSON.stringify(carried)}`); + assert.equal(carried.tunnel?.hub_endpoint, `192.0.2.10:${HUB_PORT}`); + assert.doesNotMatch(carried.broker, /^192\.0\.2\./, `the token sends the machine to the bus's public address: ${carried.broker}`); + await placeTheBusUsers(); + + const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`, 300_000); + assert.match(said, /the tunnel to the hub is up/, said); + assert.match(said, /the hub answered the tunnel/, said); + assert.match(said, /enrolled as joiner/, said); + + const shakes = await must("joiner", `wg show mesh0 latest-handshakes`); + assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`); + // And the bus is still closed to it: it joined through the tunnel, not around it. + assert.ok(!(await on("joiner", `timeout 5 bash -c ' { - await must("anchor", `nft add table ip lab_bus_closed && ` + - `nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` + - `nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport 4222 drop`); - try { - await must("joiner", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`); - const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(); - assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`); - // Asked again, the same key: a token may already have been issued for it. - assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key); - - const token = tokenFrom(await mesh(`token issue --new joiner --overlay-key ${key}`)); - await composeTheBusUsers(); - const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`); - assert.match(said, /the tunnel to the hub is up/, said); - assert.match(said, /enrolled as joiner/, said); - - const shakes = await must("joiner", `wg show mesh0 latest-handshakes`); - assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`); - } finally { - await on("anchor", `nft delete table ip lab_bus_closed`); - } -}); - +// A machine joining through the tunnel with the bus closed to it is its own bed: +// joins-through-the-tunnel.test.ts (novox/hq ADR 0169). diff --git a/test/uplink.test.ts b/test/uplink.test.ts new file mode 100644 index 0000000..7d6976d --- /dev/null +++ b/test/uplink.test.ts @@ -0,0 +1,10 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { uplinkAddress } from "../src/lifecycle/raise.ts"; + +// A workstation behind a VPN client that routes every private range leaves incus nothing to pick, so +// the uplink's range may be named; otherwise incus picks it. +test("the uplink's range is incus's to pick unless the host names one", () => { + assert.equal(uplinkAddress({}), "auto"); + assert.equal(uplinkAddress({ MESH_LAB_UPLINK_V4: " 192.168.231.1/24 " }), "192.168.231.1/24"); +}); -- 2.54.0