Prove a machine joins through the tunnel in a bed of its own

Two machines: the anchor raises the foundation, joins over its own
loopback and becomes the hub; the joiner makes its tunnel key, is issued
a token for it and enrols with the bus's port closed to its own address,
so the enrolment can arrive only over the tunnel (novox/hq ADR 0169). The
check that had been added to the two-node walk moves here, closing the
port the bundle publishes the bus on rather than the bus's own.

And the uplink's range may be named: behind a VPN client that routes
every private range, incus had none left to pick and no scenario could
be raised.

test/integration/joins-through-the-tunnel.test.ts

@@ -0,0 +1,179 @@
+/**
+ * **A machine joins through the tunnel, and the bus is never public** (novox/hq ADR 0169).
+ *
+ * The anchor raises the foundation from the bundle, joins over its own loopback — it runs the bus,
+ * so it needs no tunnel to reach it — and is placed as the hub. The joiner then does what a new
+ * machine does: makes its tunnel key and prints the public half, is issued a token for that key,
+ * and enrols with the token alone. The bus is closed to the joiner's own address on the anchor
+ * before any of that, so the enrolment can arrive only over the tunnel the token gave it.
+ *
+ * It asserts the ADR's last row: *a new machine joins from outside the hub's network with the bus
+ * closed to it.* The rows before it are the controller's own tests.
+ *
+ * It needs a host binary, the foundation bundle and the control plane's image:
+ *
+ *   MESH_LAB_HOST_BINARY=.../mesh-host
+ *   MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
+ *   (mesh-controller:development, from mesh-controller's `make image`)
+ */
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync } from "node:fs";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec } from "../../src/lifecycle/operate.ts";
+import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
+import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
+
+const capability = await labIsUsable();
+const binary = hostBinaryPath();
+const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
+
+const skip = !capability.usable
+  ? `lab not usable: ${capability.why}`
+  : !binary || !existsSync(binary)
+    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
+    : !bundle || !existsSync(bundle)
+      ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
+      : false;
+
+const SCENARIO = "joins-through-the-tunnel";
+/** The hub's tunnel port, and the bus's port as the bundle publishes it on the anchor. */
+const HUB_PORT = 51820;
+const BUS_PORT = 5671;
+let instanceId = "";
+
+function quote(s: string): string {
+  return `'${s.replaceAll("'", `'\\''`)}'`;
+}
+
+async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
+  const { stdout } = await exec(instanceId, machine, [
+    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
+  ], timeoutMs);
+  const marker = stdout.lastIndexOf("__exit=");
+  if (marker < 0) return { out: stdout, ok: false };
+  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
+}
+
+async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
+  const { out, ok } = await on(machine, command, timeoutMs);
+  if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
+  return out;
+}
+
+async function mesh(command: string, timeoutMs?: number): Promise<string> {
+  return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
+}
+
+function tokenFrom(said: string): string {
+  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
+  assert.ok(found, `no token in:\n${said}`);
+  return found;
+}
+
+async function until(what: string, within: number, check: () => Promise<boolean>, why: () => Promise<string>): Promise<void> {
+  const end = Date.now() + within;
+  while (Date.now() < end) {
+    if (await check()) return;
+    await new Promise((r) => setTimeout(r, 3000));
+  }
+  assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`);
+}
+
+/**
+ * Put the mesh's composed user list where the anchor's bus reads it, and make it re-read.
+ *
+ * **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146):
+ * the bus here is the bundle's, so an account the mesh composes reaches it only if whoever raised it
+ * places it — the enrolment's when a token is issued, the node's own once it enrols.
+ */
+async function placeTheBusUsers(): Promise<void> {
+  await must("anchor",
+    `docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` +
+      `docker kill -s HUP mesh-broker >/dev/null`);
+}
+
+before(async () => {
+  if (skip) return;
+  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
+    onProgress: (m) => console.log(`raise: ${m}`),
+  });
+  instanceId = raised.instanceId;
+
+  await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
+  await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000);
+  const up = await must("anchor", `docker ps --format '{{.Names}}'`);
+  for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
+    assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
+  }
+}, { timeout: 1_800_000 });
+
+after(async () => {
+  if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
+  if (instanceId) await destroy(instanceId);
+  await destroyAll(`${SCENARIO}-`);
+}, { timeout: 600_000 });
+
+test("the machine that runs the bus joins over its own loopback and becomes the hub", { skip, timeout: 900_000 }, async () => {
+  await mesh("node add anchor");
+  const token = tokenFrom(await mesh("token issue --node anchor"));
+  await placeTheBusUsers();
+  const said = await must("anchor", `${HOST_PATH} enrol --token ${quote(token)}`);
+  assert.match(said, /enrolled as anchor/, said);
+  assert.doesNotMatch(said, /tunnel to the hub/, `the machine running the bus went through a tunnel:\n${said}`);
+  await placeTheBusUsers();
+  await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
+
+  await mesh(`overlay place anchor --hub --endpoint 192.0.2.10:${HUB_PORT} --site lab`);
+  await mesh("assign anchor mesh-wireguard");
+  await mesh("push anchor", 600_000);
+  await until("the hub's tunnel coming up", 300_000,
+    async () => (await on("anchor", `wg show mesh0 listen-port`)).out.trim() === String(HUB_PORT),
+    async () => `--- anchor host ---\n${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
+  // The hub's port, opened as the derived filter opens it — scenery here: the filter module is the
+  // two-node walk's to prove, and the bundle's own filter admits only ssh, the bus and the registry.
+  await must("anchor", `nft insert rule inet mesh input udp dport ${HUB_PORT} accept`);
+});
+
+test("a machine joins through the tunnel, with the bus closed to it", { skip, timeout: 900_000 }, async () => {
+  // **The bus closed to the joiner**, at the anchor's very first hook — before the container
+  // runtime's forwarding — so nothing from its own address reaches the bus, by any path.
+  await must("anchor", `nft add table ip lab_bus_closed && ` +
+    `nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` +
+    `nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport ${BUS_PORT} drop`);
+  const reached = await on("joiner", `timeout 5 bash -c '</dev/tcp/192.0.2.10/${BUS_PORT}'`);
+  assert.ok(!reached.ok, "the bus is still reachable from the joiner's own address; the bed proves nothing");
+
+  await must("joiner", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`, 300_000);
+  const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim();
+  assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`);
+  // Asked again, the same key: a token may already have been issued for it.
+  assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key);
+
+  // The token is issued for the key; the hub is sent the machine as a peer before it is shown.
+  const issued = await mesh(`token issue --new joiner --overlay-key ${key}`, 600_000);
+  const token = tokenFrom(issued);
+  const carried = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as {
+    broker: string; tunnel?: { key: string; address: string; hub_endpoint: string };
+  };
+  assert.equal(carried.tunnel?.key, key, `the token was not issued for the machine's key: ${JSON.stringify(carried)}`);
+  assert.equal(carried.tunnel?.hub_endpoint, `192.0.2.10:${HUB_PORT}`);
+  assert.doesNotMatch(carried.broker, /^192\.0\.2\./, `the token sends the machine to the bus's public address: ${carried.broker}`);
+  await placeTheBusUsers();
+
+  const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`, 300_000);
+  assert.match(said, /the tunnel to the hub is up/, said);
+  assert.match(said, /the hub answered the tunnel/, said);
+  assert.match(said, /enrolled as joiner/, said);
+
+  const shakes = await must("joiner", `wg show mesh0 latest-handshakes`);
+  assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`);
+  // And the bus is still closed to it: it joined through the tunnel, not around it.
+  assert.ok(!(await on("joiner", `timeout 5 bash -c '</dev/tcp/192.0.2.10/${BUS_PORT}'`)).ok,
+    "the bus opened to the joiner's own address during the join");
+
+  const recorded = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
+    `-c "select overlay_key from node where name = 'joiner'"`)).trim();
+  assert.equal(recorded, key, "the mesh does not hold the tunnel key the machine made");
+});

test/integration/mesh.test.ts

@@ -1706,30 +1706,5 @@ test("a third-party workload is adopted, with the credential it already had", {
 // assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into
 // mesh-grant-end-to-end, against the catalogue's redis.
 
-// **A machine joins through the tunnel, and needs the bus only through it** (novox/hq ADR 0169).
-// The bus is closed to this machine at the anchor's very first hook, before the container runtime's
-// forwarding, so the only way its enrolment can arrive is over the tunnel the token gave it.
-test("a machine joins through the tunnel, with the bus closed to it", { skip, timeout: 900_000 }, async () => {
-  await must("anchor", `nft add table ip lab_bus_closed && ` +
-    `nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` +
-    `nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport 4222 drop`);
-  try {
-    await must("joiner", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`);
-    const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim();
-    assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`);
-    // Asked again, the same key: a token may already have been issued for it.
-    assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key);
-
-    const token = tokenFrom(await mesh(`token issue --new joiner --overlay-key ${key}`));
-    await composeTheBusUsers();
-    const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`);
-    assert.match(said, /the tunnel to the hub is up/, said);
-    assert.match(said, /enrolled as joiner/, said);
-
-    const shakes = await must("joiner", `wg show mesh0 latest-handshakes`);
-    assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`);
-  } finally {
-    await on("anchor", `nft delete table ip lab_bus_closed`);
-  }
-});
-
+// A machine joining through the tunnel with the bus closed to it is its own bed:
+// joins-through-the-tunnel.test.ts (novox/hq ADR 0169).

test/uplink.test.ts

@@ -0,0 +1,10 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { uplinkAddress } from "../src/lifecycle/raise.ts";
+
+// A workstation behind a VPN client that routes every private range leaves incus nothing to pick, so
+// the uplink's range may be named; otherwise incus picks it.
+test("the uplink's range is incus's to pick unless the host names one", () => {
+  assert.equal(uplinkAddress({}), "auto");
+  assert.equal(uplinkAddress({ MESH_LAB_UPLINK_V4: " 192.168.231.1/24 " }), "192.168.231.1/24");
+});