README.md

@@ -45,6 +45,10 @@ mesh-lab diagram --live <instance>   draw what is actually standing
 correctly and snapshots roughly 76× slower — which does not make the lab slow, it makes it
 unused, and a warning about that is read once and ignored forever.
 
+Behind a VPN client that routes every private range, incus finds no free range for the uplink a
+machine fetches its images over, and nothing can be raised. Name one the host does not route:
+`MESH_LAB_UPLINK_V4=192.168.231.1/24`.
+
 If the incus socket is not reachable as your user — the group was granted to a session that
 already existed — set `MESH_LAB_INCUS="sudo -n incus"`.
 

scenarios/joins-through-the-tunnel.yml

@@ -0,0 +1,33 @@
+# A machine joins through the tunnel, with the bus closed to it (novox/hq ADR 0169).
+#
+# The anchor raises the foundation from the bundle, joins over its own loopback and becomes the hub.
+# The joiner has a host and nothing else: it makes its tunnel key, a token is issued for that key,
+# and it enrols. The bus is closed to the joiner's own address on the anchor, so the only way its
+# enrolment can arrive is over the tunnel the token gave it.
+scenario: joins-through-the-tunnel
+
+segments:
+  hosting:
+    kind: public
+    cidr: [192.0.2.0/24]
+
+machines:
+  anchor:
+    at: { segment: hosting, address: [192.0.2.10] }
+    egress: true
+    inbound: allow
+    memory: 3GiB
+    cpus: 2
+  joiner:
+    at: { segment: hosting, address: [192.0.2.30] }
+    egress: true
+    inbound: allow
+    memory: 1GiB
+    # It carries none of the images: it only has to join.
+    images: []
+
+images:
+  - mesh-controller:development
+
+place:
+  all: [host, runtime]

scenarios/two-nodes.yml

@@ -30,6 +30,9 @@ machines:
     memory: 2GiB
 
 images:
+  # The packet filter's seat runtime (novox/hq ADR 0170): the filter module now serves its verbs from
+  # a runtime the mesh builds, so a bed that installs the filter stocks it.
+  - mesh-runtime-nftables:development
   - mesh-controller:development
   # And the builder, because it is a module the mesh assigns rather than a program somebody
   # starts by hand — which is the only way its credential can be one the mesh delivered.

src/lifecycle/raise.ts

@@ -135,6 +135,18 @@ async function createNetwork(
   return name;
 }
 
+/**
+ * The uplink's own address and prefix: incus picks a free one, unless `MESH_LAB_UPLINK_V4` names it.
+ *
+ * **A workstation behind a VPN client may have no free range left to pick.** A corporate client that
+ * routes all of 10.0.0.0/8 and 172.16.0.0/12 leaves incus refusing with "failed to automatically find
+ * an unused IPv4 subnet", and no scenario can be raised at all. Naming one the host does not route —
+ * `192.168.231.1/24` — is the way past it; it is the host's fact, never the scenario's.
+ */
+export function uplinkAddress(env: NodeJS.ProcessEnv = process.env): string {
+  return env["MESH_LAB_UPLINK_V4"]?.trim() || "auto";
+}
+
 /**
  * The one network the lab supplies rather than the declaration.
  *
@@ -153,7 +165,7 @@ async function createUplink(instanceId: string): Promise<string> {
   if (await succeeds(["network", "show", name], 15_000)) return name;
   await incus([
     "network", "create", name,
-    "ipv4.address=auto",
+    `ipv4.address=${uplinkAddress()}`,
     "ipv4.nat=true",
     "ipv6.address=none",
     `user.mesh-lab.instance=${instanceId}`,

test/integration/harness.ts

@@ -376,11 +376,15 @@ function shellQuote(s: string): string {
  */
 export async function deriveTheFilterOn(o: {
   machine: string; node: string; hubPort: number;
+  /** The images the machines hold. Given, the filter module's runtime — the packet-filter seat's,
+   *  which the mesh would build (novox/hq ADR 0170) — is the stocked one, as for any catalogue
+   *  module a bed installs; the scenario must then stock `mesh-runtime-nftables:development`. */
+  held?: HeldImage[];
   must: (machine: string, command: string, timeoutMs?: number) => Promise<string>;
   mesh: (command: string, timeoutMs?: number) => Promise<string>;
   on: (machine: string, command: string, timeoutMs?: number) => Promise<{ out: string; ok: boolean }>;
 }): Promise<string> {
-  const manifest = readFileSync(catalogueManifest(FILTER_MODULE), "utf8");
+  const manifest = o.held ? catalogueModule(FILTER_MODULE, o.held) : readFileSync(catalogueManifest(FILTER_MODULE), "utf8");
   await o.must(o.machine,
     `printf %s ${shellQuote(manifest)} > /tmp/${FILTER_MODULE}.json && docker cp /tmp/${FILTER_MODULE}.json mesh-controller:/${FILTER_MODULE}.json`);
   await o.mesh(`module add /${FILTER_MODULE}.json`);

test/integration/joins-through-the-tunnel.test.ts

@@ -0,0 +1,179 @@
+/**
+ * **A machine joins through the tunnel, and the bus is never public** (novox/hq ADR 0169).
+ *
+ * The anchor raises the foundation from the bundle, joins over its own loopback — it runs the bus,
+ * so it needs no tunnel to reach it — and is placed as the hub. The joiner then does what a new
+ * machine does: makes its tunnel key and prints the public half, is issued a token for that key,
+ * and enrols with the token alone. The bus is closed to the joiner's own address on the anchor
+ * before any of that, so the enrolment can arrive only over the tunnel the token gave it.
+ *
+ * It asserts the ADR's last row: *a new machine joins from outside the hub's network with the bus
+ * closed to it.* The rows before it are the controller's own tests.
+ *
+ * It needs a host binary, the foundation bundle and the control plane's image:
+ *
+ *   MESH_LAB_HOST_BINARY=.../mesh-host
+ *   MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
+ *   (mesh-controller:development, from mesh-controller's `make image`)
+ */
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync } from "node:fs";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec } from "../../src/lifecycle/operate.ts";
+import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
+import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
+
+const capability = await labIsUsable();
+const binary = hostBinaryPath();
+const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
+
+const skip = !capability.usable
+  ? `lab not usable: ${capability.why}`
+  : !binary || !existsSync(binary)
+    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
+    : !bundle || !existsSync(bundle)
+      ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
+      : false;
+
+const SCENARIO = "joins-through-the-tunnel";
+/** The hub's tunnel port, and the bus's port as the bundle publishes it on the anchor. */
+const HUB_PORT = 51820;
+const BUS_PORT = 5671;
+let instanceId = "";
+
+function quote(s: string): string {
+  return `'${s.replaceAll("'", `'\\''`)}'`;
+}
+
+async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
+  const { stdout } = await exec(instanceId, machine, [
+    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
+  ], timeoutMs);
+  const marker = stdout.lastIndexOf("__exit=");
+  if (marker < 0) return { out: stdout, ok: false };
+  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
+}
+
+async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
+  const { out, ok } = await on(machine, command, timeoutMs);
+  if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
+  return out;
+}
+
+async function mesh(command: string, timeoutMs?: number): Promise<string> {
+  return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
+}
+
+function tokenFrom(said: string): string {
+  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
+  assert.ok(found, `no token in:\n${said}`);
+  return found;
+}
+
+async function until(what: string, within: number, check: () => Promise<boolean>, why: () => Promise<string>): Promise<void> {
+  const end = Date.now() + within;
+  while (Date.now() < end) {
+    if (await check()) return;
+    await new Promise((r) => setTimeout(r, 3000));
+  }
+  assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`);
+}
+
+/**
+ * Put the mesh's composed user list where the anchor's bus reads it, and make it re-read.
+ *
+ * **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146):
+ * the bus here is the bundle's, so an account the mesh composes reaches it only if whoever raised it
+ * places it — the enrolment's when a token is issued, the node's own once it enrols.
+ */
+async function placeTheBusUsers(): Promise<void> {
+  await must("anchor",
+    `docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` +
+      `docker kill -s HUP mesh-broker >/dev/null`);
+}
+
+before(async () => {
+  if (skip) return;
+  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
+    onProgress: (m) => console.log(`raise: ${m}`),
+  });
+  instanceId = raised.instanceId;
+
+  await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
+  await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000);
+  const up = await must("anchor", `docker ps --format '{{.Names}}'`);
+  for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
+    assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
+  }
+}, { timeout: 1_800_000 });
+
+after(async () => {
+  if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
+  if (instanceId) await destroy(instanceId);
+  await destroyAll(`${SCENARIO}-`);
+}, { timeout: 600_000 });
+
+test("the machine that runs the bus joins over its own loopback and becomes the hub", { skip, timeout: 900_000 }, async () => {
+  await mesh("node add anchor");
+  const token = tokenFrom(await mesh("token issue --node anchor"));
+  await placeTheBusUsers();
+  const said = await must("anchor", `${HOST_PATH} enrol --token ${quote(token)}`);
+  assert.match(said, /enrolled as anchor/, said);
+  assert.doesNotMatch(said, /tunnel to the hub/, `the machine running the bus went through a tunnel:\n${said}`);
+  await placeTheBusUsers();
+  await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
+
+  await mesh(`overlay place anchor --hub --endpoint 192.0.2.10:${HUB_PORT} --site lab`);
+  await mesh("assign anchor mesh-wireguard");
+  await mesh("push anchor", 600_000);
+  await until("the hub's tunnel coming up", 300_000,
+    async () => (await on("anchor", `wg show mesh0 listen-port`)).out.trim() === String(HUB_PORT),
+    async () => `--- anchor host ---\n${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
+  // The hub's port, opened as the derived filter opens it — scenery here: the filter module is the
+  // two-node walk's to prove, and the bundle's own filter admits only ssh, the bus and the registry.
+  await must("anchor", `nft insert rule inet mesh input udp dport ${HUB_PORT} accept`);
+});
+
+test("a machine joins through the tunnel, with the bus closed to it", { skip, timeout: 900_000 }, async () => {
+  // **The bus closed to the joiner**, at the anchor's very first hook — before the container
+  // runtime's forwarding — so nothing from its own address reaches the bus, by any path.
+  await must("anchor", `nft add table ip lab_bus_closed && ` +
+    `nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` +
+    `nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport ${BUS_PORT} drop`);
+  const reached = await on("joiner", `timeout 5 bash -c '</dev/tcp/192.0.2.10/${BUS_PORT}'`);
+  assert.ok(!reached.ok, "the bus is still reachable from the joiner's own address; the bed proves nothing");
+
+  await must("joiner", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`, 300_000);
+  const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim();
+  assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`);
+  // Asked again, the same key: a token may already have been issued for it.
+  assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key);
+
+  // The token is issued for the key; the hub is sent the machine as a peer before it is shown.
+  const issued = await mesh(`token issue --new joiner --overlay-key ${key}`, 600_000);
+  const token = tokenFrom(issued);
+  const carried = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as {
+    broker: string; tunnel?: { key: string; address: string; hub_endpoint: string };
+  };
+  assert.equal(carried.tunnel?.key, key, `the token was not issued for the machine's key: ${JSON.stringify(carried)}`);
+  assert.equal(carried.tunnel?.hub_endpoint, `192.0.2.10:${HUB_PORT}`);
+  assert.doesNotMatch(carried.broker, /^192\.0\.2\./, `the token sends the machine to the bus's public address: ${carried.broker}`);
+  await placeTheBusUsers();
+
+  const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`, 300_000);
+  assert.match(said, /the tunnel to the hub is up/, said);
+  assert.match(said, /the hub answered the tunnel/, said);
+  assert.match(said, /enrolled as joiner/, said);
+
+  const shakes = await must("joiner", `wg show mesh0 latest-handshakes`);
+  assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`);
+  // And the bus is still closed to it: it joined through the tunnel, not around it.
+  assert.ok(!(await on("joiner", `timeout 5 bash -c '</dev/tcp/192.0.2.10/${BUS_PORT}'`)).ok,
+    "the bus opened to the joiner's own address during the join");
+
+  const recorded = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
+    `-c "select overlay_key from node where name = 'joiner'"`)).trim();
+  assert.equal(recorded, key, "the mesh does not hold the tunnel key the machine made");
+});

test/integration/mesh.test.ts

@@ -193,6 +193,20 @@ function bundleFor(images: HeldImage[]): string {
   return foundationBundle(bundle, images);
 }
 
+/**
+ * Put the mesh's composed user list where the anchor's bus reads it, and make it re-read.
+ *
+ * **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146).
+ * The bus here is the bundle's, not a module the mesh delivers, so an account the mesh composes —
+ * the enrolment when a token is issued, the node's own when it enrols — reaches it only if whoever
+ * raised it places it. Without this the first join is refused with an authorisation violation.
+ */
+async function composeTheBusUsers(): Promise<void> {
+  await must("anchor",
+    `docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` +
+      `docker kill -s HUP mesh-broker >/dev/null`);
+}
+
 /** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */
 function tokenFrom(said: string): string {
   const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
@@ -308,15 +322,44 @@ test("a bare machine becomes a mesh", { skip, timeout: 600_000 }, async () => {
 });
 
 test("both machines join it, and the token is all they need", { skip, timeout: 900_000 }, async () => {
-  for (const [machine, node] of [["anchor", "anchor"], ["laptop", "laptop"]] as const) {
-    await mesh(`node add ${node}`);
-    const token = tokenFrom(await mesh(`token issue --node ${node}`));
-    // No --name. The token says what the mesh calls the machine, which is the fault this walk
-    // found the first time it was run.
-    const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
-    assert.match(said, new RegExp(`enrolled as ${node}`), said);
-    assert.match(said, /sealing key/, "no sealing key was generated");
+  // **The anchor runs the bus, so it joins over its own loopback.** Every other machine joins
+  // through the tunnel (novox/hq ADR 0169): the anchor is the hub, and its tunnel comes up first.
+  await mesh(`node add anchor`);
+  const first = tokenFrom(await mesh(`token issue --node anchor`));
+  await composeTheBusUsers();
+  // No --name. The token says what the mesh calls the machine, which is the fault this walk
+  // found the first time it was run.
+  const anchorSaid = await must("anchor", `${HOST_PATH} enrol --token ${quote(first)}`);
+  await composeTheBusUsers();
+  assert.match(anchorSaid, /enrolled as anchor/, anchorSaid);
+  assert.match(anchorSaid, /sealing key/, "no sealing key was generated");
+
+  await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
+  await mesh("assign anchor networking");
+  await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
+  await mesh("push anchor");
+  // The host reports which of its links face outside with every apply, and the filter is not
+  // composed for a machine that has not — so its first apply is waited for, not slept past.
+  const appliedBy = Date.now() + 180_000;
+  while (!(await on("anchor", `grep -q 'applied [0-9]' /var/log/mesh-host.log`)).ok) {
+    if (Date.now() > appliedBy) assert.fail(`the anchor never applied what it was sent:\n${(await on("anchor", "tail -30 /var/log/mesh-host.log")).out}`);
+    await new Promise((r) => setTimeout(r, 3000));
   }
+  await new Promise((r) => setTimeout(r, 5000));
+  await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, held, must, mesh, on });
+
+  // **The laptop makes its tunnel key, and the token is issued for it.** The hub is told the key
+  // before the token is shown, so the tunnel answers the first time the laptop knocks.
+  await must("laptop", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`);
+  const key = (await must("laptop", `${HOST_PATH} key 2>/dev/null`)).trim();
+  await mesh(`node add laptop`);
+  const second = tokenFrom(await mesh(`token issue --node laptop --overlay-key ${key}`));
+  await composeTheBusUsers();
+  const laptopSaid = await must("laptop", `${HOST_PATH} enrol --token ${quote(second)}`);
+  await composeTheBusUsers();
+  assert.match(laptopSaid, /the tunnel to the hub is up/, laptopSaid);
+  assert.match(laptopSaid, /enrolled as laptop/, laptopSaid);
+  assert.match(laptopSaid, /sealing key/, "no sealing key was generated");
 
   const recorded = await must("anchor",
     `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
@@ -366,7 +409,7 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
   // The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
   // and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter
   // keeps the hub closed and nothing on the laptop reaches anchor over the private network.
-  await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
+  await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, held, must, mesh, on });
 
   const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
   // Named after the machine *and* the module, because a consumer is both (novox/hq
@@ -1662,3 +1705,6 @@ test("a third-party workload is adopted, with the credential it already had", {
 // a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy
 // assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into
 // mesh-grant-end-to-end, against the catalogue's redis.
+
+// A machine joining through the tunnel with the bus closed to it is its own bed:
+// joins-through-the-tunnel.test.ts (novox/hq ADR 0169).

test/uplink.test.ts

@@ -0,0 +1,10 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { uplinkAddress } from "../src/lifecycle/raise.ts";
+
+// A workstation behind a VPN client that routes every private range leaves incus nothing to pick, so
+// the uplink's range may be named; otherwise incus picks it.
+test("the uplink's range is incus's to pick unless the host names one", () => {
+  assert.equal(uplinkAddress({}), "auto");
+  assert.equal(uplinkAddress({ MESH_LAB_UPLINK_V4: " 192.168.231.1/24 " }), "192.168.231.1/24");
+});