mesh-controller/lab/adopt-the-tunnel/adopt-the-tunnel.test.ts

/**
 * THE HUB ADOPTS THE PREDECESSOR'S TUNNEL (novox/hq ADR 0105). Skeleton — NOT YET RUN.
 *
 * The bed and every assertion are described in README.md beside this file; the numbered
 * assertions here are that document's. Follows adoption.test.ts: same harness, same `on`/`must`
 * helpers, same genesis wrapper.
 *
 *   MESH_LAB_INCUS='sudo -n incus'
 *   MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
 *   MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock
 *   MESH_LAB_CATALOG=.../mesh-catalog/modules
 */
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, catalogueIsPresent } from "./harness.ts";
import { genesis } from "./genesis.ts";

const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable ? `lab not usable: ${capability.why}`
  : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
    : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle"
      : catalogueIsPresent();

const SCENARIO = "adopt-the-tunnel";
const ANCHOR = "anchor", PEER_A = "peer-a", PEER_B = "peer-b", FRESH = "fresh";
const TUNNEL = { port: 51900, range: "10.10.0.0/24", hub: "10.10.0.1", a: "10.10.0.2", b: "10.10.0.3", fresh: "10.10.0.4" };
const SERVICE = `http://${TUNNEL.hub}:8081/`;

let instanceId = "";
let wg0Key = "";      // the anchor's wg0 public key, recorded before genesis
let wg0Digest = "";   // sha256 of /etc/wireguard/wg0.conf before genesis

async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
  const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
  const marker = stdout.lastIndexOf("__exit=");
  if (marker < 0) return { out: stdout, ok: false };
  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
  const { out, ok } = await on(machine, command, timeoutMs);
  if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
  return out;
}
/** The controller, a container on the anchor. */
async function control(args: string): Promise<string> {
  return must(ANCHOR, `docker exec mesh-controller mesh-controller ${args}`);
}

/** Prepares a machine the way the predecessor leaves it: a keypair and a wg0 — see README.md. */
async function predecessorTunnelOn(machine: string, conf: (keys: Record<string, string>) => string, keys: Record<string, string>): Promise<void> {
  await must(machine, "apt-get install -y wireguard-tools >/dev/null 2>&1 || pacman -S --noconfirm wireguard-tools >/dev/null");
  await must(machine, `umask 077; printf '%s' '${conf(keys)}' > /etc/wireguard/wg0.conf`);
  await must(machine, "systemctl enable --now wg-quick@wg0");
}

before(async () => {
  if (skip) return;
  await destroyAll(SCENARIO);
  const scenario = loadScenario(`scenarios/${SCENARIO}.yml`);
  instanceId = (await raise(scenario)).instanceId;

  // Keys for the three predecessor machines, made where they live and never moved.
  const keys: Record<string, string> = {};
  for (const m of [ANCHOR, PEER_A, PEER_B]) {
    await must(m, "umask 077; wg genkey > /etc/wireguard/predecessor.key");
    keys[m] = (await must(m, "wg pubkey < /etc/wireguard/predecessor.key")).trim();
  }
  await predecessorTunnelOn(ANCHOR, k => [
    "[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `ListenPort = ${TUNNEL.port}`, `Address = ${TUNNEL.hub}/24`,
    "[Peer]", `PublicKey = ${k[PEER_A]}`, `AllowedIPs = ${TUNNEL.a}/32`,
    "[Peer]", `PublicKey = ${k[PEER_B]}`, `AllowedIPs = ${TUNNEL.b}/32`,
  ].join("\n"), keys);
  for (const [m, addr] of [[PEER_A, TUNNEL.a], [PEER_B, TUNNEL.b]] as const) {
    await predecessorTunnelOn(m, k => [
      "[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `Address = ${addr}/24`,
      "[Peer]", `PublicKey = ${k[ANCHOR]}`, `Endpoint = 192.0.2.10:${TUNNEL.port}`, `AllowedIPs = ${TUNNEL.range}`, "PersistentKeepalive = 25",
    ].join("\n"), keys);
  }
  // A service reachable only over the tunnel, under a name no catalogue module uses.
  await must(ANCHOR, `docker run -d --name predecessor-page -p ${TUNNEL.hub}:8081:80 nginx:alpine`);
  // The predecessor's firewall: the tunnel's port and ssh, nothing else (as ADR 0100's bed).
  await must(ANCHOR, `ufw --force reset >/dev/null; ufw default deny incoming; ufw allow 22/tcp; ufw allow ${TUNNEL.port}/udp; ufw --force enable`);
  for (const m of [PEER_A, PEER_B]) assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service over the tunnel before genesis`);

  wg0Key = (await must(ANCHOR, "wg show wg0 public-key")).trim();
  wg0Digest = (await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0];
  // The probe the peers keep running through the switch: one call a second, failures counted.
  for (const m of [PEER_A, PEER_B]) await must(m, `nohup sh -c 'while :; do curl -fsS --max-time 1 ${SERVICE} >/dev/null 2>&1 || date +%s >> /tmp/failed; sleep 1; done' >/dev/null 2>&1 &`);
});

after(async () => { if (instanceId) await destroy(instanceId); });

test("T1 — adopted, the tunnel changes hands and the peers notice nothing", { skip }, async () => {
  const ran = await genesis({ instanceId, machine: ANCHOR, node: ANCHOR, site: "hosting",
    flags: ["--adopted", "--endpoint", `192.0.2.10:${TUNNEL.port}`] } as never);
  assert.match(ran.said, /tunnel\s+wg0/i, `genesis did not say it found and took the tunnel:\n${ran.said}`);

  const ifaces = await must(ANCHOR, "wg show interfaces");
  assert.match(ifaces, /\bmesh0\b/); assert.doesNotMatch(ifaces, /\bwg0\b/);
  assert.equal((await on(ANCHOR, "systemctl is-active wg-quick@wg0")).out.trim(), "inactive");
  assert.equal((await on(ANCHOR, "systemctl is-enabled wg-quick@wg0")).out.trim(), "disabled");
  assert.equal((await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0], wg0Digest, "the found configuration was changed or flushed");
  assert.equal((await must(ANCHOR, "wg show mesh0 public-key")).trim(), wg0Key, "the mesh's interface is not up with the found key");
  assert.equal((await must(ANCHOR, "wg show mesh0 listen-port")).trim(), String(TUNNEL.port));
  assert.match(await must(ANCHOR, "ip -o addr show dev mesh0"), new RegExp(TUNNEL.hub.replaceAll(".", "\\.")));
  const peers = await must(ANCHOR, "wg show mesh0 allowed-ips");
  assert.match(peers, new RegExp(`${TUNNEL.a}/32`)); assert.match(peers, new RegExp(`${TUNNEL.b}/32`));

  for (const m of [PEER_A, PEER_B]) {
    const failed = (await on(m, "cat /tmp/failed 2>/dev/null | wc -l")).out.trim();
    assert.ok(Number(failed) <= 5, `${m} lost the service for ${failed} seconds through the switch`);
    assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service after the switch`);
  }
  const shown = await control("overlay show");
  assert.match(shown, /anchor.*hub.*took over on wg0/);
  assert.match(shown, /peers of the tunnel/); assert.match(shown, /not yet enrolled/);
  assert.match(await control(`node show ${ANCHOR}`), /tunnel found\s+wg0 on port 51900/);
});

test("T2 — a peer enrols over the tunnel and keeps its address", { skip }, async () => {
  await control(`node add ${PEER_A} --adopted`);
  const token = (await control(`token issue --node ${PEER_A}`)).match(/token\s+(\S+)/)?.[1] ?? "";
  // The token's broker address is the hub's tunnel address: only the tunnel routes it.
  await must(PEER_A, `mesh-host enrol --token '${token}'`);
  await control(`overlay place ${PEER_A} --site house`);
  await control(`assign ${PEER_A} networking`);
  await control(`push ${PEER_A} --wait 2m`);

  assert.match(await control("overlay show"), new RegExp(`${PEER_A}\\s+${TUNNEL.a.replaceAll(".", "\\.")}`));
  assert.match(await control("overlay show"), new RegExp(`enrolled as ${PEER_A}`));
  const onHub = await must(ANCHOR, "wg show mesh0 allowed-ips");
  assert.equal(onHub.split("\n").filter(l => l.includes(`${TUNNEL.a}/32`)).length, 1, "the enrolled peer's key appears twice on the hub");
  assert.doesNotMatch(await must(PEER_A, "wg show interfaces"), /\bwg0\b/);
  assert.ok((await on(PEER_A, `curl -fsS --max-time 3 ${SERVICE}`)).ok);
  assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok, "the peer that never enrols lost the service");
});

test("T3 — a new machine gets a fresh address from the same range", { skip }, async () => {
  await control(`node add ${FRESH}`);
  const token = (await control(`token issue --node ${FRESH}`)).match(/token\s+(\S+)/)?.[1] ?? "";
  await must(FRESH, `mesh-host enrol --token '${token}'`);
  await control(`overlay place ${FRESH} --nothing`);
  await control(`assign ${FRESH} networking`);
  await control(`push ${FRESH} --wait 2m`);
  assert.match(await control("overlay show"), new RegExp(`${FRESH}\\s+${TUNNEL.fresh.replaceAll(".", "\\.")}`));
  assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.hub}`)).ok, "the new machine does not reach the hub");
  assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.a}`)).ok, "the new machine does not reach the enrolled peer");
  assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok);
});

test("T4 — nothing derived from the address is stale", { skip }, async () => {
  for (const n of [ANCHOR, PEER_A, FRESH]) {
    const plan = await control(`plan ${n} --json`);
    assert.doesNotMatch(plan, /10\.42\./, `${n}'s plan names the mesh's default range`);
    assert.match(plan, new RegExp(TUNNEL.hub.replaceAll(".", "\\.")));
    assert.match(await must(n, "cat /etc/hosts"), new RegExp(`${TUNNEL.hub.replaceAll(".", "\\.")}\\s+anchor\\.internal`));
  }
  const first = await control(`plan ${PEER_A} --json`);
  await control("push");
  assert.equal(await control(`plan ${PEER_A} --json`), first, "a push changed what the plan says");
});

test("N — where a tunnel is not adopted, the ranges must still differ (ADR 0100)", { skip }, async () => {
  await must(FRESH, "umask 077; printf '[Interface]\\nPrivateKey = %s\\nAddress = 10.10.0.9/24\\n' \"$(wg genkey)\" > /etc/wireguard/wg1.conf; systemctl start wg-quick@wg1");
  const ran = await genesis({ instanceId, machine: FRESH, node: FRESH, flags: ["--dry-run", "--overlay-range", TUNNEL.range], attempts: 1, verify: false, hostService: false } as never);
  assert.match(ran.said, /wg1/, `a converged genesis did not refuse the overlapping tunnel it does not adopt:\n${ran.said}`);
});