Integration tests, each named for the decision it defends

Reviewed and the criticism was right: 1,072 of 2,128 lines untested, all of
it the half that touches the hypervisor, and no gate. The verification I had
done was real — pings across NAT, TTL counts, ruleset comparisons — and none
of it survived the terminal it ran in, which is 04-ISSUES/005 in miniature.

Ten integration tests against a real hypervisor, each named for what it
defends. ADR 0031: a raised machine carries no overlay, no wireguard, no
mesh config — a scenario that pre-built peering would certify its own work.
ADR 0032: exec is the only way in. ADR 0033: routers are containers while
machines are virtual machines. And the design's claims: raise waits for
usable, snapshots are whole-scenario, NAT hides a private address,
published reaches the machine at the gateway's address.

Mocking the hypervisor is forbidden, so they skip with a reason on a
machine that cannot raise scenarios rather than passing green having
checked nothing.

The suite earned itself on its first run. It found that a snapshot of a
running machine could miss a file written seconds earlier — not stale,
absent — because the write was still in the guest's page cache. That is
exactly the question the lifecycle design listed as open: does a scenario
snapshot need the machines stopped? It does not, but it does need them
flushed. snapshot now syncs every machine before capturing, and the design
records the answer.

The fix buys write-durability, not application-consistency: anything
mid-transaction is still captured mid-transaction, and that is now stated
rather than assumed.

npm run check is the gate — typecheck, 40 unit tests, 10 integration tests.

test/integration/harness.ts

@@ -0,0 +1,44 @@
+/**
+ * Integration tests run against a real hypervisor. Mocking it is forbidden — a test that
+ * fakes the system under integration asserts that the fake behaves as expected, which is
+ * the shape of test this project exists to stop shipping (novox/hq ADR 0034).
+ *
+ * Consequence, accepted: these are slow, and they need a machine that can raise scenarios.
+ * They skip rather than fail where it cannot, so that a machine without a hypervisor gets
+ * an honest "not run" instead of a green suite that checked nothing.
+ */
+
+import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts";
+import { destroy, list } from "../../src/lifecycle/operate.ts";
+
+export interface Capability {
+  usable: boolean;
+  why: string;
+}
+
+/** Can this machine run scenarios at all? Checked once, reported honestly. */
+export async function labIsUsable(): Promise<Capability> {
+  if (!(await isReachable())) {
+    return {
+      usable: false,
+      why: "the incus daemon is not reachable as this user (try MESH_LAB_INCUS='sudo -n incus')",
+    };
+  }
+  const drivers = await supportedDrivers();
+  if (!drivers.some((d) => d === "btrfs" || d === "zfs")) {
+    return { usable: false, why: "no copy-on-write driver — snapshots would be full copies" };
+  }
+  if (!(await pools()).some((p) => p.driver === "btrfs" || p.driver === "zfs")) {
+    return { usable: false, why: "no pool uses a copy-on-write driver" };
+  }
+  return { usable: true, why: "" };
+}
+
+/** Tear down anything a test left behind, whether it passed or not. */
+export async function destroyAll(prefix: string): Promise<void> {
+  for (const instance of await list()) {
+    if (instance.instanceId.startsWith(prefix)) {
+      await destroy(instance.instanceId);
+    }
+  }
+}

test/integration/underlay.test.ts

@@ -0,0 +1,133 @@
+/**
+ * Each test names the decision it defends. A decision with no test is one that will quietly
+ * stop being true (novox/hq ADR 0034).
+ */
+
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec, list, restore, snapshot } from "../../src/lifecycle/operate.ts";
+import { incus, incusOk } from "../../src/incus/client.ts";
+import { labIsUsable, destroyAll } from "./harness.ts";
+
+const capability = await labIsUsable();
+const skip = capability.usable ? false : `lab not usable: ${capability.why}`;
+
+let instanceId = "";
+
+before(async () => {
+  if (skip) return;
+  const scenario = loadScenario("scenarios/behind-nat.yml");
+  const raised = await raise(scenario, {});
+  instanceId = raised.instanceId;
+}, { timeout: 900_000 });
+
+after(async () => {
+  if (instanceId) await destroy(instanceId);
+}, { timeout: 400_000 });
+
+test("ADR 0031 — the lab provides the underlay and NOTHING of the overlay", { skip }, async () => {
+  // A scenario that pre-built peering would certify its own work. Whatever the mesh is
+  // responsible for must be absent from a freshly raised machine.
+  const { stdout } = await exec(instanceId, "home-server", [
+    "sh", "-c",
+    "ip link show type wireguard 2>/dev/null | wc -l; " +
+      "ls /etc/wireguard 2>/dev/null | wc -l; " +
+      "ls /etc/hal /etc/mesh 2>/dev/null | wc -l",
+  ]);
+  const counts = stdout.trim().split("\n").map((n) => Number(n.trim()));
+  assert.deepEqual(counts, [0, 0, 0], "a raised machine carries no overlay, no mesh config");
+}, { timeout: 120_000 });
+
+test("ADR 0031 — the declared address IS what the machine holds", { skip }, async () => {
+  const { stdout } = await exec(instanceId, "home-server", ["ip", "-o", "-4", "addr", "show"]);
+  assert.match(stdout, /192\.168\.1\.135\/24/);
+});
+
+test("design — raise waits for USABLE, not for the call to return", { skip }, async () => {
+  // The measured gap is 3.4s to 14.3s. Reporting the earlier number is transport reported
+  // as effect. If raise has returned, every machine must answer immediately.
+  for (const machine of ["anchor", "home-server"]) {
+    const { stdout } = await exec(instanceId, machine, ["sh", "-c", "echo alive"]);
+    assert.equal(stdout.trim(), "alive", `${machine} was not usable when raise returned`);
+  }
+}, { timeout: 120_000 });
+
+test("ADR 0033 — a router is scenery: containers, while machines are virtual machines", { skip }, async () => {
+  const json = (await incusOk(["list", "--format", "json"], 30_000)) ?? "[]";
+  const all = JSON.parse(json) as { name?: string; type?: string; config?: Record<string, string> }[];
+  const mine = all.filter((i) => i.config?.["user.mesh-lab.instance"] === instanceId);
+  assert.ok(mine.length >= 3, "expected machines and a router");
+
+  for (const item of mine) {
+    const isRouter = item.config?.["user.mesh-lab.router"] !== undefined;
+    assert.equal(
+      item.type,
+      isRouter ? "container" : "virtual-machine",
+      `${item.name} is a ${item.type} but ${isRouter ? "is" : "is not"} a router`,
+    );
+  }
+}, { timeout: 120_000 });
+
+test("design — NAT: a private address is not reachable from outside", { skip }, async () => {
+  const { stdout } = await exec(instanceId, "anchor", [
+    "sh", "-c", "ping -c1 -W2 192.168.1.135 >/dev/null 2>&1 && echo reachable || echo unreachable",
+  ]);
+  assert.equal(stdout.trim(), "unreachable");
+}, { timeout: 120_000 });
+
+test("design — published: reachable at the GATEWAY's address, never its own", { skip }, async () => {
+  await exec(instanceId, "home-server", [
+    "sh", "-c", "nohup python3 -m http.server 8080 --bind 0.0.0.0 >/tmp/s.log 2>&1 & sleep 2",
+  ]);
+  const { stdout } = await exec(instanceId, "anchor", [
+    "sh", "-c", "curl -s -m5 -o /dev/null -w '%{http_code}' http://192.0.2.50:8080/ || echo failed",
+  ]);
+  assert.equal(stdout.trim(), "200", "the forwarded port did not reach the machine behind NAT");
+}, { timeout: 180_000 });
+
+test("design — snapshots are WHOLE-scenario: restore returns every machine", { skip }, async () => {
+  // Restoring a subset would produce a mesh that has never existed, so faults found there
+  // would be artefacts of the lab.
+  await exec(instanceId, "anchor", ["sh", "-c", "echo dirty > /root/marker"]);
+  await exec(instanceId, "home-server", ["sh", "-c", "echo dirty > /root/marker"]);
+  await snapshot(instanceId, "test-point");
+
+  await exec(instanceId, "anchor", ["sh", "-c", "echo changed > /root/marker"]);
+  await exec(instanceId, "home-server", ["sh", "-c", "echo changed > /root/marker"]);
+  await restore(instanceId, "test-point");
+
+  for (const machine of ["anchor", "home-server"]) {
+    const { stdout } = await exec(instanceId, machine, ["cat", "/root/marker"]);
+    assert.equal(stdout.trim(), "dirty", `${machine} was not returned to the snapshot`);
+  }
+}, { timeout: 600_000 });
+
+test("design — restore leaves the scenario USABLE, not merely running", { skip }, async () => {
+  // The restore call returns in under a second while the agent is still starting. Reporting
+  // that as restored would be transport reported as effect.
+  const { stdout } = await exec(instanceId, "anchor", ["sh", "-c", "echo alive"]);
+  assert.equal(stdout.trim(), "alive");
+}, { timeout: 120_000 });
+
+test("ADR 0032 — the workstation has no route into the scenario", { skip }, async () => {
+  // Reachability is asked from INSIDE. If the workstation could reach a scenario address,
+  // two scenarios carrying the same prefix would put one's traffic in the other.
+  const { stdout } = await incus(["exec", `mlab-${instanceId}-anchor`, "--", "echo", "inside"]);
+  assert.equal(stdout.trim(), "inside", "exec is the only way in, and it works");
+}, { timeout: 60_000 });
+
+test("housekeeping — destroy removes machines, routers and segments", { skip }, async () => {
+  const before = (await list()).find((i) => i.instanceId === instanceId);
+  assert.ok(before, "the instance should exist before it is destroyed");
+
+  const { machines, networks } = await destroy(instanceId);
+  assert.ok(machines >= 3, `expected machines and a router, removed ${machines}`);
+  assert.ok(networks >= 2, `expected both segments removed, removed ${networks}`);
+
+  const after = (await list()).find((i) => i.instanceId === instanceId);
+  assert.equal(after, undefined, "the instance should be gone");
+  instanceId = "";
+  await destroyAll("behind-nat-");
+}, { timeout: 400_000 });