README.md

@@ -190,6 +190,31 @@ npm test          the declaration layer, offline
 npm run typecheck
 ```
 
-The lifecycle is not unit-tested. It talks to a hypervisor, and a fake one would assert that
-the fake behaves as expected — which is the shape of test this project exists to stop
-shipping. It is exercised by raising real scenarios.
+```
+npm test               the declaration layer, offline, 40 tests
+npm run test:integration   real scenarios against a real hypervisor, 10 tests
+npm run check          typecheck + both — this is the gate
+```
+
+**A test names the decision it defends** (`novox/hq` ADR 0034). A decision with no test is one
+that will quietly stop being true, and nobody learns that from a document:
+
+| Test | Defends |
+|---|---|
+| the lab provides the underlay and nothing of the overlay | ADR 0031 |
+| the workstation has no route into the scenario | ADR 0032 |
+| a router is a container while machines are virtual machines | ADR 0033 |
+| raise waits for *usable*, not for the call to return | the lifecycle design |
+| snapshots are whole-scenario | the lifecycle design |
+| a public range that is not documentation space is refused | the declaration design |
+| a scenario declaring what cannot be materialised is refused | the declaration design |
+
+**Mocking the hypervisor is forbidden.** A fake would assert that the fake behaves as expected,
+which is the shape of test this project exists to stop shipping. Integration tests skip with a
+reason on a machine that cannot raise scenarios, rather than passing green having checked
+nothing.
+
+That suite earned itself on its first run: it found that a snapshot of a running machine could
+miss a file written seconds earlier — not stale, **absent** — because the write was still in
+the guest's page cache. The design had listed that as an open question. The test answered it,
+and `snapshot` now flushes first.

package.json

@@ -8,7 +8,9 @@
   },
   "scripts": {
     "typecheck": "tsc --noEmit",
-    "test": "node --test --experimental-strip-types 'test/*.test.ts'"
+    "test": "node --test --experimental-strip-types 'test/*.test.ts'",
+    "test:integration": "node --test --experimental-strip-types 'test/integration/*.test.ts'",
+    "check": "npm run typecheck && npm test && npm run test:integration"
   },
   "dependencies": {
     "yaml": "^2.6.0"

src/lifecycle/operate.ts

@@ -65,13 +65,26 @@ export async function exec(
 /**
  * Capture the whole scenario as one state. Every machine, one name.
  *
- * Machines are snapshotted while running, so what is captured is the disk and not memory —
- * crash-consistent rather than a paused mesh. Whether a mesh restored that way is coherent
- * is an open question in the design, not something this silently assumes away.
+ * **Every machine is flushed first, and that is not a precaution.** A snapshot of a running
+ * machine captures its disk, not its memory, so a write still sitting in the guest's page
+ * cache is simply not in the snapshot. Without the flush a file written seconds earlier can
+ * be absent after restore — not stale, absent.
+ *
+ * Found by the integration test on its first run, which is the question the design listed as
+ * open: *does a scenario snapshot need the machines stopped?* It does not, but it does need
+ * them flushed.
+ *
+ * This buys write-durability, not application-consistency. A database mid-transaction is
+ * still captured mid-transaction — the snapshot is crash-consistent, and anything needing
+ * more has to quiesce itself.
  */
 export async function snapshot(instanceId: string, label: string): Promise<number> {
   const machines = await machinesOf(instanceId);
   const started = Date.now();
+
+  for (const name of machines) {
+    await incusOk(["exec", name, "--", "sync"], 60_000);
+  }
   for (const name of machines) {
     await incus(["snapshot", "create", name, label], 300_000);
   }

test/integration/harness.ts

@@ -0,0 +1,44 @@
+/**
+ * Integration tests run against a real hypervisor. Mocking it is forbidden — a test that
+ * fakes the system under integration asserts that the fake behaves as expected, which is
+ * the shape of test this project exists to stop shipping (novox/hq ADR 0034).
+ *
+ * Consequence, accepted: these are slow, and they need a machine that can raise scenarios.
+ * They skip rather than fail where it cannot, so that a machine without a hypervisor gets
+ * an honest "not run" instead of a green suite that checked nothing.
+ */
+
+import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts";
+import { destroy, list } from "../../src/lifecycle/operate.ts";
+
+export interface Capability {
+  usable: boolean;
+  why: string;
+}
+
+/** Can this machine run scenarios at all? Checked once, reported honestly. */
+export async function labIsUsable(): Promise<Capability> {
+  if (!(await isReachable())) {
+    return {
+      usable: false,
+      why: "the incus daemon is not reachable as this user (try MESH_LAB_INCUS='sudo -n incus')",
+    };
+  }
+  const drivers = await supportedDrivers();
+  if (!drivers.some((d) => d === "btrfs" || d === "zfs")) {
+    return { usable: false, why: "no copy-on-write driver — snapshots would be full copies" };
+  }
+  if (!(await pools()).some((p) => p.driver === "btrfs" || p.driver === "zfs")) {
+    return { usable: false, why: "no pool uses a copy-on-write driver" };
+  }
+  return { usable: true, why: "" };
+}
+
+/** Tear down anything a test left behind, whether it passed or not. */
+export async function destroyAll(prefix: string): Promise<void> {
+  for (const instance of await list()) {
+    if (instance.instanceId.startsWith(prefix)) {
+      await destroy(instance.instanceId);
+    }
+  }
+}

test/integration/underlay.test.ts

@@ -0,0 +1,133 @@
+/**
+ * Each test names the decision it defends. A decision with no test is one that will quietly
+ * stop being true (novox/hq ADR 0034).
+ */
+
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec, list, restore, snapshot } from "../../src/lifecycle/operate.ts";
+import { incus, incusOk } from "../../src/incus/client.ts";
+import { labIsUsable, destroyAll } from "./harness.ts";
+
+const capability = await labIsUsable();
+const skip = capability.usable ? false : `lab not usable: ${capability.why}`;
+
+let instanceId = "";
+
+before(async () => {
+  if (skip) return;
+  const scenario = loadScenario("scenarios/behind-nat.yml");
+  const raised = await raise(scenario, {});
+  instanceId = raised.instanceId;
+}, { timeout: 900_000 });
+
+after(async () => {
+  if (instanceId) await destroy(instanceId);
+}, { timeout: 400_000 });
+
+test("ADR 0031 — the lab provides the underlay and NOTHING of the overlay", { skip }, async () => {
+  // A scenario that pre-built peering would certify its own work. Whatever the mesh is
+  // responsible for must be absent from a freshly raised machine.
+  const { stdout } = await exec(instanceId, "home-server", [
+    "sh", "-c",
+    "ip link show type wireguard 2>/dev/null | wc -l; " +
+      "ls /etc/wireguard 2>/dev/null | wc -l; " +
+      "ls /etc/hal /etc/mesh 2>/dev/null | wc -l",
+  ]);
+  const counts = stdout.trim().split("\n").map((n) => Number(n.trim()));
+  assert.deepEqual(counts, [0, 0, 0], "a raised machine carries no overlay, no mesh config");
+}, { timeout: 120_000 });
+
+test("ADR 0031 — the declared address IS what the machine holds", { skip }, async () => {
+  const { stdout } = await exec(instanceId, "home-server", ["ip", "-o", "-4", "addr", "show"]);
+  assert.match(stdout, /192\.168\.1\.135\/24/);
+});
+
+test("design — raise waits for USABLE, not for the call to return", { skip }, async () => {
+  // The measured gap is 3.4s to 14.3s. Reporting the earlier number is transport reported
+  // as effect. If raise has returned, every machine must answer immediately.
+  for (const machine of ["anchor", "home-server"]) {
+    const { stdout } = await exec(instanceId, machine, ["sh", "-c", "echo alive"]);
+    assert.equal(stdout.trim(), "alive", `${machine} was not usable when raise returned`);
+  }
+}, { timeout: 120_000 });
+
+test("ADR 0033 — a router is scenery: containers, while machines are virtual machines", { skip }, async () => {
+  const json = (await incusOk(["list", "--format", "json"], 30_000)) ?? "[]";
+  const all = JSON.parse(json) as { name?: string; type?: string; config?: Record<string, string> }[];
+  const mine = all.filter((i) => i.config?.["user.mesh-lab.instance"] === instanceId);
+  assert.ok(mine.length >= 3, "expected machines and a router");
+
+  for (const item of mine) {
+    const isRouter = item.config?.["user.mesh-lab.router"] !== undefined;
+    assert.equal(
+      item.type,
+      isRouter ? "container" : "virtual-machine",
+      `${item.name} is a ${item.type} but ${isRouter ? "is" : "is not"} a router`,
+    );
+  }
+}, { timeout: 120_000 });
+
+test("design — NAT: a private address is not reachable from outside", { skip }, async () => {
+  const { stdout } = await exec(instanceId, "anchor", [
+    "sh", "-c", "ping -c1 -W2 192.168.1.135 >/dev/null 2>&1 && echo reachable || echo unreachable",
+  ]);
+  assert.equal(stdout.trim(), "unreachable");
+}, { timeout: 120_000 });
+
+test("design — published: reachable at the GATEWAY's address, never its own", { skip }, async () => {
+  await exec(instanceId, "home-server", [
+    "sh", "-c", "nohup python3 -m http.server 8080 --bind 0.0.0.0 >/tmp/s.log 2>&1 & sleep 2",
+  ]);
+  const { stdout } = await exec(instanceId, "anchor", [
+    "sh", "-c", "curl -s -m5 -o /dev/null -w '%{http_code}' http://192.0.2.50:8080/ || echo failed",
+  ]);
+  assert.equal(stdout.trim(), "200", "the forwarded port did not reach the machine behind NAT");
+}, { timeout: 180_000 });
+
+test("design — snapshots are WHOLE-scenario: restore returns every machine", { skip }, async () => {
+  // Restoring a subset would produce a mesh that has never existed, so faults found there
+  // would be artefacts of the lab.
+  await exec(instanceId, "anchor", ["sh", "-c", "echo dirty > /root/marker"]);
+  await exec(instanceId, "home-server", ["sh", "-c", "echo dirty > /root/marker"]);
+  await snapshot(instanceId, "test-point");
+
+  await exec(instanceId, "anchor", ["sh", "-c", "echo changed > /root/marker"]);
+  await exec(instanceId, "home-server", ["sh", "-c", "echo changed > /root/marker"]);
+  await restore(instanceId, "test-point");
+
+  for (const machine of ["anchor", "home-server"]) {
+    const { stdout } = await exec(instanceId, machine, ["cat", "/root/marker"]);
+    assert.equal(stdout.trim(), "dirty", `${machine} was not returned to the snapshot`);
+  }
+}, { timeout: 600_000 });
+
+test("design — restore leaves the scenario USABLE, not merely running", { skip }, async () => {
+  // The restore call returns in under a second while the agent is still starting. Reporting
+  // that as restored would be transport reported as effect.
+  const { stdout } = await exec(instanceId, "anchor", ["sh", "-c", "echo alive"]);
+  assert.equal(stdout.trim(), "alive");
+}, { timeout: 120_000 });
+
+test("ADR 0032 — the workstation has no route into the scenario", { skip }, async () => {
+  // Reachability is asked from INSIDE. If the workstation could reach a scenario address,
+  // two scenarios carrying the same prefix would put one's traffic in the other.
+  const { stdout } = await incus(["exec", `mlab-${instanceId}-anchor`, "--", "echo", "inside"]);
+  assert.equal(stdout.trim(), "inside", "exec is the only way in, and it works");
+}, { timeout: 60_000 });
+
+test("housekeeping — destroy removes machines, routers and segments", { skip }, async () => {
+  const before = (await list()).find((i) => i.instanceId === instanceId);
+  assert.ok(before, "the instance should exist before it is destroyed");
+
+  const { machines, networks } = await destroy(instanceId);
+  assert.ok(machines >= 3, `expected machines and a router, removed ${machines}`);
+  assert.ok(networks >= 2, `expected both segments removed, removed ${networks}`);
+
+  const after = (await list()).find((i) => i.instanceId === instanceId);
+  assert.equal(after, undefined, "the instance should be gone");
+  instanceId = "";
+  await destroyAll("behind-nat-");
+}, { timeout: 400_000 });