diff --git a/README.md b/README.md index 43a78ca..cebe26c 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,31 @@ npm test the declaration layer, offline npm run typecheck ``` -The lifecycle is not unit-tested. It talks to a hypervisor, and a fake one would assert that -the fake behaves as expected — which is the shape of test this project exists to stop -shipping. It is exercised by raising real scenarios. +``` +npm test the declaration layer, offline, 40 tests +npm run test:integration real scenarios against a real hypervisor, 10 tests +npm run check typecheck + both — this is the gate +``` + +**A test names the decision it defends** (`novox/hq` ADR 0034). A decision with no test is one +that will quietly stop being true, and nobody learns that from a document: + +| Test | Defends | +|---|---| +| the lab provides the underlay and nothing of the overlay | ADR 0031 | +| the workstation has no route into the scenario | ADR 0032 | +| a router is a container while machines are virtual machines | ADR 0033 | +| raise waits for *usable*, not for the call to return | the lifecycle design | +| snapshots are whole-scenario | the lifecycle design | +| a public range that is not documentation space is refused | the declaration design | +| a scenario declaring what cannot be materialised is refused | the declaration design | + +**Mocking the hypervisor is forbidden.** A fake would assert that the fake behaves as expected, +which is the shape of test this project exists to stop shipping. Integration tests skip with a +reason on a machine that cannot raise scenarios, rather than passing green having checked +nothing. + +That suite earned itself on its first run: it found that a snapshot of a running machine could +miss a file written seconds earlier — not stale, **absent** — because the write was still in +the guest's page cache. The design had listed that as an open question. The test answered it, +and `snapshot` now flushes first. diff --git a/package.json b/package.json index c6b3d42..6fa6c0b 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,9 @@ }, "scripts": { "typecheck": "tsc --noEmit", - "test": "node --test --experimental-strip-types 'test/*.test.ts'" + "test": "node --test --experimental-strip-types 'test/*.test.ts'", + "test:integration": "node --test --experimental-strip-types 'test/integration/*.test.ts'", + "check": "npm run typecheck && npm test && npm run test:integration" }, "dependencies": { "yaml": "^2.6.0" diff --git a/src/lifecycle/operate.ts b/src/lifecycle/operate.ts index 028f73b..f2f2180 100644 --- a/src/lifecycle/operate.ts +++ b/src/lifecycle/operate.ts @@ -65,13 +65,26 @@ export async function exec( /** * Capture the whole scenario as one state. Every machine, one name. * - * Machines are snapshotted while running, so what is captured is the disk and not memory — - * crash-consistent rather than a paused mesh. Whether a mesh restored that way is coherent - * is an open question in the design, not something this silently assumes away. + * **Every machine is flushed first, and that is not a precaution.** A snapshot of a running + * machine captures its disk, not its memory, so a write still sitting in the guest's page + * cache is simply not in the snapshot. Without the flush a file written seconds earlier can + * be absent after restore — not stale, absent. + * + * Found by the integration test on its first run, which is the question the design listed as + * open: *does a scenario snapshot need the machines stopped?* It does not, but it does need + * them flushed. + * + * This buys write-durability, not application-consistency. A database mid-transaction is + * still captured mid-transaction — the snapshot is crash-consistent, and anything needing + * more has to quiesce itself. */ export async function snapshot(instanceId: string, label: string): Promise { const machines = await machinesOf(instanceId); const started = Date.now(); + + for (const name of machines) { + await incusOk(["exec", name, "--", "sync"], 60_000); + } for (const name of machines) { await incus(["snapshot", "create", name, label], 300_000); } diff --git a/test/integration/harness.ts b/test/integration/harness.ts new file mode 100644 index 0000000..5b75e3f --- /dev/null +++ b/test/integration/harness.ts @@ -0,0 +1,44 @@ +/** + * Integration tests run against a real hypervisor. Mocking it is forbidden — a test that + * fakes the system under integration asserts that the fake behaves as expected, which is + * the shape of test this project exists to stop shipping (novox/hq ADR 0034). + * + * Consequence, accepted: these are slow, and they need a machine that can raise scenarios. + * They skip rather than fail where it cannot, so that a machine without a hypervisor gets + * an honest "not run" instead of a green suite that checked nothing. + */ + +import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts"; +import { destroy, list } from "../../src/lifecycle/operate.ts"; + +export interface Capability { + usable: boolean; + why: string; +} + +/** Can this machine run scenarios at all? Checked once, reported honestly. */ +export async function labIsUsable(): Promise { + if (!(await isReachable())) { + return { + usable: false, + why: "the incus daemon is not reachable as this user (try MESH_LAB_INCUS='sudo -n incus')", + }; + } + const drivers = await supportedDrivers(); + if (!drivers.some((d) => d === "btrfs" || d === "zfs")) { + return { usable: false, why: "no copy-on-write driver — snapshots would be full copies" }; + } + if (!(await pools()).some((p) => p.driver === "btrfs" || p.driver === "zfs")) { + return { usable: false, why: "no pool uses a copy-on-write driver" }; + } + return { usable: true, why: "" }; +} + +/** Tear down anything a test left behind, whether it passed or not. */ +export async function destroyAll(prefix: string): Promise { + for (const instance of await list()) { + if (instance.instanceId.startsWith(prefix)) { + await destroy(instance.instanceId); + } + } +} diff --git a/test/integration/underlay.test.ts b/test/integration/underlay.test.ts new file mode 100644 index 0000000..874b0a7 --- /dev/null +++ b/test/integration/underlay.test.ts @@ -0,0 +1,133 @@ +/** + * Each test names the decision it defends. A decision with no test is one that will quietly + * stop being true (novox/hq ADR 0034). + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec, list, restore, snapshot } from "../../src/lifecycle/operate.ts"; +import { incus, incusOk } from "../../src/incus/client.ts"; +import { labIsUsable, destroyAll } from "./harness.ts"; + +const capability = await labIsUsable(); +const skip = capability.usable ? false : `lab not usable: ${capability.why}`; + +let instanceId = ""; + +before(async () => { + if (skip) return; + const scenario = loadScenario("scenarios/behind-nat.yml"); + const raised = await raise(scenario, {}); + instanceId = raised.instanceId; +}, { timeout: 900_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); +}, { timeout: 400_000 }); + +test("ADR 0031 — the lab provides the underlay and NOTHING of the overlay", { skip }, async () => { + // A scenario that pre-built peering would certify its own work. Whatever the mesh is + // responsible for must be absent from a freshly raised machine. + const { stdout } = await exec(instanceId, "home-server", [ + "sh", "-c", + "ip link show type wireguard 2>/dev/null | wc -l; " + + "ls /etc/wireguard 2>/dev/null | wc -l; " + + "ls /etc/hal /etc/mesh 2>/dev/null | wc -l", + ]); + const counts = stdout.trim().split("\n").map((n) => Number(n.trim())); + assert.deepEqual(counts, [0, 0, 0], "a raised machine carries no overlay, no mesh config"); +}, { timeout: 120_000 }); + +test("ADR 0031 — the declared address IS what the machine holds", { skip }, async () => { + const { stdout } = await exec(instanceId, "home-server", ["ip", "-o", "-4", "addr", "show"]); + assert.match(stdout, /192\.168\.1\.135\/24/); +}); + +test("design — raise waits for USABLE, not for the call to return", { skip }, async () => { + // The measured gap is 3.4s to 14.3s. Reporting the earlier number is transport reported + // as effect. If raise has returned, every machine must answer immediately. + for (const machine of ["anchor", "home-server"]) { + const { stdout } = await exec(instanceId, machine, ["sh", "-c", "echo alive"]); + assert.equal(stdout.trim(), "alive", `${machine} was not usable when raise returned`); + } +}, { timeout: 120_000 }); + +test("ADR 0033 — a router is scenery: containers, while machines are virtual machines", { skip }, async () => { + const json = (await incusOk(["list", "--format", "json"], 30_000)) ?? "[]"; + const all = JSON.parse(json) as { name?: string; type?: string; config?: Record }[]; + const mine = all.filter((i) => i.config?.["user.mesh-lab.instance"] === instanceId); + assert.ok(mine.length >= 3, "expected machines and a router"); + + for (const item of mine) { + const isRouter = item.config?.["user.mesh-lab.router"] !== undefined; + assert.equal( + item.type, + isRouter ? "container" : "virtual-machine", + `${item.name} is a ${item.type} but ${isRouter ? "is" : "is not"} a router`, + ); + } +}, { timeout: 120_000 }); + +test("design — NAT: a private address is not reachable from outside", { skip }, async () => { + const { stdout } = await exec(instanceId, "anchor", [ + "sh", "-c", "ping -c1 -W2 192.168.1.135 >/dev/null 2>&1 && echo reachable || echo unreachable", + ]); + assert.equal(stdout.trim(), "unreachable"); +}, { timeout: 120_000 }); + +test("design — published: reachable at the GATEWAY's address, never its own", { skip }, async () => { + await exec(instanceId, "home-server", [ + "sh", "-c", "nohup python3 -m http.server 8080 --bind 0.0.0.0 >/tmp/s.log 2>&1 & sleep 2", + ]); + const { stdout } = await exec(instanceId, "anchor", [ + "sh", "-c", "curl -s -m5 -o /dev/null -w '%{http_code}' http://192.0.2.50:8080/ || echo failed", + ]); + assert.equal(stdout.trim(), "200", "the forwarded port did not reach the machine behind NAT"); +}, { timeout: 180_000 }); + +test("design — snapshots are WHOLE-scenario: restore returns every machine", { skip }, async () => { + // Restoring a subset would produce a mesh that has never existed, so faults found there + // would be artefacts of the lab. + await exec(instanceId, "anchor", ["sh", "-c", "echo dirty > /root/marker"]); + await exec(instanceId, "home-server", ["sh", "-c", "echo dirty > /root/marker"]); + await snapshot(instanceId, "test-point"); + + await exec(instanceId, "anchor", ["sh", "-c", "echo changed > /root/marker"]); + await exec(instanceId, "home-server", ["sh", "-c", "echo changed > /root/marker"]); + await restore(instanceId, "test-point"); + + for (const machine of ["anchor", "home-server"]) { + const { stdout } = await exec(instanceId, machine, ["cat", "/root/marker"]); + assert.equal(stdout.trim(), "dirty", `${machine} was not returned to the snapshot`); + } +}, { timeout: 600_000 }); + +test("design — restore leaves the scenario USABLE, not merely running", { skip }, async () => { + // The restore call returns in under a second while the agent is still starting. Reporting + // that as restored would be transport reported as effect. + const { stdout } = await exec(instanceId, "anchor", ["sh", "-c", "echo alive"]); + assert.equal(stdout.trim(), "alive"); +}, { timeout: 120_000 }); + +test("ADR 0032 — the workstation has no route into the scenario", { skip }, async () => { + // Reachability is asked from INSIDE. If the workstation could reach a scenario address, + // two scenarios carrying the same prefix would put one's traffic in the other. + const { stdout } = await incus(["exec", `mlab-${instanceId}-anchor`, "--", "echo", "inside"]); + assert.equal(stdout.trim(), "inside", "exec is the only way in, and it works"); +}, { timeout: 60_000 }); + +test("housekeeping — destroy removes machines, routers and segments", { skip }, async () => { + const before = (await list()).find((i) => i.instanceId === instanceId); + assert.ok(before, "the instance should exist before it is destroyed"); + + const { machines, networks } = await destroy(instanceId); + assert.ok(machines >= 3, `expected machines and a router, removed ${machines}`); + assert.ok(networks >= 2, `expected both segments removed, removed ${networks}`); + + const after = (await list()).find((i) => i.instanceId === instanceId); + assert.equal(after, undefined, "the instance should be gone"); + instanceId = ""; + await destroyAll("behind-nat-"); +}, { timeout: 400_000 });