diff --git a/package.json b/package.json index 2f6b338..26b5ed2 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "scripts": { "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json", "test": "node --test --experimental-strip-types 'test/*.test.ts'", - "test:integration": "node --test --experimental-strip-types 'test/integration/*.test.ts'", + "test:integration": "node --test --test-concurrency=1 --experimental-strip-types 'test/integration/*.test.ts'", "check": "npm run typecheck && npm test && npm run test:integration" }, "dependencies": { diff --git a/test/integration/harness.ts b/test/integration/harness.ts index 5b75e3f..6f9f8c0 100644 --- a/test/integration/harness.ts +++ b/test/integration/harness.ts @@ -8,8 +8,12 @@ * an honest "not run" instead of a green suite that checked nothing. */ +import assert from "node:assert/strict"; import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts"; import { destroy, list } from "../../src/lifecycle/operate.ts"; +import { diagramFromLive } from "../../src/diagram/from-live.ts"; +import { duplicateAddresses, describeConflicts, type Held } from "../../src/lifecycle/invariants.ts"; +import type { Scenario } from "../../src/declaration/types.ts"; export interface Capability { usable: boolean; @@ -42,3 +46,72 @@ export async function destroyAll(prefix: string): Promise { } } } + +/** + * Every address the hypervisor says is held, by which machine, on which segment. + * + * Read through the live diagram because that is already the one place that joins addresses + * to devices by MAC and devices to segments by tag. A second reader would be a second thing + * to get wrong in the same way — and the way it was wrong once, a virtual machine's + * addresses silently going missing, is exactly what these assertions would then miss. + */ +export async function heldAddresses(instanceId: string): Promise { + const drawn = await diagramFromLive(instanceId); + return drawn.machines.flatMap((machine) => + machine.attachments.flatMap((attachment) => + attachment.addresses.map((address) => ({ + machine: machine.name, + segment: attachment.segment, + address, + })), + ), + ); +} + +function bare(address: string): string { + const slash = address.lastIndexOf("/"); + return slash === -1 ? address : address.slice(0, slash); +} + +/** + * Invariants that hold of ANY raised scenario, whatever it declares. + * + * Asserted against what actually came up, never against the declaration — the declaration + * is what was accepted, and in the fault that prompted these, it was accepted. + */ +export async function assertUniversalInvariants( + scenario: Scenario, + instanceId: string, +): Promise { + const held = await heldAddresses(instanceId); + assert.ok(held.length > 0, `${scenario.scenario}: no addresses were read back at all`); + + const conflicts = duplicateAddresses(held); + assert.deepEqual( + conflicts, + [], + `${scenario.scenario}: address conflict — ${describeConflicts(conflicts)}`, + ); + + // Every address the scenario declared is one the machine actually holds. A machine that + // came up bare looks identical to one that came up correctly until something asks it. + const holders = new Map>(); + for (const entry of held) { + const key = `${entry.machine} ${entry.segment}`; + holders.set(key, (holders.get(key) ?? new Set()).add(bare(entry.address))); + } + + for (const [name, spec] of Object.entries(scenario.machines)) { + if (spec.at === "detached") continue; + for (const attachment of spec.at) { + const actual = holders.get(`${name} ${attachment.segment}`) ?? new Set(); + for (const address of attachment.address) { + assert.ok( + actual.has(address), + `${scenario.scenario}: '${name}' declared ${address} on '${attachment.segment}' ` + + `but holds ${[...actual].join(", ") || "nothing"}`, + ); + } + } + } +} diff --git a/test/integration/scenarios.test.ts b/test/integration/scenarios.test.ts new file mode 100644 index 0000000..a2f213f --- /dev/null +++ b/test/integration/scenarios.test.ts @@ -0,0 +1,47 @@ +/** + * Every scenario, raised for real, checked against the invariants that hold of all of them. + * + * The suite next door raises one scenario and asks deep questions of it. This one asks + * shallow questions of every scenario, which is the half that was missing: both faults found + * by hand so far — two gateways holding one address, and a gateway drawn across an unrelated + * network — lived in scenarios nothing ever built. + * + * The list grows. Each entry costs a boot, so it is added deliberately rather than by + * globbing the directory: a scenario that is expensive and adds no new shape is not worth + * the wall clock, and one that is cheap and adds a shape is. + */ + +import { test, after } from "node:test"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy } from "../../src/lifecycle/operate.ts"; +import { labIsUsable, destroyAll, assertUniversalInvariants } from "./harness.ts"; + +const capability = await labIsUsable(); +const skip = capability.usable ? false : `lab not usable: ${capability.why}`; + +/** Grows one step at a time. Each addition is a boot, and a shape not covered before. */ +const SCENARIOS = [ + "bootstrap-single", // one machine, one public network — the floor +]; + +const raised: string[] = []; + +after(async () => { + for (const instanceId of raised) await destroy(instanceId); +}, { timeout: 600_000 }); + +for (const name of SCENARIOS) { + test(`${name}: comes up holding what it declared, with no address held twice`, { skip, timeout: 900_000 }, async () => { + const scenario = loadScenario(`scenarios/${name}.yml`); + const instance = await raise(scenario, {}); + raised.push(instance.instanceId); + await assertUniversalInvariants(scenario, instance.instanceId); + }); +} + +after(async () => { + // Anything a failed raise left standing. RaiseError leaves wreckage on purpose, which is + // right for a person debugging and wrong for the next run of the suite. + for (const name of SCENARIOS) await destroyAll(`${name}-`); +}, { timeout: 600_000 }); diff --git a/test/integration/underlay.test.ts b/test/integration/underlay.test.ts index d9960e1..c9f3f8d 100644 --- a/test/integration/underlay.test.ts +++ b/test/integration/underlay.test.ts @@ -9,11 +9,10 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec, list, restore, snapshot } from "../../src/lifecycle/operate.ts"; import { incus, incusOk } from "../../src/incus/client.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, assertUniversalInvariants } from "./harness.ts"; import { diagramFromLive } from "../../src/diagram/from-live.ts"; import { diagramFromDeclaration } from "../../src/diagram/from-declaration.ts"; import { toDrawio } from "../../src/diagram/drawio.ts"; -import { duplicateAddresses, describeConflicts } from "../../src/lifecycle/invariants.ts"; const capability = await labIsUsable(); const skip = capability.usable ? false : `lab not usable: ${capability.why}`; @@ -122,23 +121,10 @@ test("ADR 0032 — the workstation has no route into the scenario", { skip, time assert.equal(stdout.trim(), "inside", "exec is the only way in, and it works"); }); -test("no two machines hold one address on one segment", { skip }, async () => { - // True of ANY raised scenario, so it is asserted against whatever is standing rather than - // against something this test declares. Two gateways with the same public address became - // two containers both holding it, and the address resolved to whichever answered ARP last. - // - // Read from the hypervisor, never from the declaration — the declaration is what was - // accepted, and it was accepted. - const drawn = await diagramFromLive(instanceId); - const held = drawn.machines.flatMap((machine) => - machine.attachments.flatMap((attachment) => - attachment.addresses.map((address) => ({ machine: machine.name, segment: attachment.segment, address })), - ), - ); - assert.ok(held.length > 0, "no addresses were read back at all"); - - const conflicts = duplicateAddresses(held); - assert.deepEqual(conflicts, [], `address conflict: ${describeConflicts(conflicts)}`); +test("what came up holds what was declared, with no address held twice", { skip, timeout: 120_000 }, async () => { + // The same invariants every scenario is held to, asserted here too — this instance is + // already standing, so it costs nothing to ask. + await assertUniversalInvariants(loadScenario("scenarios/behind-nat.yml"), instanceId); }); // The diagram tests read the instance the file raised, so they run before the one that