package.json

@@ -9,7 +9,7 @@
   "scripts": {
     "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json",
     "test": "node --test --experimental-strip-types 'test/*.test.ts'",
-    "test:integration": "node --test --experimental-strip-types 'test/integration/*.test.ts'",
+    "test:integration": "node --test --test-concurrency=1 --experimental-strip-types 'test/integration/*.test.ts'",
     "check": "npm run typecheck && npm test && npm run test:integration"
   },
   "dependencies": {

test/integration/harness.ts

@@ -8,8 +8,12 @@
  * an honest "not run" instead of a green suite that checked nothing.
  */
 
+import assert from "node:assert/strict";
 import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts";
 import { destroy, list } from "../../src/lifecycle/operate.ts";
+import { diagramFromLive } from "../../src/diagram/from-live.ts";
+import { duplicateAddresses, describeConflicts, type Held } from "../../src/lifecycle/invariants.ts";
+import type { Scenario } from "../../src/declaration/types.ts";
 
 export interface Capability {
   usable: boolean;
@@ -42,3 +46,72 @@ export async function destroyAll(prefix: string): Promise<void> {
     }
   }
 }
+
+/**
+ * Every address the hypervisor says is held, by which machine, on which segment.
+ *
+ * Read through the live diagram because that is already the one place that joins addresses
+ * to devices by MAC and devices to segments by tag. A second reader would be a second thing
+ * to get wrong in the same way — and the way it was wrong once, a virtual machine's
+ * addresses silently going missing, is exactly what these assertions would then miss.
+ */
+export async function heldAddresses(instanceId: string): Promise<Held[]> {
+  const drawn = await diagramFromLive(instanceId);
+  return drawn.machines.flatMap((machine) =>
+    machine.attachments.flatMap((attachment) =>
+      attachment.addresses.map((address) => ({
+        machine: machine.name,
+        segment: attachment.segment,
+        address,
+      })),
+    ),
+  );
+}
+
+function bare(address: string): string {
+  const slash = address.lastIndexOf("/");
+  return slash === -1 ? address : address.slice(0, slash);
+}
+
+/**
+ * Invariants that hold of ANY raised scenario, whatever it declares.
+ *
+ * Asserted against what actually came up, never against the declaration — the declaration
+ * is what was accepted, and in the fault that prompted these, it was accepted.
+ */
+export async function assertUniversalInvariants(
+  scenario: Scenario,
+  instanceId: string,
+): Promise<void> {
+  const held = await heldAddresses(instanceId);
+  assert.ok(held.length > 0, `${scenario.scenario}: no addresses were read back at all`);
+
+  const conflicts = duplicateAddresses(held);
+  assert.deepEqual(
+    conflicts,
+    [],
+    `${scenario.scenario}: address conflict — ${describeConflicts(conflicts)}`,
+  );
+
+  // Every address the scenario declared is one the machine actually holds. A machine that
+  // came up bare looks identical to one that came up correctly until something asks it.
+  const holders = new Map<string, Set<string>>();
+  for (const entry of held) {
+    const key = `${entry.machine} ${entry.segment}`;
+    holders.set(key, (holders.get(key) ?? new Set<string>()).add(bare(entry.address)));
+  }
+
+  for (const [name, spec] of Object.entries(scenario.machines)) {
+    if (spec.at === "detached") continue;
+    for (const attachment of spec.at) {
+      const actual = holders.get(`${name} ${attachment.segment}`) ?? new Set<string>();
+      for (const address of attachment.address) {
+        assert.ok(
+          actual.has(address),
+          `${scenario.scenario}: '${name}' declared ${address} on '${attachment.segment}' ` +
+            `but holds ${[...actual].join(", ") || "nothing"}`,
+        );
+      }
+    }
+  }
+}

test/integration/scenarios.test.ts

@@ -0,0 +1,47 @@
+/**
+ * Every scenario, raised for real, checked against the invariants that hold of all of them.
+ *
+ * The suite next door raises one scenario and asks deep questions of it. This one asks
+ * shallow questions of every scenario, which is the half that was missing: both faults found
+ * by hand so far — two gateways holding one address, and a gateway drawn across an unrelated
+ * network — lived in scenarios nothing ever built.
+ *
+ * The list grows. Each entry costs a boot, so it is added deliberately rather than by
+ * globbing the directory: a scenario that is expensive and adds no new shape is not worth
+ * the wall clock, and one that is cheap and adds a shape is.
+ */
+
+import { test, after } from "node:test";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy } from "../../src/lifecycle/operate.ts";
+import { labIsUsable, destroyAll, assertUniversalInvariants } from "./harness.ts";
+
+const capability = await labIsUsable();
+const skip = capability.usable ? false : `lab not usable: ${capability.why}`;
+
+/** Grows one step at a time. Each addition is a boot, and a shape not covered before. */
+const SCENARIOS = [
+  "bootstrap-single", // one machine, one public network — the floor
+];
+
+const raised: string[] = [];
+
+after(async () => {
+  for (const instanceId of raised) await destroy(instanceId);
+}, { timeout: 600_000 });
+
+for (const name of SCENARIOS) {
+  test(`${name}: comes up holding what it declared, with no address held twice`, { skip, timeout: 900_000 }, async () => {
+    const scenario = loadScenario(`scenarios/${name}.yml`);
+    const instance = await raise(scenario, {});
+    raised.push(instance.instanceId);
+    await assertUniversalInvariants(scenario, instance.instanceId);
+  });
+}
+
+after(async () => {
+  // Anything a failed raise left standing. RaiseError leaves wreckage on purpose, which is
+  // right for a person debugging and wrong for the next run of the suite.
+  for (const name of SCENARIOS) await destroyAll(`${name}-`);
+}, { timeout: 600_000 });

test/integration/underlay.test.ts

@@ -9,11 +9,10 @@ import { loadScenario } from "../../src/declaration/parse.ts";
 import { raise } from "../../src/lifecycle/raise.ts";
 import { destroy, exec, list, restore, snapshot } from "../../src/lifecycle/operate.ts";
 import { incus, incusOk } from "../../src/incus/client.ts";
-import { labIsUsable, destroyAll } from "./harness.ts";
+import { labIsUsable, destroyAll, assertUniversalInvariants } from "./harness.ts";
 import { diagramFromLive } from "../../src/diagram/from-live.ts";
 import { diagramFromDeclaration } from "../../src/diagram/from-declaration.ts";
 import { toDrawio } from "../../src/diagram/drawio.ts";
-import { duplicateAddresses, describeConflicts } from "../../src/lifecycle/invariants.ts";
 
 const capability = await labIsUsable();
 const skip = capability.usable ? false : `lab not usable: ${capability.why}`;
@@ -122,23 +121,10 @@ test("ADR 0032 — the workstation has no route into the scenario", { skip, time
   assert.equal(stdout.trim(), "inside", "exec is the only way in, and it works");
 });
 
-test("no two machines hold one address on one segment", { skip }, async () => {
-  // True of ANY raised scenario, so it is asserted against whatever is standing rather than
-  // against something this test declares. Two gateways with the same public address became
-  // two containers both holding it, and the address resolved to whichever answered ARP last.
-  //
-  // Read from the hypervisor, never from the declaration — the declaration is what was
-  // accepted, and it was accepted.
-  const drawn = await diagramFromLive(instanceId);
-  const held = drawn.machines.flatMap((machine) =>
-    machine.attachments.flatMap((attachment) =>
-      attachment.addresses.map((address) => ({ machine: machine.name, segment: attachment.segment, address })),
-    ),
-  );
-  assert.ok(held.length > 0, "no addresses were read back at all");
-
-  const conflicts = duplicateAddresses(held);
-  assert.deepEqual(conflicts, [], `address conflict: ${describeConflicts(conflicts)}`);
+test("what came up holds what was declared, with no address held twice", { skip, timeout: 120_000 }, async () => {
+  // The same invariants every scenario is held to, asserted here too — this instance is
+  // already standing, so it costs nothing to ask.
+  await assertUniversalInvariants(loadScenario("scenarios/behind-nat.yml"), instanceId);
 });
 
 // The diagram tests read the instance the file raised, so they run before the one that