jschoubben
b015068921
The suite next door raises one scenario and asks deep questions of it. This one asks shallow questions of every scenario — the half that was missing, since both faults found by hand lived in scenarios nothing ever built. Adds bootstrap-single, the cheapest, and the loop that lets the list grow. Also adds the second universal invariant: every address a scenario declared is one the machine actually holds. A machine that came up bare looks identical to one that came up correctly until something asks it. Verified to bite rather than assumed: against a live instance, the real declaration passes and a declaration claiming an address nothing holds fails with 'anchor declared 192.0.2.99 on hosting but holds 192.0.2.10'. Integration now runs with --test-concurrency=1. Two files raise real instances, node --test runs files in parallel by default, and two concurrent runs of this suite already produced a whole-suite failure once — every test red, from resource contention rather than from any fault in the code. Gate: 45.7s -> 60.2s.
118 lines
4.5 KiB
TypeScript
118 lines
4.5 KiB
TypeScript
/**
|
|
* Integration tests run against a real hypervisor. Mocking it is forbidden — a test that
|
|
* fakes the system under integration asserts that the fake behaves as expected, which is
|
|
* the shape of test this project exists to stop shipping (novox/hq ADR 0034).
|
|
*
|
|
* Consequence, accepted: these are slow, and they need a machine that can raise scenarios.
|
|
* They skip rather than fail where it cannot, so that a machine without a hypervisor gets
|
|
* an honest "not run" instead of a green suite that checked nothing.
|
|
*/
|
|
|
|
import assert from "node:assert/strict";
|
|
import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts";
|
|
import { destroy, list } from "../../src/lifecycle/operate.ts";
|
|
import { diagramFromLive } from "../../src/diagram/from-live.ts";
|
|
import { duplicateAddresses, describeConflicts, type Held } from "../../src/lifecycle/invariants.ts";
|
|
import type { Scenario } from "../../src/declaration/types.ts";
|
|
|
|
export interface Capability {
|
|
usable: boolean;
|
|
why: string;
|
|
}
|
|
|
|
/** Can this machine run scenarios at all? Checked once, reported honestly. */
|
|
export async function labIsUsable(): Promise<Capability> {
|
|
if (!(await isReachable())) {
|
|
return {
|
|
usable: false,
|
|
why: "the incus daemon is not reachable as this user (try MESH_LAB_INCUS='sudo -n incus')",
|
|
};
|
|
}
|
|
const drivers = await supportedDrivers();
|
|
if (!drivers.some((d) => d === "btrfs" || d === "zfs")) {
|
|
return { usable: false, why: "no copy-on-write driver — snapshots would be full copies" };
|
|
}
|
|
if (!(await pools()).some((p) => p.driver === "btrfs" || p.driver === "zfs")) {
|
|
return { usable: false, why: "no pool uses a copy-on-write driver" };
|
|
}
|
|
return { usable: true, why: "" };
|
|
}
|
|
|
|
/** Tear down anything a test left behind, whether it passed or not. */
|
|
export async function destroyAll(prefix: string): Promise<void> {
|
|
for (const instance of await list()) {
|
|
if (instance.instanceId.startsWith(prefix)) {
|
|
await destroy(instance.instanceId);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Every address the hypervisor says is held, by which machine, on which segment.
|
|
*
|
|
* Read through the live diagram because that is already the one place that joins addresses
|
|
* to devices by MAC and devices to segments by tag. A second reader would be a second thing
|
|
* to get wrong in the same way — and the way it was wrong once, a virtual machine's
|
|
* addresses silently going missing, is exactly what these assertions would then miss.
|
|
*/
|
|
export async function heldAddresses(instanceId: string): Promise<Held[]> {
|
|
const drawn = await diagramFromLive(instanceId);
|
|
return drawn.machines.flatMap((machine) =>
|
|
machine.attachments.flatMap((attachment) =>
|
|
attachment.addresses.map((address) => ({
|
|
machine: machine.name,
|
|
segment: attachment.segment,
|
|
address,
|
|
})),
|
|
),
|
|
);
|
|
}
|
|
|
|
function bare(address: string): string {
|
|
const slash = address.lastIndexOf("/");
|
|
return slash === -1 ? address : address.slice(0, slash);
|
|
}
|
|
|
|
/**
|
|
* Invariants that hold of ANY raised scenario, whatever it declares.
|
|
*
|
|
* Asserted against what actually came up, never against the declaration — the declaration
|
|
* is what was accepted, and in the fault that prompted these, it was accepted.
|
|
*/
|
|
export async function assertUniversalInvariants(
|
|
scenario: Scenario,
|
|
instanceId: string,
|
|
): Promise<void> {
|
|
const held = await heldAddresses(instanceId);
|
|
assert.ok(held.length > 0, `${scenario.scenario}: no addresses were read back at all`);
|
|
|
|
const conflicts = duplicateAddresses(held);
|
|
assert.deepEqual(
|
|
conflicts,
|
|
[],
|
|
`${scenario.scenario}: address conflict — ${describeConflicts(conflicts)}`,
|
|
);
|
|
|
|
// Every address the scenario declared is one the machine actually holds. A machine that
|
|
// came up bare looks identical to one that came up correctly until something asks it.
|
|
const holders = new Map<string, Set<string>>();
|
|
for (const entry of held) {
|
|
const key = `${entry.machine} ${entry.segment}`;
|
|
holders.set(key, (holders.get(key) ?? new Set<string>()).add(bare(entry.address)));
|
|
}
|
|
|
|
for (const [name, spec] of Object.entries(scenario.machines)) {
|
|
if (spec.at === "detached") continue;
|
|
for (const attachment of spec.at) {
|
|
const actual = holders.get(`${name} ${attachment.segment}`) ?? new Set<string>();
|
|
for (const address of attachment.address) {
|
|
assert.ok(
|
|
actual.has(address),
|
|
`${scenario.scenario}: '${name}' declared ${address} on '${attachment.segment}' ` +
|
|
`but holds ${[...actual].join(", ") || "nothing"}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|