Scenario lifecycle: raise, exec, snapshot, restore, destroy
A declaration goes in and a disposable mesh comes out. Verified on a workstation, not asserted: two machines raised and addressed in 14.6s, snapshot 0.28s, restore-to-usable 11.6s, both families pinging with no loss, and the workstation with no route into any of it. The declaration layer implements the model in full — three positions a machine can be in, keyed on forwardability; gateways carrying the address the world sees them as; both address families; multi-homing; MTU; inter-segment policy. It is validated hard because the failures it prevents are silent: a private range on a public segment produces no error, the mesh simply never forms. Public segments are refused unless they use RFC 5737 or RFC 3849 space, and a range wider than the reserved block is refused too. 33 tests, all offline. The runtime implements less than the model, and refuses the difference. A scenario declaring gateways, published ports, policy, inbound deny or place is rejected at raise with every gap named. Raising it would produce a mesh that silently lacks what it declared, which is the fault this lab exists to catch — 04-ISSUES/003, where a firewall key is declared in five manifests and read by no code. Three bugs found by review and by running it, all of one family: The readiness check truthiness-tested incusOk's return. `exec … true` succeeds with EMPTY output, so every machine reported unreachable while incus exec on it worked perfectly. succeeds() now exists so the mistake is not available, and network delete had the same bug — it counted zero segments removed while removing them. list() split instance from machine on the last dash, so a machine called home-server absorbed half the instance id and destroy found nothing. Resources are now found by the metadata they carry, never by name. restore reported success in 0.79s while the machine's agent was still starting, so the next command failed. Both raise and restore now wait for usable and say how long that took — reporting the earlier number is transport reported as effect, which is the fault the lab is being built to find. Two incus behaviours worth recording. Its CLI reads a YAML definition from stdin when stdin is not a terminal, so a spawned command hangs until the timeout kills it and arrives with empty stderr — a failure with no explanation, on a command that works when typed. And it assigns a MAC at runtime without recording it in device config, so MACs are derived and set explicitly, which the guest needs anyway: it names interfaces by bus position, and matching by name configures the wrong one on a multi-homed machine. No build step; Node strips the types. The lifecycle has no unit tests because a fake hypervisor would assert that the fake behaves as expected, which is the shape of test this project exists to stop shipping.
This commit is contained in:
Executable
+188
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env -S node --experimental-strip-types
|
||||
/**
|
||||
* The lab's two callers want different things from the same verbs: a coordinator wants
|
||||
* structured results and clean teardown, a person wants readable output and the failing
|
||||
* scenario left standing. Both use these verbs; the difference is what happens after.
|
||||
*/
|
||||
|
||||
import { loadScenario } from "./declaration/parse.ts";
|
||||
import { DeclarationError } from "./declaration/validate.ts";
|
||||
import { isReachable, supportedDrivers, pools } from "./incus/client.ts";
|
||||
import { raise, RaiseError } from "./lifecycle/raise.ts";
|
||||
import { assertSupported, UnsupportedError } from "./lifecycle/supported.ts";
|
||||
import { destroy, exec, list, restore, snapshot, snapshots } from "./lifecycle/operate.ts";
|
||||
|
||||
const USAGE = `mesh-lab — raise a disposable mesh on one machine
|
||||
|
||||
check verify this machine can run scenarios
|
||||
validate <scenario.yml> parse and check a declaration, raising nothing
|
||||
raise <scenario.yml> materialise it, and wait until the machines are usable
|
||||
list scenario instances currently standing
|
||||
exec <instance> <machine> -- <cmd...>
|
||||
snapshot <instance> <label> capture the whole scenario as one state
|
||||
restore <instance> <label> return the whole scenario to it
|
||||
snapshots <instance>
|
||||
destroy <instance>
|
||||
|
||||
Set MESH_LAB_INCUS if the daemon needs a different invocation, e.g. "sudo -n incus".
|
||||
`;
|
||||
|
||||
function fail(message: string): never {
|
||||
console.error(message);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuse to run degraded rather than warning. A warning about a slow inner loop is read
|
||||
* once and ignored forever, and the loop stays slow.
|
||||
*/
|
||||
async function check(): Promise<void> {
|
||||
const problems: string[] = [];
|
||||
|
||||
if (!(await isReachable())) {
|
||||
problems.push(
|
||||
"the incus daemon is not reachable as this user. If the group was granted recently, " +
|
||||
"a session that predates it cannot see it — log out and back in, or set MESH_LAB_INCUS.",
|
||||
);
|
||||
console.error(problems.map((p) => ` ✗ ${p}`).join("\n"));
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(" ✓ daemon reachable");
|
||||
|
||||
const drivers = await supportedDrivers();
|
||||
const cowDrivers = drivers.filter((d) => d === "btrfs" || d === "zfs");
|
||||
if (cowDrivers.length === 0) {
|
||||
problems.push(
|
||||
"no copy-on-write storage driver is offered. Snapshots would be full copies — " +
|
||||
"measured at roughly 76x slower, which does not make the lab slow, it makes it unused.",
|
||||
);
|
||||
} else {
|
||||
console.log(` ✓ copy-on-write driver available (${cowDrivers.join(", ")})`);
|
||||
}
|
||||
|
||||
const available = await pools();
|
||||
const cowPool = available.find((p) => p.driver === "btrfs" || p.driver === "zfs");
|
||||
if (!cowPool) {
|
||||
problems.push(
|
||||
`no pool uses a copy-on-write driver (have: ${available.map((p) => `${p.name}/${p.driver}`).join(", ") || "none"}). ` +
|
||||
"A pool that exists and is the slow kind is the failure with no symptom.",
|
||||
);
|
||||
} else {
|
||||
console.log(` ✓ copy-on-write pool '${cowPool.name}' (${cowPool.driver})`);
|
||||
}
|
||||
|
||||
if (problems.length > 0) {
|
||||
console.error(`\n${problems.map((p) => ` ✗ ${p}`).join("\n\n")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("\nthis machine can run scenarios");
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const [verb, ...rest] = process.argv.slice(2);
|
||||
|
||||
switch (verb) {
|
||||
case "check":
|
||||
return check();
|
||||
|
||||
case "validate": {
|
||||
const path = rest[0] ?? fail("validate needs a scenario file");
|
||||
const scenario = loadScenario(path);
|
||||
console.log(
|
||||
`${scenario.scenario}: ${Object.keys(scenario.segments).length} segments, ` +
|
||||
`${Object.keys(scenario.machines).length} machines — valid`,
|
||||
);
|
||||
// Valid and raisable are different questions, and a scenario can be the first
|
||||
// without being the second.
|
||||
try {
|
||||
assertSupported(scenario);
|
||||
} catch (err) {
|
||||
if (err instanceof UnsupportedError) {
|
||||
console.log(`\nnot yet raisable:\n - ${err.missing.join("\n - ")}`);
|
||||
} else throw err;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
case "raise": {
|
||||
const path = rest[0] ?? fail("raise needs a scenario file");
|
||||
const scenario = loadScenario(path);
|
||||
const started = Date.now();
|
||||
const raised = await raise(scenario, {
|
||||
onProgress: (m) => console.log(m),
|
||||
...(process.env["MESH_LAB_IMAGE"] ? { image: process.env["MESH_LAB_IMAGE"] } : {}),
|
||||
});
|
||||
const seconds = ((Date.now() - started) / 1000).toFixed(1);
|
||||
console.log(`\nraised ${raised.instanceId} in ${seconds}s — ${raised.machines.length} machines usable`);
|
||||
if (scenario.snapshot) {
|
||||
const took = await snapshot(raised.instanceId, scenario.snapshot);
|
||||
console.log(`snapshot '${scenario.snapshot}' in ${took.toFixed(2)}s`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
case "list": {
|
||||
const found = await list();
|
||||
if (found.length === 0) return console.log("no scenario instances standing");
|
||||
for (const instance of found) {
|
||||
console.log(`${instance.instanceId}`);
|
||||
for (const m of instance.machines) console.log(` ${m.machine.padEnd(16)} ${m.status}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
case "exec": {
|
||||
const [instanceId, machine, ...command] = rest;
|
||||
if (!instanceId || !machine || command.length === 0) fail("exec <instance> <machine> <cmd...>");
|
||||
const result = await exec(instanceId, machine, command.filter((c) => c !== "--"));
|
||||
process.stdout.write(result.stdout);
|
||||
process.stderr.write(result.stderr);
|
||||
return;
|
||||
}
|
||||
|
||||
case "snapshot": {
|
||||
const [instanceId, label] = rest;
|
||||
if (!instanceId || !label) fail("snapshot <instance> <label>");
|
||||
const took = await snapshot(instanceId, label);
|
||||
console.log(`snapshot '${label}' in ${took.toFixed(2)}s`);
|
||||
return;
|
||||
}
|
||||
|
||||
case "restore": {
|
||||
const [instanceId, label] = rest;
|
||||
if (!instanceId || !label) fail("restore <instance> <label>");
|
||||
const { restoreSeconds, usableSeconds } = await restore(instanceId, label, 180, (m) =>
|
||||
console.log(m),
|
||||
);
|
||||
console.log(
|
||||
`restored '${label}' in ${restoreSeconds.toFixed(2)}s — usable again after ` +
|
||||
`${usableSeconds.toFixed(1)}s`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
case "snapshots": {
|
||||
const instanceId = rest[0] ?? fail("snapshots <instance>");
|
||||
const found = await snapshots(instanceId);
|
||||
console.log(found.length ? found.join("\n") : "none");
|
||||
return;
|
||||
}
|
||||
|
||||
case "destroy": {
|
||||
const instanceId = rest[0] ?? fail("destroy <instance>");
|
||||
const { machines, networks } = await destroy(instanceId);
|
||||
console.log(`destroyed ${instanceId} — ${machines} machines, ${networks} segments`);
|
||||
return;
|
||||
}
|
||||
|
||||
default:
|
||||
console.log(USAGE);
|
||||
process.exit(verb ? 1 : 0);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err: unknown) => {
|
||||
if (err instanceof DeclarationError || err instanceof RaiseError) fail(err.message);
|
||||
if (err instanceof UnsupportedError) fail(err.message);
|
||||
fail(err instanceof Error ? err.message : String(err));
|
||||
});
|
||||
Reference in New Issue
Block a user