233 lines
11 KiB
TypeScript
233 lines
11 KiB
TypeScript
/**
|
|
* Integration tests run against a real hypervisor. Mocking it is forbidden — a test that
|
|
* fakes the system under integration asserts that the fake behaves as expected, which is
|
|
* the shape of test this project exists to stop shipping (novox/hq ADR 0017).
|
|
*
|
|
* Consequence, accepted: these are slow, and they need a machine that can raise scenarios.
|
|
* They skip rather than fail where it cannot, so that a machine without a hypervisor gets
|
|
* an honest "not run" instead of a green suite that checked nothing.
|
|
*/
|
|
|
|
import assert from "node:assert/strict";
|
|
import { readFileSync } from "node:fs";
|
|
import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts";
|
|
import { destroy, list } from "../../src/lifecycle/operate.ts";
|
|
import { diagramFromLive } from "../../src/diagram/from-live.ts";
|
|
import { duplicateAddresses, describeConflicts, type Held } from "../../src/lifecycle/invariants.ts";
|
|
import type { Scenario } from "../../src/declaration/types.ts";
|
|
import { mustBeHandedOver, pinnedInto, referenceFor, repositoryOf, type HeldImage } from "../../src/pinning.ts";
|
|
|
|
// --- the foundation bundle, and what its three images are on a real machine ---------------------
|
|
|
|
/**
|
|
* The example bundle in mesh-host names a registry that no longer exists.
|
|
*
|
|
* `examples/foundation-first-node.lock` was written **for a target**, and the target was the lab: it
|
|
* pins `192.0.2.250:5000/…` because that is where the registry the lab used to raise served from.
|
|
* That registry is gone, so those three references name nothing.
|
|
*
|
|
* Two of them are ordinary third-party images and belong to the internet. Rather than invent
|
|
* digests here, they are the ones the mesh's own modules already pin — mesh-catalog's `postgres`
|
|
* and `lavinmq` — so the foundation's store and broker are literally the images the mesh runs. The
|
|
* third, mesh-controller, exists in no registry at all and becomes the ID the machine holds it under.
|
|
*
|
|
* **The bundle itself should be fixed in mesh-host**, and this substitution deleted with it. It is
|
|
* here because the file lives in another repository and because a fixture that lies about where an
|
|
* image comes from is exactly what this change is removing.
|
|
*/
|
|
const UPSTREAM_STORE =
|
|
"postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
|
|
const UPSTREAM_BROKER =
|
|
"cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b";
|
|
|
|
/**
|
|
* The foundation bundle as a machine should receive it.
|
|
*
|
|
* Third-party references become upstream ones, which the machine pulls over its uplink; ours
|
|
* become the ID the machine was handed. Nothing points inside the scenario any more, which is the
|
|
* whole of this change: what the bed proves about a bootstrap is now what would happen anywhere.
|
|
*/
|
|
export function foundationBundle(path: string, held: HeldImage[]): string {
|
|
let text = readFileSync(path, "utf8");
|
|
text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE);
|
|
text = text.replaceAll(
|
|
/[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER);
|
|
return pinnedInto(text, held);
|
|
}
|
|
|
|
/**
|
|
* The upstream reference for a third-party image, as the mesh's own catalogue pins it.
|
|
*
|
|
* A bed that writes a manifest by hand still has to name an image exactly — mesh-host refuses a
|
|
* tag, and rightly (novox/hq ADR 0006). While the lab had a registry the beds sidestepped that by
|
|
* naming a repository and letting the rewrite supply a digest; there is nothing to supply one now,
|
|
* so the digest has to be written down.
|
|
*
|
|
* These are the digests mesh-catalog's own modules pin, taken from `mesh-catalog/modules/*` — so a
|
|
* bed runs the image the mesh runs, and a bed that drifts from the catalogue is a bed testing a
|
|
* different postgres than the mesh ships.
|
|
*/
|
|
const UPSTREAM = new Map<string, string>([
|
|
["alpine", "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"],
|
|
["baserow/baserow", "baserow/baserow@sha256:834424a10413798567f76428f255dc259445b7f8dcec56598c05b4073bb2a124"],
|
|
["cloudamqp/lavinmq", "cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b"],
|
|
["eclipse-mosquitto", "eclipse-mosquitto@sha256:6f8d8a947c506f8a2290ec65cd4bd2bc7cb4d43fb5f6271f861cb013e2ef9797"],
|
|
["ghcr.io/umami-software/umami", "ghcr.io/umami-software/umami@sha256:fa32d116cf20cad52cbc3fad9a63b46e7fa02299d8f967168eb453d49c476b4a"],
|
|
["letta/letta", "letta/letta@sha256:1d2e0692514287c5ed1a483e14e16ed945f8632d315539f5e66373bb7d7c471b"],
|
|
["lscr.io/linuxserver/radarr", "lscr.io/linuxserver/radarr@sha256:119aaa4a4f7349bcd2a136c5373a0d7925b5479915c7dfe0c0ad352db2a6d438"],
|
|
["lscr.io/linuxserver/sonarr", "lscr.io/linuxserver/sonarr@sha256:c19aa4ecdf03d73e1d5c901da33744cb7eb4d921f89bafed1ca264601d7fa224"],
|
|
["lscr.io/linuxserver/unifi-controller", "lscr.io/linuxserver/unifi-controller@sha256:fcd5d8b13a77a588c79c1b49e5fc9ad08115aa3bb1a3576c589c64908a68845f"],
|
|
["minio/minio", "quay.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e"], // docker.io denies anonymous pulls; the catalogue pins quay.io (mesh-catalog #29)
|
|
["mongo", "mongo@sha256:e3fa459b4f4b72f3257c67a23c145e250b8b5700f033860392c68539b998bbe3"],
|
|
["ollama/ollama", "ollama/ollama@sha256:32931b46719f673c05fdbaa81ccb26da18ea4a1c57590a754874ab28ba269eb2"],
|
|
["postgres", "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee"],
|
|
["redis", "redis@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"],
|
|
["registry", "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"],
|
|
["synesthesiam/marytts", "synesthesiam/marytts@sha256:45970ecb3e21a2981c66c60563a70cf00be8e95c02565e7d74b3a73dcec7db2c"],
|
|
]);
|
|
|
|
/**
|
|
* What a manifest's image reference becomes on the machine.
|
|
*
|
|
* Four cases, and the second one is the whole change:
|
|
*
|
|
* - **Ours** becomes the ID the machine holds it under. Nothing serves it, and nothing needs to.
|
|
* - **Anything already pinned by digest** is returned exactly as written. The machine pulls it
|
|
* from the internet, over its uplink, which is what a real machine does and what the lab spent
|
|
* a long time serving from a registry of its own instead.
|
|
* - **A bare repository a bed names by hand** is given the digest mesh-catalog pins for it, so a
|
|
* bed runs the image the mesh ships. A tag would be refused by mesh-host anyway.
|
|
* - **A tag this harness has never heard of** is passed through untouched, and said out loud.
|
|
*
|
|
* That last case is not politeness, it is a finding the lab's registry was hiding. Seven catalogue
|
|
* modules name `registry-api.…/novox/…:latest` — a TAG, which ADR 0006 forbids and mesh-host
|
|
* refuses. It never showed, because the rewrite replaced every reference with a digest the lab's
|
|
* registry had assigned, tag or not. There is nothing to replace it with now, and the honest
|
|
* outcome is that those modules fail to apply, saying exactly why, on the node that carries them —
|
|
* rather than an assertion here taking the whole bed down before it starts.
|
|
*/
|
|
export function onTheMachine(reference: string, held: HeldImage[]): string {
|
|
if (mustBeHandedOver(reference)) {
|
|
const found = referenceFor(held, repositoryOf(reference));
|
|
assert.ok(
|
|
found,
|
|
`nothing loaded ${reference} onto the machines. They hold:\n ` +
|
|
held.map((i) => `${i.repository} ${i.reference}`).join("\n "),
|
|
);
|
|
return found;
|
|
}
|
|
if (reference.includes("@sha256:")) return reference;
|
|
|
|
const upstream = UPSTREAM.get(repositoryOf(reference));
|
|
if (upstream) return upstream;
|
|
|
|
console.log(
|
|
`UNPINNED: ${reference} names a tag, not a digest. The host will refuse it (novox/hq ` +
|
|
`ADR 0006). The lab's own registry used to paper over this by assigning a digest to ` +
|
|
`whatever was pushed; nothing does now. Fix the manifest, or add its digest to the ` +
|
|
`harness's UPSTREAM table.`,
|
|
);
|
|
return reference;
|
|
}
|
|
|
|
export interface Capability {
|
|
usable: boolean;
|
|
why: string;
|
|
}
|
|
|
|
/** Can this machine run scenarios at all? Checked once, reported honestly. */
|
|
export async function labIsUsable(): Promise<Capability> {
|
|
if (!(await isReachable())) {
|
|
return {
|
|
usable: false,
|
|
why: "the incus daemon is not reachable as this user (try MESH_LAB_INCUS='sudo -n incus')",
|
|
};
|
|
}
|
|
const drivers = await supportedDrivers();
|
|
if (!drivers.some((d) => d === "btrfs" || d === "zfs")) {
|
|
return { usable: false, why: "no copy-on-write driver — snapshots would be full copies" };
|
|
}
|
|
if (!(await pools()).some((p) => p.driver === "btrfs" || p.driver === "zfs")) {
|
|
return { usable: false, why: "no pool uses a copy-on-write driver" };
|
|
}
|
|
return { usable: true, why: "" };
|
|
}
|
|
|
|
/** Tear down anything a test left behind, whether it passed or not. */
|
|
export async function destroyAll(prefix: string): Promise<void> {
|
|
for (const instance of await list()) {
|
|
if (instance.instanceId.startsWith(prefix)) {
|
|
await destroy(instance.instanceId);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Every address the hypervisor says is held, by which machine, on which segment.
|
|
*
|
|
* Read through the live diagram because that is already the one place that joins addresses
|
|
* to devices by MAC and devices to segments by tag. A second reader would be a second thing
|
|
* to get wrong in the same way — and the way it was wrong once, a virtual machine's
|
|
* addresses silently going missing, is exactly what these assertions would then miss.
|
|
*/
|
|
export async function heldAddresses(instanceId: string): Promise<Held[]> {
|
|
const drawn = await diagramFromLive(instanceId);
|
|
return drawn.machines.flatMap((machine) =>
|
|
machine.attachments.flatMap((attachment) =>
|
|
attachment.addresses.map((address) => ({
|
|
machine: machine.name,
|
|
segment: attachment.segment,
|
|
address,
|
|
})),
|
|
),
|
|
);
|
|
}
|
|
|
|
function bare(address: string): string {
|
|
const slash = address.lastIndexOf("/");
|
|
return slash === -1 ? address : address.slice(0, slash);
|
|
}
|
|
|
|
/**
|
|
* Invariants that hold of ANY raised scenario, whatever it declares.
|
|
*
|
|
* Asserted against what actually came up, never against the declaration — the declaration
|
|
* is what was accepted, and in the fault that prompted these, it was accepted.
|
|
*/
|
|
export async function assertUniversalInvariants(
|
|
scenario: Scenario,
|
|
instanceId: string,
|
|
): Promise<void> {
|
|
const held = await heldAddresses(instanceId);
|
|
assert.ok(held.length > 0, `${scenario.scenario}: no addresses were read back at all`);
|
|
|
|
const conflicts = duplicateAddresses(held);
|
|
assert.deepEqual(
|
|
conflicts,
|
|
[],
|
|
`${scenario.scenario}: address conflict — ${describeConflicts(conflicts)}`,
|
|
);
|
|
|
|
// Every address the scenario declared is one the machine actually holds. A machine that
|
|
// came up bare looks identical to one that came up correctly until something asks it.
|
|
const holders = new Map<string, Set<string>>();
|
|
for (const entry of held) {
|
|
const key = `${entry.machine} ${entry.segment}`;
|
|
holders.set(key, (holders.get(key) ?? new Set<string>()).add(bare(entry.address)));
|
|
}
|
|
|
|
for (const [name, spec] of Object.entries(scenario.machines)) {
|
|
if (spec.at === "detached") continue;
|
|
for (const attachment of spec.at) {
|
|
const actual = holders.get(`${name} ${attachment.segment}`) ?? new Set<string>();
|
|
for (const address of attachment.address) {
|
|
assert.ok(
|
|
actual.has(address),
|
|
`${scenario.scenario}: '${name}' declared ${address} on '${attachment.segment}' ` +
|
|
`but holds ${[...actual].join(", ") || "nothing"}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|