mesh-lab/test/integration/harness.ts

/**
 * Integration tests run against a real hypervisor. Mocking it is forbidden — a test that
 * fakes the system under integration asserts that the fake behaves as expected, which is
 * the shape of test this project exists to stop shipping (novox/hq ADR 0017).
 *
 * Consequence, accepted: these are slow, and they need a machine that can raise scenarios.
 * They skip rather than fail where it cannot, so that a machine without a hypervisor gets
 * an honest "not run" instead of a green suite that checked nothing.
 */

import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts";
import { destroy, list } from "../../src/lifecycle/operate.ts";
import { diagramFromLive } from "../../src/diagram/from-live.ts";
import { duplicateAddresses, describeConflicts, type Held } from "../../src/lifecycle/invariants.ts";
import type { Scenario } from "../../src/declaration/types.ts";
import { mustBeHandedOver, pinnedInto, referenceFor, repositoryOf, type HeldImage } from "../../src/pinning.ts";

// --- the foundation bundle, and what its three images are on a real machine ---------------------

/**
 * The example bundle in mesh-host names a registry that no longer exists.
 *
 * `examples/foundation-first-node.lock` was written **for a target**, and the target was the lab: it
 * pins `192.0.2.250:5000/…` because that is where the registry the lab used to raise served from.
 * That registry is gone, so those three references name nothing.
 *
 * Two of them are ordinary third-party images and belong to the internet. Rather than invent
 * digests here, they are the ones the mesh's own modules already pin — mesh-catalog's `postgres`
 * and `lavinmq` — so the foundation's store and broker are literally the images the mesh runs. The
 * third, mesh-controller, exists in no registry at all and becomes the ID the machine holds it under.
 *
 * **The bundle itself should be fixed in mesh-host**, and this substitution deleted with it. It is
 * here because the file lives in another repository and because a fixture that lies about where an
 * image comes from is exactly what this change is removing.
 */
const UPSTREAM_STORE =
  "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
const UPSTREAM_BROKER =
  "cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b";

/**
 * The foundation bundle as a machine should receive it.
 *
 * Third-party references become upstream ones, which the machine pulls over its uplink; ours
 * become the ID the machine was handed. Nothing points inside the scenario any more, which is the
 * whole of this change: what the bed proves about a bootstrap is now what would happen anywhere.
 */
export function foundationBundle(path: string, held: HeldImage[]): string {
  let text = readFileSync(path, "utf8");
  text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE);
  text = text.replaceAll(
    /[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER);
  return pinnedInto(text, held);
}

/**
 * The upstream reference for a third-party image, as the mesh's own catalogue pins it.
 *
 * A bed that writes a manifest by hand still has to name an image exactly — mesh-host refuses a
 * tag, and rightly (novox/hq ADR 0006). While the lab had a registry the beds sidestepped that by
 * naming a repository and letting the rewrite supply a digest; there is nothing to supply one now,
 * so the digest has to be written down.
 *
 * These are the digests mesh-catalog's own modules pin, taken from `mesh-catalog/modules/*` — so a
 * bed runs the image the mesh runs, and a bed that drifts from the catalogue is a bed testing a
 * different postgres than the mesh ships.
 */
const UPSTREAM = new Map<string, string>([
  ["alpine", "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"],
  ["baserow/baserow", "baserow/baserow@sha256:834424a10413798567f76428f255dc259445b7f8dcec56598c05b4073bb2a124"],
  ["cloudamqp/lavinmq", "cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b"],
  ["eclipse-mosquitto", "eclipse-mosquitto@sha256:6f8d8a947c506f8a2290ec65cd4bd2bc7cb4d43fb5f6271f861cb013e2ef9797"],
  ["ghcr.io/umami-software/umami", "ghcr.io/umami-software/umami@sha256:fa32d116cf20cad52cbc3fad9a63b46e7fa02299d8f967168eb453d49c476b4a"],
  ["letta/letta", "letta/letta@sha256:1d2e0692514287c5ed1a483e14e16ed945f8632d315539f5e66373bb7d7c471b"],
  ["lscr.io/linuxserver/radarr", "lscr.io/linuxserver/radarr@sha256:119aaa4a4f7349bcd2a136c5373a0d7925b5479915c7dfe0c0ad352db2a6d438"],
  ["lscr.io/linuxserver/sonarr", "lscr.io/linuxserver/sonarr@sha256:c19aa4ecdf03d73e1d5c901da33744cb7eb4d921f89bafed1ca264601d7fa224"],
  ["lscr.io/linuxserver/unifi-controller", "lscr.io/linuxserver/unifi-controller@sha256:fcd5d8b13a77a588c79c1b49e5fc9ad08115aa3bb1a3576c589c64908a68845f"],
  ["minio/minio", "quay.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e"], // docker.io denies anonymous pulls; the catalogue pins quay.io (mesh-catalog #29)
  ["mongo", "mongo@sha256:e3fa459b4f4b72f3257c67a23c145e250b8b5700f033860392c68539b998bbe3"],
  ["ollama/ollama", "ollama/ollama@sha256:32931b46719f673c05fdbaa81ccb26da18ea4a1c57590a754874ab28ba269eb2"],
  ["postgres", "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee"],
  ["redis", "redis@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"],
  ["registry", "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"],
  ["synesthesiam/marytts", "synesthesiam/marytts@sha256:45970ecb3e21a2981c66c60563a70cf00be8e95c02565e7d74b3a73dcec7db2c"],
]);

/**
 * What a manifest's image reference becomes on the machine.
 *
 * Four cases, and the second one is the whole change:
 *
 *   - **Ours** becomes the ID the machine holds it under. Nothing serves it, and nothing needs to.
 *   - **Anything already pinned by digest** is returned exactly as written. The machine pulls it
 *     from the internet, over its uplink, which is what a real machine does and what the lab spent
 *     a long time serving from a registry of its own instead.
 *   - **A bare repository a bed names by hand** is given the digest mesh-catalog pins for it, so a
 *     bed runs the image the mesh ships. A tag would be refused by mesh-host anyway.
 *   - **A tag this harness has never heard of** is passed through untouched, and said out loud.
 *
 * That last case is not politeness, it is a finding the lab's registry was hiding. Seven catalogue
 * modules name `registry-api.…/novox/…:latest` — a TAG, which ADR 0006 forbids and mesh-host
 * refuses. It never showed, because the rewrite replaced every reference with a digest the lab's
 * registry had assigned, tag or not. There is nothing to replace it with now, and the honest
 * outcome is that those modules fail to apply, saying exactly why, on the node that carries them —
 * rather than an assertion here taking the whole bed down before it starts.
 */
export function onTheMachine(reference: string, held: HeldImage[]): string {
  if (mustBeHandedOver(reference)) {
    const found = referenceFor(held, repositoryOf(reference));
    assert.ok(
      found,
      `nothing loaded ${reference} onto the machines. They hold:\n  ` +
        held.map((i) => `${i.repository} ${i.reference}`).join("\n  "),
    );
    return found;
  }
  if (reference.includes("@sha256:")) return reference;

  const upstream = UPSTREAM.get(repositoryOf(reference));
  if (upstream) return upstream;

  console.log(
    `UNPINNED: ${reference} names a tag, not a digest. The host will refuse it (novox/hq ` +
      `ADR 0006). The lab's own registry used to paper over this by assigning a digest to ` +
      `whatever was pushed; nothing does now. Fix the manifest, or add its digest to the ` +
      `harness's UPSTREAM table.`,
  );
  return reference;
}

export interface Capability {
  usable: boolean;
  why: string;
}

/** Can this machine run scenarios at all? Checked once, reported honestly. */
export async function labIsUsable(): Promise<Capability> {
  if (!(await isReachable())) {
    return {
      usable: false,
      why: "the incus daemon is not reachable as this user (try MESH_LAB_INCUS='sudo -n incus')",
    };
  }
  const drivers = await supportedDrivers();
  if (!drivers.some((d) => d === "btrfs" || d === "zfs")) {
    return { usable: false, why: "no copy-on-write driver — snapshots would be full copies" };
  }
  if (!(await pools()).some((p) => p.driver === "btrfs" || p.driver === "zfs")) {
    return { usable: false, why: "no pool uses a copy-on-write driver" };
  }
  return { usable: true, why: "" };
}

/** Tear down anything a test left behind, whether it passed or not. */
export async function destroyAll(prefix: string): Promise<void> {
  for (const instance of await list()) {
    if (instance.instanceId.startsWith(prefix)) {
      await destroy(instance.instanceId);
    }
  }
}

/**
 * Every address the hypervisor says is held, by which machine, on which segment.
 *
 * Read through the live diagram because that is already the one place that joins addresses
 * to devices by MAC and devices to segments by tag. A second reader would be a second thing
 * to get wrong in the same way — and the way it was wrong once, a virtual machine's
 * addresses silently going missing, is exactly what these assertions would then miss.
 */
export async function heldAddresses(instanceId: string): Promise<Held[]> {
  const drawn = await diagramFromLive(instanceId);
  return drawn.machines.flatMap((machine) =>
    machine.attachments.flatMap((attachment) =>
      attachment.addresses.map((address) => ({
        machine: machine.name,
        segment: attachment.segment,
        address,
      })),
    ),
  );
}

function bare(address: string): string {
  const slash = address.lastIndexOf("/");
  return slash === -1 ? address : address.slice(0, slash);
}

/**
 * Invariants that hold of ANY raised scenario, whatever it declares.
 *
 * Asserted against what actually came up, never against the declaration — the declaration
 * is what was accepted, and in the fault that prompted these, it was accepted.
 */
export async function assertUniversalInvariants(
  scenario: Scenario,
  instanceId: string,
): Promise<void> {
  const held = await heldAddresses(instanceId);
  assert.ok(held.length > 0, `${scenario.scenario}: no addresses were read back at all`);

  const conflicts = duplicateAddresses(held);
  assert.deepEqual(
    conflicts,
    [],
    `${scenario.scenario}: address conflict — ${describeConflicts(conflicts)}`,
  );

  // Every address the scenario declared is one the machine actually holds. A machine that
  // came up bare looks identical to one that came up correctly until something asks it.
  const holders = new Map<string, Set<string>>();
  for (const entry of held) {
    const key = `${entry.machine} ${entry.segment}`;
    holders.set(key, (holders.get(key) ?? new Set<string>()).add(bare(entry.address)));
  }

  for (const [name, spec] of Object.entries(scenario.machines)) {
    if (spec.at === "detached") continue;
    for (const attachment of spec.at) {
      const actual = holders.get(`${name} ${attachment.segment}`) ?? new Set<string>();
      for (const address of attachment.address) {
        assert.ok(
          actual.has(address),
          `${scenario.scenario}: '${name}' declared ${address} on '${attachment.segment}' ` +
            `but holds ${[...actual].join(", ") || "nothing"}`,
        );
      }
    }
  }
}

// --- the packet filter, where a bed raises the foundation without genesis ------------------------

/** The packet filter genesis installs on the control-node, by the name the catalogue gives it. */
export const FILTER_MODULE = "nftables";

// --- the catalogue: a bed installs a module by reading its manifest, never by carrying a copy ----

/**
 * The catalogue's `modules/` directory: MESH_LAB_CATALOG under either spelling (the checkout, or
 * its modules directory). Named, or absent — never guessed from a sibling path: the receipt claims
 * the catalogue the run was pointed at (src/repos.ts), and a catalogue read from somewhere the
 * receipt does not name is the drift this exists to close.
 *
 * Beds used to build the manifests they install inline, as literals copied from the catalogue when
 * each bed was written. The copies did not move when the catalogue did, so a catalogue change was
 * proven nowhere — and a bed that installs a copy proves the copy (novox/hq 04-ISSUES/073). A bed
 * reads the catalogue, or it does not install a catalogue module; `beds-read-the-catalogue.test.ts`
 * refuses an inline copy that names one.
 */
export function catalogueDir(): string {
  const named = process.env["MESH_LAB_CATALOG"];
  if (!named) throw new Error("MESH_LAB_CATALOG is not set to a checkout of mesh-catalog (or its modules directory)");
  const candidates = [resolve(named, "modules"), resolve(named)];
  for (const dir of candidates) {
    // Known by the registry's manifest, which genesis reads from the catalogue and always will —
    // not the control plane's, which lives in the control plane's own repository (ADR 0069).
    if (existsSync(resolve(dir, "distribution", "module.json"))) return dir;
  }
  throw new Error(`no catalogue: MESH_LAB_CATALOG=${named} and no distribution/module.json under ${candidates.join(" or ")}`);
}

/** Whether a catalogue is where a bed will look — for a skip guard, which says so instead of failing. */
export function catalogueIsPresent(): string | false {
  try { catalogueDir(); return false; } catch (err) { return (err as Error).message; }
}

/** The catalogue's manifest for a module, as a path. */
export function catalogueManifest(module: string): string {
  const path = resolve(catalogueDir(), module, "module.json");
  if (!existsSync(path)) throw new Error(`no manifest for ${module} at ${path}`);
  return path;
}

/** What the lab may rewrite in a catalogue manifest, and nothing else. */
export interface ForTheLab {
  /**
   * The image repository each build artifact was built as on this workstation, by artifact name.
   * A module's own runtime is `mesh-runtime-<module>` by default — what `scripts/build-module-runtime.sh`
   * tags and what the scenarios stock; a bed names it only where the scenario stocks another name.
   * An artifact this does not name is refused: the bed must say what stands in for the builder.
   */
  artifacts?: Record<string, string>;
  /**
   * Host-port remaps, where one machine carries modules whose published ports collide —
   * `{ "8080": "8090:8080" }`, applied to every container of the module. The container side never
   * changes.
   */
  ports?: Record<string, string> | undefined;
  /**
   * Environment a container gets in the lab that it does not get in the mesh — an address the bed
   * stands up in place of a real upstream, and nothing else. Merged over the manifest's own.
   */
  env?: Record<string, Record<string, string>>;
}

/**
 * A catalogue manifest as a machine in the lab can run it: the mesh's build section gone (the lab
 * stocks images rather than building), each artifact replaced by the image the machine holds for it,
 * every image pinned to what the machine holds or the upstream digest the catalogue pins, and the
 * declared lab rewrites applied. Everything else is the catalogue's, verbatim — which is the point.
 */
export function catalogueModule(module: string, held: HeldImage[], lab: ForTheLab = {}): string {
  const m = JSON.parse(readFileSync(catalogueManifest(module), "utf8")) as {
    resources?: { id: string; type: string; image?: string; artifact?: string; ports?: string[]; env?: Record<string, string> }[];
    build?: unknown;
  };
  const artifacts: Record<string, string> = { runtime: `mesh-runtime-${module}`, ...(lab.artifacts ?? {}) };
  for (const r of m.resources ?? []) {
    if (r.type !== "container") continue;
    if (typeof r.artifact === "string") {
      const repository = artifacts[r.artifact];
      assert.ok(repository,
        `${module}'s container '${r.id}' names the "${r.artifact}" artifact, which the mesh would ` +
        `build. The lab does not build: the bed must say which stocked image stands in for it ` +
        `(artifacts: { ${r.artifact}: "<repository>" }).`);
      const reference = referenceFor(held, repository);
      assert.ok(reference,
        `${module}'s "${r.artifact}" artifact is ${repository} and this scenario stocked no such ` +
        `image. Add it to the scenario's images: and build it (scripts/build-module-runtime.sh ${module}).`);
      r.image = reference;
      delete r.artifact;
    } else if (typeof r.image === "string") {
      r.image = onTheMachine(r.image, held);
    }
    if (lab.ports && Array.isArray(r.ports)) r.ports = r.ports.map((p) => lab.ports![p] ?? p);
    const env = lab.env?.[r.id];
    if (env) r.env = { ...(r.env ?? {}), ...env };
  }
  delete m.build;
  return JSON.stringify(m);
}

/** Whether a manifest's runtime dials the broker — the module then needs a scoped broker account. */
export function needsBrokerAccount(manifest: string): boolean {
  return manifest.includes("MESH_BROKER_FILE");
}

function shellQuote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

/**
 * The foundation is raised behind a base ruleset that admits ssh, the bus and the registry and
 * nothing else, "until the mesh derives one" (novox/hq ADR 0088) — and the mesh derives one only
 * where the packet-filter module is assigned, which genesis does on the control-node. A bed that
 * raises the foundation from the bundle skips genesis, so it must do the same before it relies on
 * an overlay hub there: the hub's port is derived from its endpoint, and until the derived ruleset
 * lands no joined node's tunnel forms, and every module that dials the anchor by its overlay name
 * times out fetching the broker's certificate — the failure this helper was written after.
 *
 * Registered, assigned, pushed, and then WAITED FOR: what the machine loaded must admit the hub's
 * port, which the base ruleset cannot. Call it after the hub is placed, so there is a port to derive.
 */
export async function deriveTheFilterOn(o: {
  machine: string; node: string; hubPort: number;
  must: (machine: string, command: string, timeoutMs?: number) => Promise<string>;
  mesh: (command: string, timeoutMs?: number) => Promise<string>;
  on: (machine: string, command: string, timeoutMs?: number) => Promise<{ out: string; ok: boolean }>;
}): Promise<string> {
  const manifest = readFileSync(catalogueManifest(FILTER_MODULE), "utf8");
  await o.must(o.machine,
    `printf %s ${shellQuote(manifest)} > /tmp/${FILTER_MODULE}.json && docker cp /tmp/${FILTER_MODULE}.json mesh-controller:/${FILTER_MODULE}.json`);
  await o.mesh(`module add /${FILTER_MODULE}.json`);
  await o.mesh(`assign ${o.node} ${FILTER_MODULE}`);
  await o.mesh(`push ${o.node}`, 600_000);
  const admits = new RegExp(`udp dport ${o.hubPort} accept`);
  const deadline = Date.now() + 180_000;
  let ruleset = "";
  while (Date.now() < deadline) {
    ruleset = (await o.on(o.machine, `nft list table inet mesh 2>&1`)).out;
    if (admits.test(ruleset)) return ruleset;
    await new Promise((r) => setTimeout(r, 5_000));
  }
  assert.fail(`${FILTER_MODULE} is assigned to ${o.node} and the ruleset it loaded does not admit the hub's udp/${o.hubPort}:\n${ruleset}`);
}