mesh-lab/src/lifecycle/base.ts

/**
 * Building the base image a scenario's machines are raised from.
 *
 * A sealed scenario cannot install a container runtime: its segments use documentation ranges
 * and there is no route out (novox/hq ADR 0016). ADR 0006 records the consequence — *the lab
 * needs a way to place images, and the machine it places them into needs a container runtime,
 * which a sealed scenario cannot install either.*
 *
 * This is that, and it is research 012's reframing applied literally: **fetch at build time on
 * a machine that has a network, apply on a target that then needs nothing.** The build happens
 * here, once per lab, on a machine with a network. What a scenario raises afterwards needs
 * neither.
 *
 * Measured while writing it: installing the runtime takes about 30 seconds, publishing about a
 * minute, and the result is roughly 700 MiB.
 */

import { incus, incusOk, succeeds } from "../incus/client.ts";
import { BASE_IMAGE_ALIAS } from "./place.ts";

/** The stock image the base is built FROM. */
export const UPSTREAM_IMAGE = "images:archlinux/current";

const BUILDER = "mesh-lab-base-builder";

/**
 * **The mirrors the builder installs from, several and in order** (novox/hq issue 700).
 *
 * The stock image names one mirror, mirrors.kernel.org, and a base build failed twice on it in one morning:
 * timeouts while installing the runtime, then a 500 on wireguard-tools. pacman tries the next server in the
 * list for each file it could not fetch, so a list of several turns one mirror's bad minute into a slower
 * download instead of a failed build.
 *
 * A fixed list rather than ranking at build time: ranking (rankmirrors, reflector) needs a package installed
 * first, from the one mirror that is the problem, and gives a different list on every build. These are
 * Arch's own geo-routed mirror first, then well-kept European mirrors near the lab's machine, then the
 * stock image's own. Each answered for core.db on 2026-10-14. A mirror that goes away costs a fallback,
 * not a build, and is replaced here.
 */
export const MIRRORS = [
  "https://geo.mirror.pkgbuild.com/$repo/os/$arch",
  "https://mirror.netcologne.de/archlinux/$repo/os/$arch",
  "https://ftp.fau.de/archlinux/$repo/os/$arch",
  "https://archlinux.mirror.liteserver.nl/$repo/os/$arch",
  "https://mirror.leaseweb.net/archlinux/$repo/os/$arch",
  "https://mirrors.kernel.org/archlinux/$repo/os/$arch",
];

/** The mirrorlist the builder is given: every mirror, in order. */
export function mirrorlist(): string {
  return "# Written by the mesh-lab base build (novox/hq issue 700): several mirrors, tried in order.\n" +
    MIRRORS.map((m) => `Server = ${m}`).join("\n") + "\n";
}

/** The mirrors a failed pacman step names: "from <host>" in what it said, each once. */
export function failedMirrors(said: string): string[] {
  const hosts = [...said.matchAll(/\bfrom ([a-z0-9.-]+\.[a-z]{2,})\b/gi)].map((m) => m[1]!.toLowerCase());
  return [...new Set(hosts)];
}

/** How long to wait before each retry of a failed pacman step: three retries, backing off. */
export const PACMAN_BACKOFF_MS = [10_000, 30_000, 90_000];

/** What a base build runs through, so a test can stand in for the hypervisor and the clock. */
export interface BaseDeps {
  incus: (args: string[], timeoutMs?: number) => Promise<unknown>;
  incusOk: (args: string[], timeoutMs?: number) => Promise<string | null>;
  succeeds: (args: string[], timeoutMs?: number) => Promise<boolean>;
  sleep: (ms: number) => Promise<void>;
}

const realDeps: BaseDeps = {
  incus, incusOk, succeeds, sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
};

/**
 * pacmanIn runs one pacman step in the builder, and **runs it again after a pause when it fails**, up to
 * three more times, saying each time which mirror failed and what it said. Installing is idempotent — what is
 * installed stays installed — so a retry repeats only what did not happen. The last failure is thrown, naming
 * every mirror that failed along the way.
 */
export async function pacmanIn(name: string, args: string[], log: (m: string) => void, deps: BaseDeps = realDeps):
  Promise<void> {
  const failed: string[] = [];
  for (let attempt = 0; ; attempt++) {
    try {
      await deps.incus(["exec", name, "--", "pacman", ...args], 600_000);
      return;
    } catch (err) {
      // What pacman said, where incus kept it apart; otherwise the whole message.
      const said = (err as { stderr?: string }).stderr?.trim() || (err as Error).message;
      const mirrors = failedMirrors(said);
      for (const m of mirrors) if (!failed.includes(m)) failed.push(m);
      const where = mirrors.length > 0 ? `from ${mirrors.join(", ")}` : "naming no mirror";
      const first = said.split("\n").find((l) => /error/i.test(l))?.trim() ?? said.split("\n")[0]?.trim() ?? "";
      if (attempt >= PACMAN_BACKOFF_MS.length) {
        throw new BaseImageError(`pacman ${args.join(" ")} failed ${attempt + 1} times; the mirrors that failed: ` +
          `${failed.join(", ") || "none named"}. Last: ${said}`);
      }
      const wait = PACMAN_BACKOFF_MS[attempt]!;
      log(`  pacman ${args.join(" ")} failed ${where} (${first}); trying again in ${wait / 1000}s ` +
        `(attempt ${attempt + 2} of ${PACMAN_BACKOFF_MS.length + 1})`);
      await deps.sleep(wait);
    }
  }
}

export class BaseImageError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "BaseImageError";
  }
}

/**
 * Build the base image, replacing any previous one.
 *
 * Every step is read back. A published image that turns out not to have a working runtime is
 * worse than no image, because every scenario raised from it fails somewhere else.
 */
export async function buildBaseImage(
  log: (message: string) => void = () => {},
  deps: BaseDeps = realDeps,
): Promise<{ alias: string; runtime: string }> {
  const { incus, incusOk, succeeds } = deps;
  await succeeds(["delete", "-f", BUILDER], 120_000);

  log(`  launching ${BUILDER} from ${UPSTREAM_IMAGE}, with a network`);
  await incus([
    "launch", UPSTREAM_IMAGE, BUILDER, "--vm",
    "-c", "security.secureboot=false",
    "-c", "limits.memory=2GiB",
    "-c", "limits.cpu=2",
  ], 300_000);

  try {
    await waitForAgent(BUILDER, deps);

    // Several mirrors before anything is fetched (MIRRORS), and read back: a list that was not written is
    // the one mirror again, and the failure would look like the one this prevents.
    log(`  giving pacman ${MIRRORS.length} mirrors`);
    await incus(["exec", BUILDER, "--", "sh", "-c", `printf '%s' '${mirrorlist()}' > /etc/pacman.d/mirrorlist`], 60_000);
    const written = await incusOk(["exec", BUILDER, "--", "cat", "/etc/pacman.d/mirrorlist"], 60_000);
    if (written?.trim() !== mirrorlist().trim()) {
      throw new BaseImageError(`the mirrorlist written into ${BUILDER} reads back as ${JSON.stringify(written)}`);
    }

    log("  installing a container runtime");
    await pacmanIn(BUILDER, ["-Sy", "--noconfirm", "docker"], log, deps);

    // And the tools for the private network, for the same reason as the runtime: a sealed
    // scenario cannot install them, so a lab that omits them cannot test connectivity at all —
    // which is most of what the mesh does between machines.
    //
    // Installed here and NOT started. What a node runs is the mesh's decision, delivered as a
    // declaration; a lab that brought the interface up itself would be testing its own setup
    // rather than the mesh's.
    log("  installing the tools for the private network");
    await pacmanIn(BUILDER, ["-S", "--noconfirm", "wireguard-tools"], log, deps);

    // And git, for the same reason again: a machine that builds modules clones them, and a sealed
    // scenario cannot install it. On a real build machine the mesh installs it as a package like
    // anything else — the lab is the special case, because its machines reach no mirror.
    log("  installing git, so a machine can build modules");
    await pacmanIn(BUILDER, ["-S", "--noconfirm", "git"], log, deps);

    // And nftables, because the mesh computes a machine's filtering and delivers it as a file
    // that a service reflects — and neither the file nor the service can install what loads it.
    // Installed and NOT enabled: whether a machine filters is the mesh's decision, and a lab that
    // turned it on itself would be testing its own setup.
    log("  installing nftables, so a machine can enforce what the mesh computed");
    await pacmanIn(BUILDER, ["-S", "--noconfirm", "nftables"], log, deps);

    // And dnsmasq, because a service is named under the machine it runs on — postgres.novox.internal
    // — and only a resolver can answer a name the mesh was never told about. Installed and NOT
    // started: whether a machine resolves for the mesh is the mesh's decision, and a lab that
    // turned it on itself would be testing its own setup.
    log("  installing dnsmasq, so a machine can answer names under another machine");
    await pacmanIn(BUILDER, ["-S", "--noconfirm", "dnsmasq"], log, deps);

    // Trust the documentation ranges as plain-HTTP registries.
    //
    // **Kept after the lab's own registry was deleted, because it was never only for that.** The
    // mesh HAS a registry — the `registry` module, `mesh-registry`, serving artifacts to the whole
    // mesh on port 5000 over plain HTTP from whatever node runs it. In a scenario that node's
    // address is a documentation-range address, and a runtime will not pull from a plain-HTTP
    // registry without being told to. Take this away and the artifact store is unusable from every
    // machine but the one hosting it.
    //
    // Scoped to RFC 5737 and RFC 3849 ranges rather than a specific address, because those never
    // route on the real internet — so this cannot make a real machine trust a real registry,
    // whatever it is copied onto.
    await incus([
      "exec", BUILDER, "--", "sh", "-c",
      `mkdir -p /etc/docker && printf '%s' '${JSON.stringify({
        "insecure-registries": ["192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24"],
      })}' > /etc/docker/daemon.json`,
    ], 60_000);
    await incus(["exec", BUILDER, "--", "systemctl", "enable", "docker"], 60_000);
    await incus(["exec", BUILDER, "--", "systemctl", "start", "docker"], 120_000);

    // Read back from the tool, not from the package manager (novox/hq 04-ISSUES/007).
    const wg = await incusOk(["exec", BUILDER, "--", "wg", "--version"], 60_000);
    if (!wg?.trim()) {
      throw new BaseImageError(
        `wireguard-tools was installed in ${BUILDER} and \`wg\` does not answer. Publishing ` +
          `this would give every scenario a machine that cannot join a private network.`,
      );
    }
    log(`  ${wg.trim()}`);

    // The same, for git. An installed package is not a capability, and this is the one place to
    // catch it — a build machine whose clone fails does so three minutes into a scenario, with
    // the failure reported as a build problem rather than a lab one.
    const git = await incusOk(["exec", BUILDER, "--", "git", "--version"], 60_000);
    if (!git?.trim()) {
      throw new BaseImageError(
        `git was installed in ${BUILDER} and \`git --version\` does not answer. Publishing ` +
          `this would give every scenario a machine that cannot build a module.`,
      );
    }
    log(`  ${git.trim()}`);

    // The same again, for nftables. A machine that cannot load a rule set applies the mesh's
    // filtering, reports success, and filters nothing — which is precisely the fault the whole
    // derivation exists to remove, reappearing in the lab.
    const nft = await incusOk(["exec", BUILDER, "--", "nft", "--version"], 60_000);
    if (!nft?.trim()) {
      throw new BaseImageError(
        `nftables was installed in ${BUILDER} and \`nft\` does not answer. Publishing this would ` +
          `give every scenario a machine that cannot enforce what the mesh computed for it.`,
      );
    }
    log(`  ${nft.trim()}`);

    // The same again for dnsmasq. A machine that cannot answer names applies the mesh's resolver
    // data, reports success, and resolves nothing — the shape of fault this lab exists to catch.
    const dns = await incusOk(["exec", BUILDER, "--", "dnsmasq", "--version"], 60_000);
    if (!dns?.trim()) {
      throw new BaseImageError(
        `dnsmasq was installed in ${BUILDER} and does not answer. Publishing this would give ` +
          `every scenario a machine that cannot resolve a name under another machine.`,
      );
    }
    log(`  ${dns.trim().split("\n")[0]}`);

    // Read back from the runtime, not from the package manager. An installed package is not a
    // capability (novox/hq 04-ISSUES/007), and this is the one place to catch that — after
    // publishing, every scenario pays for it instead.
    const runtime = (await incusOk(
      ["exec", BUILDER, "--", "docker", "info", "--format", "{{.ServerVersion}}"],
      120_000,
    ))?.trim();
    if (!runtime) {
      throw new BaseImageError(
        `the runtime was installed in ${BUILDER} and does not answer. Publishing this would ` +
          `give every scenario an image that looks right and is not.`,
      );
    }
    log(`  runtime works (docker ${runtime})`);

    // Read back that the runtime will actually pull over plain HTTP from a documentation
    // range. Writing the file is not the same as the daemon honouring it, and a base image
    // that looks right here fails much later — as a container that cannot fetch its image
    // from the mesh's own artifact store, which is a long way from the cause.
    const trusted = await incusOk(
      ["exec", BUILDER, "--", "docker", "info", "--format", "{{.RegistryConfig.InsecureRegistryCIDRs}}"],
      60_000,
    );
    if (!trusted?.includes("192.0.2.0/24")) {
      throw new BaseImageError(
        `the runtime in ${BUILDER} does not trust the documentation ranges as plain-HTTP ` +
          `registries. It reported: ${trusted?.trim() || "nothing"}\n` +
          `  Every scenario raised from this image would fail to pull from the mesh's own ` +
          `artifact store, which serves plain HTTP inside the scenario.`,
      );
    }
    log("  trusts the documentation ranges as registries");

    log("  publishing");
    await incus(["stop", BUILDER, "--timeout", "120"], 300_000);
    await incus(["publish", BUILDER, "--alias", BASE_IMAGE_ALIAS, "--reuse"], 900_000);

    const listed = await incusOk(["image", "list", BASE_IMAGE_ALIAS, "--format", "csv", "-c", "l"], 60_000);
    if (!listed?.includes(BASE_IMAGE_ALIAS)) {
      throw new BaseImageError(
        `publishing reported success and '${BASE_IMAGE_ALIAS}' is not in the image list.`,
      );
    }

    log(`  published ${BASE_IMAGE_ALIAS}`);
    return { alias: BASE_IMAGE_ALIAS, runtime };
  } finally {
    // The builder is scaffolding. Leaving it standing would be a machine with a network in a
    // lab whose whole point is that scenarios do not have one.
    await succeeds(["delete", "-f", BUILDER], 120_000);
  }
}

/** Whether the base image exists, for a scenario to check before it raises. */
export async function baseImageExists(): Promise<boolean> {
  const listed = await incusOk(
    ["image", "list", BASE_IMAGE_ALIAS, "--format", "csv", "-c", "l"], 30_000,
  );
  return Boolean(listed?.includes(BASE_IMAGE_ALIAS));
}

/**
 * Wait for the guest agent, because `launch` returning means the VM started, not that anything
 * inside it will answer.
 */
async function waitForAgent(name: string, deps: BaseDeps = realDeps): Promise<void> {
  for (let i = 0; i < 90; i++) {
    if (await deps.succeeds(["exec", name, "--", "true"], 10_000)) return;
    await deps.sleep(2_000);
  }
  throw new BaseImageError(`${name} started and its agent never answered.`);
}