The lab can give a sealed machine a container runtime

ADR 0046's open consequence: "the lab needs a way to place images, and the
machine it places them into needs a container runtime, which a sealed scenario
cannot install either."

The runtime half is done, and it is research 012's reframing applied literally
-- fetch at build time on a machine with a network, apply on a target that
needs nothing. `mesh-lab base build` launches a machine WITH a network,
installs a runtime, verifies it by asking the runtime rather than the package
manager, and publishes the result. Measured: ~30s to install, ~60s to publish,
~700MiB, paid once per lab rather than per scenario.

A scenario that places `runtime` or an image is then raised from that base
image, chosen rather than declared -- a scenario says what it needs, not which
image provides it. If the base does not exist it says so and how to build it.

Verified in a genuinely sealed machine (no route out, confirmed by ping):
package, service including the new `boot: enabled`, and action all applied,
were idempotent on a second run, and read back correctly. Those three had never
run anywhere but a workstation.

The image half is NOT done, and testing found why: a digest-pinned image cannot
be placed from an archive. `docker save alpine@sha256:...` produces an archive
with no repo tag, because a repo digest only exists for an image a registry
served -- so it loads dangling and a container declaring that digest reaches
for a registry the machine cannot see.

That collides with ADR 0046, which has the host REFUSE an unpinned image. Tag
refused by the host, digest unusable in the lab: there is currently no
declaration the lab can raise that exercises the container shape at all. Filed
as 04-ISSUES/009, whose resolution is a registry inside the scenario -- which is
what the real mesh does rather than a workaround for the lab.

Also fixed a weak check of my own, which is the same fault in miniature: the
load was tested with `includes("Loaded image")`, a prefix of both `Loaded
image:` and `Loaded image ID:`. So an unusable dangling load reported success
and the failure surfaced later as a container that would not start.

src/cli.ts

@@ -91,6 +91,16 @@ async function main(): Promise<void> {
     case "check":
       return check();
 
+    case "base": {
+      // `base build` exists because a sealed scenario cannot install a container runtime, and
+      // the runtime has to come from somewhere with a network (novox/hq ADR 0046).
+      if (rest[0] !== "build") fail("base needs a subcommand: build");
+      const { buildBaseImage } = await import("./lifecycle/base.ts");
+      const built = await buildBaseImage((line) => console.log(line));
+      console.log(`${built.alias}: built, with docker ${built.runtime}`);
+      return;
+    }
+
     case "validate": {
       const path = rest[0] ?? fail("validate needs a scenario file");
       const scenario = loadScenario(path);

src/lifecycle/base.ts

@@ -0,0 +1,113 @@
+/**
+ * Building the base image a scenario's machines are raised from.
+ *
+ * A sealed scenario cannot install a container runtime: its segments use documentation ranges
+ * and there is no route out (novox/hq ADR 0032). ADR 0046 records the consequence — *the lab
+ * needs a way to place images, and the machine it places them into needs a container runtime,
+ * which a sealed scenario cannot install either.*
+ *
+ * This is that, and it is research 012's reframing applied literally: **fetch at build time on
+ * a machine that has a network, apply on a target that then needs nothing.** The build happens
+ * here, once per lab, on a machine with a network. What a scenario raises afterwards needs
+ * neither.
+ *
+ * Measured while writing it: installing the runtime takes about 30 seconds, publishing about a
+ * minute, and the result is roughly 700 MiB.
+ */
+
+import { incus, incusOk, succeeds } from "../incus/client.ts";
+import { BASE_IMAGE_ALIAS } from "./place.ts";
+
+/** The stock image the base is built FROM. */
+export const UPSTREAM_IMAGE = "images:archlinux/current";
+
+const BUILDER = "mesh-lab-base-builder";
+
+export class BaseImageError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "BaseImageError";
+  }
+}
+
+/**
+ * Build the base image, replacing any previous one.
+ *
+ * Every step is read back. A published image that turns out not to have a working runtime is
+ * worse than no image, because every scenario raised from it fails somewhere else.
+ */
+export async function buildBaseImage(
+  log: (message: string) => void = () => {},
+): Promise<{ alias: string; runtime: string }> {
+  await succeeds(["delete", "-f", BUILDER], 120_000);
+
+  log(`  launching ${BUILDER} from ${UPSTREAM_IMAGE}, with a network`);
+  await incus([
+    "launch", UPSTREAM_IMAGE, BUILDER, "--vm",
+    "-c", "security.secureboot=false",
+    "-c", "limits.memory=2GiB",
+    "-c", "limits.cpu=2",
+  ], 300_000);
+
+  try {
+    await waitForAgent(BUILDER);
+
+    log("  installing a container runtime");
+    await incus(["exec", BUILDER, "--", "pacman", "-Sy", "--noconfirm", "docker"], 600_000);
+    await incus(["exec", BUILDER, "--", "systemctl", "enable", "docker"], 60_000);
+    await incus(["exec", BUILDER, "--", "systemctl", "start", "docker"], 120_000);
+
+    // Read back from the runtime, not from the package manager. An installed package is not a
+    // capability (novox/hq 04-ISSUES/007), and this is the one place to catch that — after
+    // publishing, every scenario pays for it instead.
+    const runtime = (await incusOk(
+      ["exec", BUILDER, "--", "docker", "info", "--format", "{{.ServerVersion}}"],
+      120_000,
+    ))?.trim();
+    if (!runtime) {
+      throw new BaseImageError(
+        `the runtime was installed in ${BUILDER} and does not answer. Publishing this would ` +
+          `give every scenario an image that looks right and is not.`,
+      );
+    }
+    log(`  runtime works (docker ${runtime})`);
+
+    log("  publishing");
+    await incus(["stop", BUILDER, "--timeout", "120"], 300_000);
+    await incus(["publish", BUILDER, "--alias", BASE_IMAGE_ALIAS, "--reuse"], 900_000);
+
+    const listed = await incusOk(["image", "list", BASE_IMAGE_ALIAS, "--format", "csv", "-c", "l"], 60_000);
+    if (!listed?.includes(BASE_IMAGE_ALIAS)) {
+      throw new BaseImageError(
+        `publishing reported success and '${BASE_IMAGE_ALIAS}' is not in the image list.`,
+      );
+    }
+
+    log(`  published ${BASE_IMAGE_ALIAS}`);
+    return { alias: BASE_IMAGE_ALIAS, runtime };
+  } finally {
+    // The builder is scaffolding. Leaving it standing would be a machine with a network in a
+    // lab whose whole point is that scenarios do not have one.
+    await succeeds(["delete", "-f", BUILDER], 120_000);
+  }
+}
+
+/** Whether the base image exists, for a scenario to check before it raises. */
+export async function baseImageExists(): Promise<boolean> {
+  const listed = await incusOk(
+    ["image", "list", BASE_IMAGE_ALIAS, "--format", "csv", "-c", "l"], 30_000,
+  );
+  return Boolean(listed?.includes(BASE_IMAGE_ALIAS));
+}
+
+/**
+ * Wait for the guest agent, because `launch` returning means the VM started, not that anything
+ * inside it will answer.
+ */
+async function waitForAgent(name: string): Promise<void> {
+  for (let i = 0; i < 90; i++) {
+    if (await succeeds(["exec", name, "--", "true"], 10_000)) return;
+    await new Promise((r) => setTimeout(r, 2_000));
+  }
+  throw new BaseImageError(`${name} started and its agent never answered.`);
+}

src/lifecycle/place.ts

@@ -12,12 +12,31 @@
  */
 
 import type { Scenario } from "../declaration/types.ts";
+import { spawn } from "node:child_process";
+import { unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
 import { incus, incusOk } from "../incus/client.ts";
 
-/** What this stage can put inside a machine. */
-export const PLACEABLE = ["host"] as const;
+/**
+ * What this stage can put inside a machine.
+ *
+ * `image:<reference>` is the one that takes an argument — `image:alpine@sha256:...` — because
+ * WHICH image is the point of placing one.
+ */
+export const PLACEABLE = ["host", "runtime"] as const;
 export type Placeable = (typeof PLACEABLE)[number];
 
+/** The prefix that marks a container image, and what follows it is passed through unchanged. */
+export const IMAGE_PREFIX = "image:";
+
+/** Whether an artifact names something this stage can place. */
+export function isPlaceable(artifact: string): boolean {
+  return (PLACEABLE as readonly string[]).includes(artifact) ||
+    (artifact.startsWith(IMAGE_PREFIX) && artifact.slice(IMAGE_PREFIX.length).trim() !== "");
+}
+
 /** Where the host binary lives on a machine once placed. */
 export const HOST_PATH = "/usr/local/bin/mesh-host";
 
@@ -134,8 +153,12 @@ export async function applyPlacements(
   const placements = planPlacements(scenario);
   if (placements.length === 0) return [];
 
+  // Only demanded when something actually needs it. A scenario placing a runtime and an image
+  // and no host is a legitimate thing to want, and asking it for a host binary would be
+  // refusing a scenario for not doing something it never said it would.
+  const wantsHost = placements.some((p) => p.artifacts.includes("host"));
   const binary = hostBinaryPath();
-  if (!binary) {
+  if (wantsHost && !binary) {
     throw new Error(
       `this scenario places the host, and no host binary was given. Set ` +
         `MESH_LAB_HOST_BINARY to a built mesh-host. Guessing at a path would place ` +
@@ -147,10 +170,172 @@ export async function applyPlacements(
   for (const { machine, artifacts } of placements) {
     const name = machineNames.get(machine);
     if (!name) continue;
+
+    // Order within a machine is the order declared, because a scenario placing an image
+    // before a runtime means something different from the reverse and the lab should not
+    // silently reorder it into working.
     for (const artifact of artifacts) {
-      if (artifact !== "host") continue; // refused earlier; belt and braces
-      placed.push(await placeHost(name, machine, binary, log));
+      if (artifact === "host") {
+        placed.push(await placeHost(name, machine, binary as string, log));
+      } else if (artifact === "runtime") {
+        await placeRuntime(name, machine, log);
+      } else if (artifact.startsWith(IMAGE_PREFIX)) {
+        await placeImage(name, machine, artifact.slice(IMAGE_PREFIX.length).trim(), log);
+      }
+      // Anything else was refused by the validator; reaching here would be a validator bug.
     }
   }
   return placed;
 }
+
+/**
+ * The base image a scenario's machines are built from, when they need a container runtime.
+ *
+ * A sealed scenario cannot install one: its segments use documentation ranges and there is no
+ * route out (novox/hq ADR 0032). So the runtime arrives the way research 012 says everything
+ * awkward should — **fetched at build time on a machine that has a network, applied on a target
+ * that then needs nothing.** Building this image is that build time.
+ *
+ * Measured before this was written: an Arch VM installs docker in about 30 seconds, and
+ * publishing the result takes about a minute and yields roughly 700 MiB. That is a one-time
+ * cost paid once per lab, not per scenario.
+ */
+export const BASE_IMAGE_ALIAS = "mesh-lab/base";
+
+/** How to build it, said in one place so an error can point at it. */
+export const BASE_IMAGE_HOWTO =
+  `Build it with: mesh-lab base build\n` +
+  `  It launches a machine WITH a network, installs a container runtime, and publishes the\n` +
+  `  result as '${BASE_IMAGE_ALIAS}'. A scenario is sealed and cannot do that for itself.`;
+
+/**
+ * Confirm a machine has a working container runtime.
+ *
+ * Asks the runtime, never the filesystem. A binary being present is not a runtime working —
+ * that is 04-ISSUES/007, and it is the reason this runs `info` rather than checking a path.
+ *
+ * It does not INSTALL one. A sealed machine cannot fetch, so an absent runtime is a scenario
+ * built on the wrong image, and saying that is more useful than failing inside a package
+ * manager with no network.
+ */
+export async function placeRuntime(
+  instanceName: string,
+  machine: string,
+  log: (message: string) => void = () => {},
+): Promise<string> {
+  const version = (await incusOk(
+    ["exec", instanceName, "--", "docker", "info", "--format", "{{.ServerVersion}}"],
+    60_000,
+  ))?.trim();
+
+  if (!version) {
+    throw new PlacementError(
+      machine,
+      `${machine} has no working container runtime, and a sealed scenario cannot install one.\n` +
+        `  This machine needs to be built from '${BASE_IMAGE_ALIAS}'.\n` +
+        BASE_IMAGE_HOWTO,
+    );
+  }
+  log(`  ${machine} has a container runtime (docker ${version})`);
+  return version;
+}
+
+/**
+ * Put a container image inside a machine.
+ *
+ * Exported from this workstation and loaded in the machine, because the machine cannot reach a
+ * registry. The reference is passed through unchanged — including its digest — so what runs in
+ * the lab is the image the declaration names rather than whatever a tag pointed at.
+ */
+export async function placeImage(
+  instanceName: string,
+  machine: string,
+  reference: string,
+  log: (message: string) => void = () => {},
+): Promise<void> {
+  // A digest reference cannot be placed this way, and finding that out here is much better
+  // than finding it out when a container fails to start.
+  //
+  // `docker save alpine@sha256:...` produces an archive with NO repo tag — only an image ID —
+  // because a repo digest exists only for an image a registry served. Loading it gives a
+  // dangling image, so `docker run <that digest>` falls through to the registry, which a
+  // sealed machine cannot reach. Measured, not assumed: the load says `Loaded image ID:`
+  // instead of `Loaded image:`, and `docker images` then lists nothing.
+  //
+  // This collides with novox/hq ADR 0046, which pins bundle images BY DIGEST and has the host
+  // refuse anything else. Reconciling the two needs a registry inside the scenario, which is
+  // real design work — see 04-ISSUES/009.
+  if (reference.includes("@sha256:")) {
+    throw new PlacementError(
+      machine,
+      `${reference} is pinned by digest, and an image placed from an archive cannot keep its ` +
+        `digest — a repo digest only exists for an image a registry served.\n` +
+        `  Placing it would load an image with no name, and a container declaring that digest ` +
+        `would try to reach a registry the machine cannot see.\n` +
+        `  Place it by tag, or give the scenario a registry (novox/hq 04-ISSUES/009).`,
+    );
+  }
+
+  const tar = join(tmpdir(), `mesh-lab-image-${process.pid}-${Date.now()}.tar`);
+
+  // Exported from whatever this workstation has. The lab places images; it does not fetch
+  // them, so an image nobody pulled here is an error rather than a download.
+  const saved = await local("docker", ["save", reference, "-o", tar], 600_000);
+  if (!saved.ok) {
+    throw new PlacementError(
+      machine,
+      `cannot export ${reference} from this workstation: ${saved.stderr.trim()}\n` +
+        `  The lab places images it already has, and never fetches on a scenario's behalf.\n` +
+        `  Pull it here first, then raise.`,
+    );
+  }
+
+  try {
+    await incus(["file", "push", tar, `${instanceName}/tmp/image.tar`], 900_000);
+
+    // Read back what the runtime says, not that the push returned. A file arriving is not an
+    // image present, and `docker load` names what it loaded — so that is what is checked.
+    const loaded = await incusOk(
+      ["exec", instanceName, "--", "docker", "load", "-i", "/tmp/image.tar"],
+      600_000,
+    );
+    // `Loaded image:` and `Loaded image ID:` are different outcomes and only one is useful.
+    // Matching the shorter string accepted both, so an archive that loaded as a dangling
+    // image reported success — a check that passes on the wrong thing.
+    if (!loaded?.includes("Loaded image:")) {
+      throw new PlacementError(
+        machine,
+        `${reference} was pushed to ${machine} and did not arrive as a usable image.\n` +
+          `  The runtime said: ${loaded?.trim() || "nothing"}\n` +
+          `  'Loaded image ID:' means it loaded with no name, which nothing can run by name.`,
+      );
+    }
+    log(`  placed ${reference} on ${machine}`);
+  } finally {
+    await unlink(tar).catch(() => {});
+  }
+}
+
+/** Run something on THIS workstation. The incus client only speaks to incus. */
+function local(
+  command: string,
+  args: string[],
+  timeoutMs: number,
+): Promise<{ ok: boolean; stdout: string; stderr: string }> {
+  return new Promise((resolve) => {
+    const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
+    child.stdout.on("data", (d) => (stdout += d));
+    child.stderr.on("data", (d) => (stderr += d));
+    child.on("error", (err) => {
+      clearTimeout(timer);
+      resolve({ ok: false, stdout, stderr: err.message });
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      resolve({ ok: code === 0, stdout, stderr });
+    });
+  });
+}

src/lifecycle/raise.ts

@@ -22,7 +22,8 @@ import { applyAddresses, applyDefaultRoutes } from "./address.ts";
 import { assertSupported } from "./supported.ts";
 import { planRouters, raiseRouters, raiseTransit } from "./router.ts";
 import { applyHostFirewalls } from "./firewall.ts";
-import { applyPlacements } from "./place.ts";
+import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts";
+import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts";
 
 /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */
 const COW_DRIVERS = ["btrfs", "zfs"];
@@ -166,7 +167,20 @@ export async function raise(
   options: RaiseOptions = {},
 ): Promise<RaisedScenario> {
   const log = options.onProgress ?? (() => {});
-  const image = options.image ?? "images:archlinux/current";
+
+  // A scenario that places a runtime or an image needs machines built from the base image,
+  // because a sealed machine cannot install one (novox/hq ADR 0046). Chosen here rather than
+  // declared, so a scenario says WHAT it needs and not which image provides it.
+  const needsRuntime = planPlacements(scenario).some(({ artifacts }) =>
+    artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX))
+  );
+  if (needsRuntime && !options.image && !(await baseImageExists())) {
+    throw new Error(
+      `this scenario needs a container runtime inside its machines, and '${BASE_IMAGE_ALIAS}' ` +
+        `does not exist.\n${BASE_IMAGE_HOWTO}`,
+    );
+  }
+  const image = options.image ?? (needsRuntime ? BASE_IMAGE_ALIAS : UPSTREAM_IMAGE);
   const readyTimeout = options.readyTimeoutSeconds ?? 180;
   const instanceId = options.instanceId ?? newInstanceId(scenario.scenario, new Date());
 

src/lifecycle/supported.ts

@@ -14,7 +14,7 @@
  */
 
 import type { Scenario } from "../declaration/types.ts";
-import { PLACEABLE, planPlacements } from "./place.ts";
+import { IMAGE_PREFIX, isPlaceable, PLACEABLE, planPlacements } from "./place.ts";
 
 export class UnsupportedError extends Error {
   readonly missing: string[];
@@ -34,19 +34,19 @@ export class UnsupportedError extends Error {
 export function assertSupported(scenario: Scenario): void {
   const missing: string[] = [];
 
-  // `place: [host]` works. Everything else in the vocabulary is named individually rather
-  // than refused as a whole, so a scenario that places a host and a substrate is told exactly
-  // which half the lab cannot do.
+  // `host`, `runtime` and `image:<reference>` work. Everything else in the vocabulary is named
+  // individually rather than refused as a whole, so a scenario that places a host and a
+  // substrate is told exactly which half the lab cannot do.
   const unplaceable = new Set<string>();
   for (const { artifacts } of planPlacements(scenario)) {
     for (const artifact of artifacts) {
-      if (!(PLACEABLE as readonly string[]).includes(artifact)) unplaceable.add(artifact);
+      if (!isPlaceable(artifact)) unplaceable.add(artifact);
     }
   }
   for (const artifact of [...unplaceable].sort()) {
     missing.push(
-      `place: ${artifact} — only ${PLACEABLE.join(", ")} can be placed; the tiers above ` +
-        `tier 0 do not exist yet`,
+      `place: ${artifact} — only ${PLACEABLE.join(", ")} and ${IMAGE_PREFIX}<reference> can ` +
+        `be placed; the tiers above tier 0 do not exist yet`,
     );
   }