Automate the lab registry: a sealed machine pulls by digest

Closes 04-ISSUES/009. A scenario declares `images:` by tag; the lab stocks a
registry on this workstation where there is a network, raises it inside the
scenario as scenery, and reports the references a declaration pins -- which are
the digests THIS registry assigned, and are not knowable until it is raised.

Verified in a sealed machine, confirmed by ping to have no route out: package,
service including boot state, a container pinned by digest, and an action
inside that container. Applied, idempotent on re-apply, and read back from the
machine rather than from the apply's own report. That is the first time the
container shape has worked in the lab at all, and it was the shape blocking the
substrate bootstrap.

Four faults found by running it, three of them mine and one worth keeping:

The read-back checked that the catalog endpoint answered, by looking for the
substring "repositories" -- which `{"repositories":[]}` also contains. So it
passed on a registry holding nothing, and the failure surfaced much later as a
container that could not be pulled. It now asks for each image's manifest BY
DIGEST, which is what a machine does.

A recursive push needs its destination to exist, or incus copies the source's
contents rather than the source. The data landed one directory too shallow and
the registry found nothing where it looks.

The registry writes its blobs as root through a bind mount, so the workstation
could not remove its own scratch directory afterwards. Whoever made the files
removes them -- the cleanup now runs in a container too. And a cleanup failure
no longer fails a raise that succeeded: the scenario is standing and usable,
and saying otherwise would be a false report.

The base image build did not verify that the runtime trusts the documentation
ranges as plain-HTTP registries. Writing the file is not the daemon honouring
it, and a base image that looks right fails much later, in a sealed scenario,
a long way from its cause. It is now read back from `docker info`.

src/cli.ts

@@ -130,6 +130,12 @@ async function main(): Promise<void> {
       });
       const seconds = ((Date.now() - started) / 1000).toFixed(1);
       console.log(`\nraised ${raised.instanceId} in ${seconds}s — ${raised.machines.length} machines usable`);
+      if (raised.images.length > 0) {
+        // Printed because this is what a declaration pins, and it is not knowable until the
+        // scenario has been raised — the digest belongs to this registry.
+        console.log(`\nimages served, pinned by digest:`);
+        for (const image of raised.images) console.log(`  ${image}`);
+      }
       if (scenario.snapshot) {
         const took = await snapshot(raised.instanceId, scenario.snapshot);
         console.log(`snapshot '${scenario.snapshot}' in ${took.toFixed(2)}s`);

src/declaration/parse.ts

@@ -96,6 +96,11 @@ export function parseScenario(text: string): Scenario {
     });
   }
 
+  const images = raw["images"];
+  if (Array.isArray(images)) {
+    scenario.images = images.map((i) => String(i));
+  }
+
   const place = raw["place"];
   if (place && typeof place === "object") {
     const normalised: Record<string, string[]> = {};

src/declaration/types.ts

@@ -109,6 +109,14 @@ export interface Scenario {
   machines: Record<string, Machine>;
   policy?: Policy[];
   place?: Placement;
+  /**
+   * Container images this scenario needs inside it.
+   *
+   * A sealed machine cannot reach a registry, so the lab raises one on a public segment and
+   * serves these from it. Written as tags — the digest a declaration pins is the one THIS
+   * registry assigns, and it is reported when the scenario is raised.
+   */
+  images?: string[];
   /** Name the state once placement finishes, so a run can return to it. */
   snapshot?: string;
 }

src/declaration/validate.ts

@@ -299,5 +299,29 @@ export function validate(scenario: Scenario): void {
     }
   }
 
+  for (const image of scenario.images ?? []) {
+    if (!image.trim()) {
+      problems.push("images: an empty entry names nothing");
+    } else if (image.includes("@sha256:")) {
+      // The digest a declaration pins is the one the LAB's registry assigns, which is not
+      // knowable before the scenario is raised. Naming an upstream digest here would pin
+      // something this registry will never serve.
+      problems.push(
+        `images: '${image}' is pinned by digest. Name it by tag — the lab's registry assigns ` +
+        `its own digest and reports it when the scenario is raised`,
+      );
+    }
+  }
+  if ((scenario.images ?? []).length > 0) {
+    const hasPublicV4 = Object.values(scenario.segments)
+      .some((s) => s.kind === "public" && s.cidr.some((c) => !c.includes(":")));
+    if (!hasPublicV4) {
+      problems.push(
+        "images: this scenario declares images and has no public IPv4 segment to serve them " +
+        "from. The registry stands in for the outside world, so it sits on a public segment",
+      );
+    }
+  }
+
   if (problems.length > 0) throw new DeclarationError(problems);
 }

src/lifecycle/base.ts

@@ -85,6 +85,23 @@ export async function buildBaseImage(
     }
     log(`  runtime works (docker ${runtime})`);
 
+    // Read back that the runtime will actually pull over plain HTTP from a documentation
+    // range. Writing the file is not the same as the daemon honouring it, and a base image
+    // that looks right here fails much later — in a sealed scenario, as a container that
+    // cannot fetch its image, which is a long way from the cause.
+    const trusted = await incusOk(
+      ["exec", BUILDER, "--", "docker", "info", "--format", "{{.RegistryConfig.InsecureRegistryCIDRs}}"],
+      60_000,
+    );
+    if (!trusted?.includes("192.0.2.0/24")) {
+      throw new BaseImageError(
+        `the runtime in ${BUILDER} does not trust the documentation ranges as plain-HTTP ` +
+          `registries. It reported: ${trusted?.trim() || "nothing"}\n` +
+          `  Every scenario raised from this image would fail to pull from its own registry.`,
+      );
+    }
+    log("  trusts the documentation ranges as registries");
+
     log("  publishing");
     await incus(["stop", BUILDER, "--timeout", "120"], 300_000);
     await incus(["publish", BUILDER, "--alias", BASE_IMAGE_ALIAS, "--reuse"], 900_000);

src/lifecycle/raise.ts

@@ -24,6 +24,7 @@ import { planRouters, raiseRouters, raiseTransit } from "./router.ts";
 import { applyHostFirewalls } from "./firewall.ts";
 import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts";
 import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts";
+import { discardStock, raiseRegistry, stockRegistry } from "./registry.ts";
 
 /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */
 const COW_DRIVERS = ["btrfs", "zfs"];
@@ -44,6 +45,13 @@ export interface RaisedScenario {
   machines: string[];
   networks: string[];
   pool: string;
+  /**
+   * Images the scenario's registry serves, as references a declaration can pin.
+   *
+   * Reported rather than declared, because the digest is the one this registry assigned and
+   * is not knowable before it was raised.
+   */
+  images: string[];
 }
 
 export class RaiseError extends Error {
@@ -171,9 +179,10 @@ export async function raise(
   // A scenario that places a runtime or an image needs machines built from the base image,
   // because a sealed machine cannot install one (novox/hq ADR 0006). Chosen here rather than
   // declared, so a scenario says WHAT it needs and not which image provides it.
-  const needsRuntime = planPlacements(scenario).some(({ artifacts }) =>
-    artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX))
-  );
+  const needsRuntime = (scenario.images ?? []).length > 0 ||
+    planPlacements(scenario).some(({ artifacts }) =>
+      artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX))
+    );
   if (needsRuntime && !options.image && !(await baseImageExists())) {
     throw new Error(
       `this scenario needs a container runtime inside its machines, and '${BASE_IMAGE_ALIAS}' ` +
@@ -231,6 +240,24 @@ export async function raise(
     const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log);
     if (transit) routers.push(transit);
 
+    // Stocked on this workstation, where there is a network, and served from inside the
+    // scenario, where there is not (novox/hq 04-ISSUES/009).
+    step = "stocking the registry";
+    const stock = await stockRegistry(scenario.images ?? [], log);
+    let registry: Awaited<ReturnType<typeof raiseRegistry>> = null;
+    try {
+      step = "raising the registry";
+      registry = await raiseRegistry(scenario, instanceId, stock, log);
+    } finally {
+      // Cleaning up scratch must not fail a raise that succeeded. The scenario is standing
+      // and usable; a directory left behind is untidy, and saying so is the honest report.
+      try {
+        await discardStock(stock);
+      } catch (err) {
+        log(`  (could not remove the registry's scratch directory: ${(err as Error).message})`);
+      }
+    }
+
     step = "routing machines through their gateways";
     await applyDefaultRoutes(scenario, byMachine, log);
 
@@ -247,7 +274,8 @@ export async function raise(
     return {
       instanceId,
       scenario: scenario.scenario,
-      machines: [...created, ...routers],
+      images: registry?.pinned ?? [],
+      machines: [...created, ...routers, ...(registry ? [registry.machine] : [])],
       networks,
       pool,
     };

src/lifecycle/registry.ts

@@ -19,6 +19,10 @@
  */
 
 import { spawn } from "node:child_process";
+
+import { incus, incusOk, succeeds } from "../incus/client.ts";
+import { macFor, networkName } from "./names.ts";
+import { BASE_IMAGE_ALIAS, placeImage } from "./place.ts";
 import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -116,13 +120,29 @@ export async function stockRegistry(
 
     return { dataDir, images };
   } catch (err) {
-    await rm(dataDir, { recursive: true, force: true });
+    await discardStock({ dataDir, images: [] });
     throw err;
   } finally {
     await docker(["rm", "-f", container], 60_000);
   }
 }
 
+/**
+ * Remove a stocked registry's data.
+ *
+ * Through a container, because a container wrote it. The registry runs as root inside, so the
+ * blobs it writes into a bind mount are owned by root and an ordinary process cannot remove
+ * them — `rmdir` fails with EACCES on a directory that looks like ours.
+ *
+ * Whoever made the files removes them.
+ */
+export async function discardStock(stock: Stock): Promise<void> {
+  if (!stock.dataDir) return;
+  await docker(["run", "--rm", "-v", `${stock.dataDir}:/stock`, REGISTRY_IMAGE,
+    "sh", "-c", "rm -rf /stock/* /stock/.[!.]* 2>/dev/null || true"], 120_000);
+  await rm(stock.dataDir, { recursive: true, force: true }).catch(() => {});
+}
+
 /** `alpine:3.20` and `alpine` both serve from `alpine`; `foo/bar:1` from `foo/bar`. */
 export function repositoryFor(reference: string): string {
   const withoutDigest = reference.split("@")[0] ?? reference;
@@ -197,3 +217,159 @@ export function registryAddress(cidr: string): string {
 export function pinnedReference(address: string, image: StockedImage): string {
   return `${address}:${REGISTRY_PORT}/${image.repository}@${image.digest}`;
 }
+
+// --- raising it inside a scenario ---------------------------------------------------------------
+
+/** What a raised registry is, and what a declaration needs from it. */
+export interface RaisedRegistry {
+  machine: string;
+  segment: string;
+  address: string;
+  /** Each image, as a reference a declaration can pin. */
+  pinned: string[];
+}
+
+/**
+ * Pick the segment the registry sits on.
+ *
+ * A public segment, because that is what stands in for the outside world — a first node fetches
+ * from upstream, and this is upstream. An IPv4 range, because a machine has to be pointed at it
+ * by address.
+ */
+export function registrySegment(
+  segments: Record<string, { kind: string; cidr: string[] }>,
+): { name: string; cidr: string } | null {
+  for (const [name, segment] of Object.entries(segments)) {
+    if (segment.kind !== "public") continue;
+    const v4 = segment.cidr.find((c) => !c.includes(":"));
+    if (v4) return { name, cidr: v4 };
+  }
+  return null;
+}
+
+/**
+ * Raise a registry inside the scenario and load the stocked images into it.
+ *
+ * Scenery, in the same sense the transit router is: nothing under test runs on it, it holds no
+ * identity, and no assertion is made about its internals. It exists so that a machine can fetch
+ * an image the way a real one does — over the network, from a registry, by digest.
+ */
+export async function raiseRegistry(
+  scenario: { segments: Record<string, { kind: string; cidr: string[] }> },
+  instanceId: string,
+  stock: Stock,
+  log: (message: string) => void = () => {},
+): Promise<RaisedRegistry | null> {
+  if (stock.images.length === 0) return null;
+
+  const segment = registrySegment(scenario.segments);
+  if (!segment) {
+    throw new RegistryError(
+      `this scenario declares images and has no public IPv4 segment to serve them from.\n` +
+        `  The registry stands in for the outside world, so it sits on a public segment.`,
+    );
+  }
+
+  const address = registryAddress(segment.cidr);
+  const name = `mlab-${instanceId}-registry`;
+  const prefix = segment.cidr.slice(segment.cidr.lastIndexOf("/"));
+
+  if (!(await succeeds(["config", "show", name], 15_000))) {
+    await incus([
+      "init", BASE_IMAGE_ALIAS, name, "--vm",
+      "-c", "security.secureboot=false",
+      "-c", "limits.memory=1GiB",
+      "-c", `user.mesh-lab.instance=${instanceId}`,
+      // Tagged as a machine as well, so `destroy` finds it with one query — a router that
+      // carried only its own tag was left behind and held its networks open.
+      "-c", "user.mesh-lab.machine=registry",
+      "-c", "user.mesh-lab.registry=true",
+    ], 300_000);
+    await succeeds(["config", "device", "remove", name, "eth0"], 15_000);
+    await incus([
+      "config", "device", "add", name, "eth0", "nic",
+      "nictype=bridged",
+      `parent=${networkName(instanceId, segment.name)}`,
+      `hwaddr=${macFor(instanceId, "registry", 0)}`,
+    ]);
+  }
+  await succeeds(["start", name], 60_000);
+  await waitForAgent(name);
+
+  // Address it by MAC, never by interface name: a machine with a container runtime has a
+  // `docker0` that sorts before `enp5s0`, and naive selection configures that instead — which
+  // then overlaps the segment and breaks routing on the machine.
+  const mac = macFor(instanceId, "registry", 0);
+  await incus(["exec", name, "--", "sh", "-c",
+    `dev=$(ip -o link | awk -F': ' '/${mac}/ {print $2}' | head -1); ` +
+    `[ -n "$dev" ] && ip addr add ${address}${prefix} dev "$dev" 2>/dev/null; ` +
+    `[ -n "$dev" ] && ip link set "$dev" up`], 60_000);
+
+  log(`  registry on ${segment.name} at ${address}`);
+
+  // The registry's own image, placed by tag — an archive keeps a tag and cannot keep a digest,
+  // which is the whole reason this machine exists.
+  await placeImage(name, "registry", REGISTRY_IMAGE, () => {});
+
+  // The destination must EXIST before a recursive push, or incus copies the source's contents
+  // rather than the source — the data lands one directory too shallow, the registry finds
+  // nothing where it looks, and every pull fails with `not found`.
+  await incus(["exec", name, "--", "mkdir", "-p", "/srv/registry"], 60_000);
+  await incus(["file", "push", "-r", `${stock.dataDir}/docker`, `${name}/srv/registry/`], 900_000);
+
+  await incus(["exec", name, "--", "docker", "run", "-d",
+    "--name", "registry", "--restart", "unless-stopped",
+    "-p", `${REGISTRY_PORT}:5000`,
+    "-v", "/srv/registry:/var/lib/registry",
+    REGISTRY_IMAGE], 300_000);
+
+  // Read back that each image is SERVED, by asking for its manifest by digest — which is
+  // exactly what a machine will do.
+  //
+  // Not that the catalog endpoint answers: `{"repositories":[]}` contains the word
+  // `repositories`, so checking for that passed on a registry holding nothing at all, and the
+  // failure surfaced much later as a container that could not be pulled.
+  let answered = false;
+  for (let i = 0; i < 20 && !answered; i++) {
+    const ping = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null",
+      "-w", "%{http_code}", "--max-time", "3",
+      `http://localhost:${REGISTRY_PORT}/v2/`], 30_000);
+    answered = ping?.trim() === "200";
+    if (!answered) await new Promise((r) => setTimeout(r, 2_000));
+  }
+  if (!answered) {
+    throw new RegistryError(
+      `the registry on ${name} started and never answered. Machines in this scenario cannot ` +
+        `fetch an image, so nothing that declares a container will work.`,
+    );
+  }
+
+  const pinned: string[] = [];
+  for (const image of stock.images) {
+    const code = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null",
+      "-w", "%{http_code}", "--max-time", "5",
+      "-H", "Accept: application/vnd.docker.distribution.manifest.v2+json",
+      `http://localhost:${REGISTRY_PORT}/v2/${image.repository}/manifests/${image.digest}`,
+    ], 60_000);
+    if (code?.trim() !== "200") {
+      throw new RegistryError(
+        `the registry on ${name} is running and does not serve ${image.repository}@${image.digest} ` +
+          `(it answered ${code?.trim() || "nothing"}).\n` +
+          `  The images were stocked on this workstation and did not arrive intact, so a ` +
+          `machine declaring that image would fail to pull it.`,
+      );
+    }
+    const reference = pinnedReference(address, image);
+    pinned.push(reference);
+    log(`  serving ${reference}`);
+  }
+  return { machine: name, segment: segment.name, address, pinned };
+}
+
+async function waitForAgent(name: string): Promise<void> {
+  for (let i = 0; i < 90; i++) {
+    if (await succeeds(["exec", name, "--", "true"], 10_000)) return;
+    await new Promise((r) => setTimeout(r, 2_000));
+  }
+  throw new RegistryError(`${name} started and its agent never answered.`);
+}