scenarios/bootstrap-with-registry.yml

@@ -0,0 +1,23 @@
+# One machine and a registry, which is the smallest scenario that can exercise a container.
+#
+# A sealed machine cannot reach a registry and an image placed from an archive cannot keep its
+# digest (novox/hq 04-ISSUES/009), so the lab raises one inside the scenario and serves the
+# images below from it. What a declaration pins is reported when this is raised — the digest
+# belongs to this registry, not to the one the image came from.
+scenario: bootstrap-with-registry
+
+segments:
+  hosting:
+    kind: public
+    cidr: [192.0.2.0/24]
+
+machines:
+  anchor:
+    at: { segment: hosting, address: [192.0.2.10] }
+    inbound: allow
+
+images:
+  - alpine:3.20
+
+place:
+  all: [host, runtime]

src/cli.ts

@@ -130,6 +130,12 @@ async function main(): Promise<void> {
       });
       const seconds = ((Date.now() - started) / 1000).toFixed(1);
       console.log(`\nraised ${raised.instanceId} in ${seconds}s — ${raised.machines.length} machines usable`);
+      if (raised.images.length > 0) {
+        // Printed because this is what a declaration pins, and it is not knowable until the
+        // scenario has been raised — the digest belongs to this registry.
+        console.log(`\nimages served, pinned by digest:`);
+        for (const image of raised.images) console.log(`  ${image}`);
+      }
       if (scenario.snapshot) {
         const took = await snapshot(raised.instanceId, scenario.snapshot);
         console.log(`snapshot '${scenario.snapshot}' in ${took.toFixed(2)}s`);

src/declaration/parse.ts

@@ -96,6 +96,11 @@ export function parseScenario(text: string): Scenario {
     });
   }
 
+  const images = raw["images"];
+  if (Array.isArray(images)) {
+    scenario.images = images.map((i) => String(i));
+  }
+
   const place = raw["place"];
   if (place && typeof place === "object") {
     const normalised: Record<string, string[]> = {};

src/declaration/types.ts

@@ -109,6 +109,14 @@ export interface Scenario {
   machines: Record<string, Machine>;
   policy?: Policy[];
   place?: Placement;
+  /**
+   * Container images this scenario needs inside it.
+   *
+   * A sealed machine cannot reach a registry, so the lab raises one on a public segment and
+   * serves these from it. Written as tags — the digest a declaration pins is the one THIS
+   * registry assigns, and it is reported when the scenario is raised.
+   */
+  images?: string[];
   /** Name the state once placement finishes, so a run can return to it. */
   snapshot?: string;
 }

src/declaration/validate.ts

@@ -299,5 +299,29 @@ export function validate(scenario: Scenario): void {
     }
   }
 
+  for (const image of scenario.images ?? []) {
+    if (!image.trim()) {
+      problems.push("images: an empty entry names nothing");
+    } else if (image.includes("@sha256:")) {
+      // The digest a declaration pins is the one the LAB's registry assigns, which is not
+      // knowable before the scenario is raised. Naming an upstream digest here would pin
+      // something this registry will never serve.
+      problems.push(
+        `images: '${image}' is pinned by digest. Name it by tag — the lab's registry assigns ` +
+        `its own digest and reports it when the scenario is raised`,
+      );
+    }
+  }
+  if ((scenario.images ?? []).length > 0) {
+    const hasPublicV4 = Object.values(scenario.segments)
+      .some((s) => s.kind === "public" && s.cidr.some((c) => !c.includes(":")));
+    if (!hasPublicV4) {
+      problems.push(
+        "images: this scenario declares images and has no public IPv4 segment to serve them " +
+        "from. The registry stands in for the outside world, so it sits on a public segment",
+      );
+    }
+  }
+
   if (problems.length > 0) throw new DeclarationError(problems);
 }

src/lifecycle/base.ts

@@ -85,6 +85,23 @@ export async function buildBaseImage(
     }
     log(`  runtime works (docker ${runtime})`);
 
+    // Read back that the runtime will actually pull over plain HTTP from a documentation
+    // range. Writing the file is not the same as the daemon honouring it, and a base image
+    // that looks right here fails much later — in a sealed scenario, as a container that
+    // cannot fetch its image, which is a long way from the cause.
+    const trusted = await incusOk(
+      ["exec", BUILDER, "--", "docker", "info", "--format", "{{.RegistryConfig.InsecureRegistryCIDRs}}"],
+      60_000,
+    );
+    if (!trusted?.includes("192.0.2.0/24")) {
+      throw new BaseImageError(
+        `the runtime in ${BUILDER} does not trust the documentation ranges as plain-HTTP ` +
+          `registries. It reported: ${trusted?.trim() || "nothing"}\n` +
+          `  Every scenario raised from this image would fail to pull from its own registry.`,
+      );
+    }
+    log("  trusts the documentation ranges as registries");
+
     log("  publishing");
     await incus(["stop", BUILDER, "--timeout", "120"], 300_000);
     await incus(["publish", BUILDER, "--alias", BASE_IMAGE_ALIAS, "--reuse"], 900_000);

src/lifecycle/raise.ts

@@ -24,6 +24,7 @@ import { planRouters, raiseRouters, raiseTransit } from "./router.ts";
 import { applyHostFirewalls } from "./firewall.ts";
 import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts";
 import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts";
+import { discardStock, raiseRegistry, stockRegistry } from "./registry.ts";
 
 /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */
 const COW_DRIVERS = ["btrfs", "zfs"];
@@ -44,6 +45,13 @@ export interface RaisedScenario {
   machines: string[];
   networks: string[];
   pool: string;
+  /**
+   * Images the scenario's registry serves, as references a declaration can pin.
+   *
+   * Reported rather than declared, because the digest is the one this registry assigned and
+   * is not knowable before it was raised.
+   */
+  images: string[];
 }
 
 export class RaiseError extends Error {
@@ -171,9 +179,10 @@ export async function raise(
   // A scenario that places a runtime or an image needs machines built from the base image,
   // because a sealed machine cannot install one (novox/hq ADR 0006). Chosen here rather than
   // declared, so a scenario says WHAT it needs and not which image provides it.
-  const needsRuntime = planPlacements(scenario).some(({ artifacts }) =>
+  const needsRuntime = (scenario.images ?? []).length > 0 ||
-    artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX))
+    planPlacements(scenario).some(({ artifacts }) =>
-  );
+      artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX))
+    );
   if (needsRuntime && !options.image && !(await baseImageExists())) {
     throw new Error(
       `this scenario needs a container runtime inside its machines, and '${BASE_IMAGE_ALIAS}' ` +
@@ -231,6 +240,24 @@ export async function raise(
     const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log);
     if (transit) routers.push(transit);
 
+    // Stocked on this workstation, where there is a network, and served from inside the
+    // scenario, where there is not (novox/hq 04-ISSUES/009).
+    step = "stocking the registry";
+    const stock = await stockRegistry(scenario.images ?? [], log);
+    let registry: Awaited<ReturnType<typeof raiseRegistry>> = null;
+    try {
+      step = "raising the registry";
+      registry = await raiseRegistry(scenario, instanceId, stock, log);
+    } finally {
+      // Cleaning up scratch must not fail a raise that succeeded. The scenario is standing
+      // and usable; a directory left behind is untidy, and saying so is the honest report.
+      try {
+        await discardStock(stock);
+      } catch (err) {
+        log(`  (could not remove the registry's scratch directory: ${(err as Error).message})`);
+      }
+    }
+
     step = "routing machines through their gateways";
     await applyDefaultRoutes(scenario, byMachine, log);
 
@@ -247,7 +274,8 @@ export async function raise(
     return {
       instanceId,
       scenario: scenario.scenario,
-      machines: [...created, ...routers],
+      images: registry?.pinned ?? [],
+      machines: [...created, ...routers, ...(registry ? [registry.machine] : [])],
       networks,
       pool,
     };

src/lifecycle/registry.ts

@@ -19,6 +19,10 @@
  */
 
 import { spawn } from "node:child_process";
+
+import { incus, incusOk, succeeds } from "../incus/client.ts";
+import { macFor, networkName } from "./names.ts";
+import { BASE_IMAGE_ALIAS, placeImage } from "./place.ts";
 import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -116,13 +120,29 @@ export async function stockRegistry(
 
     return { dataDir, images };
   } catch (err) {
-    await rm(dataDir, { recursive: true, force: true });
+    await discardStock({ dataDir, images: [] });
     throw err;
   } finally {
     await docker(["rm", "-f", container], 60_000);
   }
 }
 
+/**
+ * Remove a stocked registry's data.
+ *
+ * Through a container, because a container wrote it. The registry runs as root inside, so the
+ * blobs it writes into a bind mount are owned by root and an ordinary process cannot remove
+ * them — `rmdir` fails with EACCES on a directory that looks like ours.
+ *
+ * Whoever made the files removes them.
+ */
+export async function discardStock(stock: Stock): Promise<void> {
+  if (!stock.dataDir) return;
+  await docker(["run", "--rm", "-v", `${stock.dataDir}:/stock`, REGISTRY_IMAGE,
+    "sh", "-c", "rm -rf /stock/* /stock/.[!.]* 2>/dev/null || true"], 120_000);
+  await rm(stock.dataDir, { recursive: true, force: true }).catch(() => {});
+}
+
 /** `alpine:3.20` and `alpine` both serve from `alpine`; `foo/bar:1` from `foo/bar`. */
 export function repositoryFor(reference: string): string {
   const withoutDigest = reference.split("@")[0] ?? reference;
@@ -197,3 +217,159 @@ export function registryAddress(cidr: string): string {
 export function pinnedReference(address: string, image: StockedImage): string {
   return `${address}:${REGISTRY_PORT}/${image.repository}@${image.digest}`;
 }
+
+// --- raising it inside a scenario ---------------------------------------------------------------
+
+/** What a raised registry is, and what a declaration needs from it. */
+export interface RaisedRegistry {
+  machine: string;
+  segment: string;
+  address: string;
+  /** Each image, as a reference a declaration can pin. */
+  pinned: string[];
+}
+
+/**
+ * Pick the segment the registry sits on.
+ *
+ * A public segment, because that is what stands in for the outside world — a first node fetches
+ * from upstream, and this is upstream. An IPv4 range, because a machine has to be pointed at it
+ * by address.
+ */
+export function registrySegment(
+  segments: Record<string, { kind: string; cidr: string[] }>,
+): { name: string; cidr: string } | null {
+  for (const [name, segment] of Object.entries(segments)) {
+    if (segment.kind !== "public") continue;
+    const v4 = segment.cidr.find((c) => !c.includes(":"));
+    if (v4) return { name, cidr: v4 };
+  }
+  return null;
+}
+
+/**
+ * Raise a registry inside the scenario and load the stocked images into it.
+ *
+ * Scenery, in the same sense the transit router is: nothing under test runs on it, it holds no
+ * identity, and no assertion is made about its internals. It exists so that a machine can fetch
+ * an image the way a real one does — over the network, from a registry, by digest.
+ */
+export async function raiseRegistry(
+  scenario: { segments: Record<string, { kind: string; cidr: string[] }> },
+  instanceId: string,
+  stock: Stock,
+  log: (message: string) => void = () => {},
+): Promise<RaisedRegistry | null> {
+  if (stock.images.length === 0) return null;
+
+  const segment = registrySegment(scenario.segments);
+  if (!segment) {
+    throw new RegistryError(
+      `this scenario declares images and has no public IPv4 segment to serve them from.\n` +
+        `  The registry stands in for the outside world, so it sits on a public segment.`,
+    );
+  }
+
+  const address = registryAddress(segment.cidr);
+  const name = `mlab-${instanceId}-registry`;
+  const prefix = segment.cidr.slice(segment.cidr.lastIndexOf("/"));
+
+  if (!(await succeeds(["config", "show", name], 15_000))) {
+    await incus([
+      "init", BASE_IMAGE_ALIAS, name, "--vm",
+      "-c", "security.secureboot=false",
+      "-c", "limits.memory=1GiB",
+      "-c", `user.mesh-lab.instance=${instanceId}`,
+      // Tagged as a machine as well, so `destroy` finds it with one query — a router that
+      // carried only its own tag was left behind and held its networks open.
+      "-c", "user.mesh-lab.machine=registry",
+      "-c", "user.mesh-lab.registry=true",
+    ], 300_000);
+    await succeeds(["config", "device", "remove", name, "eth0"], 15_000);
+    await incus([
+      "config", "device", "add", name, "eth0", "nic",
+      "nictype=bridged",
+      `parent=${networkName(instanceId, segment.name)}`,
+      `hwaddr=${macFor(instanceId, "registry", 0)}`,
+    ]);
+  }
+  await succeeds(["start", name], 60_000);
+  await waitForAgent(name);
+
+  // Address it by MAC, never by interface name: a machine with a container runtime has a
+  // `docker0` that sorts before `enp5s0`, and naive selection configures that instead — which
+  // then overlaps the segment and breaks routing on the machine.
+  const mac = macFor(instanceId, "registry", 0);
+  await incus(["exec", name, "--", "sh", "-c",
+    `dev=$(ip -o link | awk -F': ' '/${mac}/ {print $2}' | head -1); ` +
+    `[ -n "$dev" ] && ip addr add ${address}${prefix} dev "$dev" 2>/dev/null; ` +
+    `[ -n "$dev" ] && ip link set "$dev" up`], 60_000);
+
+  log(`  registry on ${segment.name} at ${address}`);
+
+  // The registry's own image, placed by tag — an archive keeps a tag and cannot keep a digest,
+  // which is the whole reason this machine exists.
+  await placeImage(name, "registry", REGISTRY_IMAGE, () => {});
+
+  // The destination must EXIST before a recursive push, or incus copies the source's contents
+  // rather than the source — the data lands one directory too shallow, the registry finds
+  // nothing where it looks, and every pull fails with `not found`.
+  await incus(["exec", name, "--", "mkdir", "-p", "/srv/registry"], 60_000);
+  await incus(["file", "push", "-r", `${stock.dataDir}/docker`, `${name}/srv/registry/`], 900_000);
+
+  await incus(["exec", name, "--", "docker", "run", "-d",
+    "--name", "registry", "--restart", "unless-stopped",
+    "-p", `${REGISTRY_PORT}:5000`,
+    "-v", "/srv/registry:/var/lib/registry",
+    REGISTRY_IMAGE], 300_000);
+
+  // Read back that each image is SERVED, by asking for its manifest by digest — which is
+  // exactly what a machine will do.
+  //
+  // Not that the catalog endpoint answers: `{"repositories":[]}` contains the word
+  // `repositories`, so checking for that passed on a registry holding nothing at all, and the
+  // failure surfaced much later as a container that could not be pulled.
+  let answered = false;
+  for (let i = 0; i < 20 && !answered; i++) {
+    const ping = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null",
+      "-w", "%{http_code}", "--max-time", "3",
+      `http://localhost:${REGISTRY_PORT}/v2/`], 30_000);
+    answered = ping?.trim() === "200";
+    if (!answered) await new Promise((r) => setTimeout(r, 2_000));
+  }
+  if (!answered) {
+    throw new RegistryError(
+      `the registry on ${name} started and never answered. Machines in this scenario cannot ` +
+        `fetch an image, so nothing that declares a container will work.`,
+    );
+  }
+
+  const pinned: string[] = [];
+  for (const image of stock.images) {
+    const code = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null",
+      "-w", "%{http_code}", "--max-time", "5",
+      "-H", "Accept: application/vnd.docker.distribution.manifest.v2+json",
+      `http://localhost:${REGISTRY_PORT}/v2/${image.repository}/manifests/${image.digest}`,
+    ], 60_000);
+    if (code?.trim() !== "200") {
+      throw new RegistryError(
+        `the registry on ${name} is running and does not serve ${image.repository}@${image.digest} ` +
+          `(it answered ${code?.trim() || "nothing"}).\n` +
+          `  The images were stocked on this workstation and did not arrive intact, so a ` +
+          `machine declaring that image would fail to pull it.`,
+      );
+    }
+    const reference = pinnedReference(address, image);
+    pinned.push(reference);
+    log(`  serving ${reference}`);
+  }
+  return { machine: name, segment: segment.name, address, pinned };
+}
+
+async function waitForAgent(name: string): Promise<void> {
+  for (let i = 0; i < 90; i++) {
+    if (await succeeds(["exec", name, "--", "true"], 10_000)) return;
+    await new Promise((r) => setTimeout(r, 2_000));
+  }
+  throw new RegistryError(`${name} started and its agent never answered.`);
+}