From be176bab2eaf24470dbddb508c7cca9fdb535bc0 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 29 Aug 2026 00:04:55 +0200 Subject: [PATCH] Automate the lab registry: a sealed machine pulls by digest Closes 04-ISSUES/009. A scenario declares `images:` by tag; the lab stocks a registry on this workstation where there is a network, raises it inside the scenario as scenery, and reports the references a declaration pins -- which are the digests THIS registry assigned, and are not knowable until it is raised. Verified in a sealed machine, confirmed by ping to have no route out: package, service including boot state, a container pinned by digest, and an action inside that container. Applied, idempotent on re-apply, and read back from the machine rather than from the apply's own report. That is the first time the container shape has worked in the lab at all, and it was the shape blocking the substrate bootstrap. Four faults found by running it, three of them mine and one worth keeping: The read-back checked that the catalog endpoint answered, by looking for the substring "repositories" -- which `{"repositories":[]}` also contains. So it passed on a registry holding nothing, and the failure surfaced much later as a container that could not be pulled. It now asks for each image's manifest BY DIGEST, which is what a machine does. A recursive push needs its destination to exist, or incus copies the source's contents rather than the source. The data landed one directory too shallow and the registry found nothing where it looks. The registry writes its blobs as root through a bind mount, so the workstation could not remove its own scratch directory afterwards. Whoever made the files removes them -- the cleanup now runs in a container too. And a cleanup failure no longer fails a raise that succeeded: the scenario is standing and usable, and saying otherwise would be a false report. The base image build did not verify that the runtime trusts the documentation ranges as plain-HTTP registries. Writing the file is not the daemon honouring it, and a base image that looks right fails much later, in a sealed scenario, a long way from its cause. It is now read back from `docker info`. --- scenarios/bootstrap-with-registry.yml | 23 ++++ src/cli.ts | 6 + src/declaration/parse.ts | 5 + src/declaration/types.ts | 8 ++ src/declaration/validate.ts | 24 ++++ src/lifecycle/base.ts | 17 +++ src/lifecycle/raise.ts | 36 +++++- src/lifecycle/registry.ts | 178 +++++++++++++++++++++++++- 8 files changed, 292 insertions(+), 5 deletions(-) create mode 100644 scenarios/bootstrap-with-registry.yml diff --git a/scenarios/bootstrap-with-registry.yml b/scenarios/bootstrap-with-registry.yml new file mode 100644 index 0000000..83a92f0 --- /dev/null +++ b/scenarios/bootstrap-with-registry.yml @@ -0,0 +1,23 @@ +# One machine and a registry, which is the smallest scenario that can exercise a container. +# +# A sealed machine cannot reach a registry and an image placed from an archive cannot keep its +# digest (novox/hq 04-ISSUES/009), so the lab raises one inside the scenario and serves the +# images below from it. What a declaration pins is reported when this is raised — the digest +# belongs to this registry, not to the one the image came from. +scenario: bootstrap-with-registry + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + +images: + - alpine:3.20 + +place: + all: [host, runtime] diff --git a/src/cli.ts b/src/cli.ts index cf543a3..204e3e6 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -130,6 +130,12 @@ async function main(): Promise { }); const seconds = ((Date.now() - started) / 1000).toFixed(1); console.log(`\nraised ${raised.instanceId} in ${seconds}s — ${raised.machines.length} machines usable`); + if (raised.images.length > 0) { + // Printed because this is what a declaration pins, and it is not knowable until the + // scenario has been raised — the digest belongs to this registry. + console.log(`\nimages served, pinned by digest:`); + for (const image of raised.images) console.log(` ${image}`); + } if (scenario.snapshot) { const took = await snapshot(raised.instanceId, scenario.snapshot); console.log(`snapshot '${scenario.snapshot}' in ${took.toFixed(2)}s`); diff --git a/src/declaration/parse.ts b/src/declaration/parse.ts index 26714b7..d38fc24 100644 --- a/src/declaration/parse.ts +++ b/src/declaration/parse.ts @@ -96,6 +96,11 @@ export function parseScenario(text: string): Scenario { }); } + const images = raw["images"]; + if (Array.isArray(images)) { + scenario.images = images.map((i) => String(i)); + } + const place = raw["place"]; if (place && typeof place === "object") { const normalised: Record = {}; diff --git a/src/declaration/types.ts b/src/declaration/types.ts index 2721a2d..b4e5e4e 100644 --- a/src/declaration/types.ts +++ b/src/declaration/types.ts @@ -109,6 +109,14 @@ export interface Scenario { machines: Record; policy?: Policy[]; place?: Placement; + /** + * Container images this scenario needs inside it. + * + * A sealed machine cannot reach a registry, so the lab raises one on a public segment and + * serves these from it. Written as tags — the digest a declaration pins is the one THIS + * registry assigns, and it is reported when the scenario is raised. + */ + images?: string[]; /** Name the state once placement finishes, so a run can return to it. */ snapshot?: string; } diff --git a/src/declaration/validate.ts b/src/declaration/validate.ts index f705a99..20fbdd1 100644 --- a/src/declaration/validate.ts +++ b/src/declaration/validate.ts @@ -299,5 +299,29 @@ export function validate(scenario: Scenario): void { } } + for (const image of scenario.images ?? []) { + if (!image.trim()) { + problems.push("images: an empty entry names nothing"); + } else if (image.includes("@sha256:")) { + // The digest a declaration pins is the one the LAB's registry assigns, which is not + // knowable before the scenario is raised. Naming an upstream digest here would pin + // something this registry will never serve. + problems.push( + `images: '${image}' is pinned by digest. Name it by tag — the lab's registry assigns ` + + `its own digest and reports it when the scenario is raised`, + ); + } + } + if ((scenario.images ?? []).length > 0) { + const hasPublicV4 = Object.values(scenario.segments) + .some((s) => s.kind === "public" && s.cidr.some((c) => !c.includes(":"))); + if (!hasPublicV4) { + problems.push( + "images: this scenario declares images and has no public IPv4 segment to serve them " + + "from. The registry stands in for the outside world, so it sits on a public segment", + ); + } + } + if (problems.length > 0) throw new DeclarationError(problems); } diff --git a/src/lifecycle/base.ts b/src/lifecycle/base.ts index da82b2e..7f215a2 100644 --- a/src/lifecycle/base.ts +++ b/src/lifecycle/base.ts @@ -85,6 +85,23 @@ export async function buildBaseImage( } log(` runtime works (docker ${runtime})`); + // Read back that the runtime will actually pull over plain HTTP from a documentation + // range. Writing the file is not the same as the daemon honouring it, and a base image + // that looks right here fails much later — in a sealed scenario, as a container that + // cannot fetch its image, which is a long way from the cause. + const trusted = await incusOk( + ["exec", BUILDER, "--", "docker", "info", "--format", "{{.RegistryConfig.InsecureRegistryCIDRs}}"], + 60_000, + ); + if (!trusted?.includes("192.0.2.0/24")) { + throw new BaseImageError( + `the runtime in ${BUILDER} does not trust the documentation ranges as plain-HTTP ` + + `registries. It reported: ${trusted?.trim() || "nothing"}\n` + + ` Every scenario raised from this image would fail to pull from its own registry.`, + ); + } + log(" trusts the documentation ranges as registries"); + log(" publishing"); await incus(["stop", BUILDER, "--timeout", "120"], 300_000); await incus(["publish", BUILDER, "--alias", BASE_IMAGE_ALIAS, "--reuse"], 900_000); diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index f91a24f..34dcd86 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -24,6 +24,7 @@ import { planRouters, raiseRouters, raiseTransit } from "./router.ts"; import { applyHostFirewalls } from "./firewall.ts"; import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts"; import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts"; +import { discardStock, raiseRegistry, stockRegistry } from "./registry.ts"; /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */ const COW_DRIVERS = ["btrfs", "zfs"]; @@ -44,6 +45,13 @@ export interface RaisedScenario { machines: string[]; networks: string[]; pool: string; + /** + * Images the scenario's registry serves, as references a declaration can pin. + * + * Reported rather than declared, because the digest is the one this registry assigned and + * is not knowable before it was raised. + */ + images: string[]; } export class RaiseError extends Error { @@ -171,9 +179,10 @@ export async function raise( // A scenario that places a runtime or an image needs machines built from the base image, // because a sealed machine cannot install one (novox/hq ADR 0006). Chosen here rather than // declared, so a scenario says WHAT it needs and not which image provides it. - const needsRuntime = planPlacements(scenario).some(({ artifacts }) => - artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX)) - ); + const needsRuntime = (scenario.images ?? []).length > 0 || + planPlacements(scenario).some(({ artifacts }) => + artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX)) + ); if (needsRuntime && !options.image && !(await baseImageExists())) { throw new Error( `this scenario needs a container runtime inside its machines, and '${BASE_IMAGE_ALIAS}' ` + @@ -231,6 +240,24 @@ export async function raise( const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log); if (transit) routers.push(transit); + // Stocked on this workstation, where there is a network, and served from inside the + // scenario, where there is not (novox/hq 04-ISSUES/009). + step = "stocking the registry"; + const stock = await stockRegistry(scenario.images ?? [], log); + let registry: Awaited> = null; + try { + step = "raising the registry"; + registry = await raiseRegistry(scenario, instanceId, stock, log); + } finally { + // Cleaning up scratch must not fail a raise that succeeded. The scenario is standing + // and usable; a directory left behind is untidy, and saying so is the honest report. + try { + await discardStock(stock); + } catch (err) { + log(` (could not remove the registry's scratch directory: ${(err as Error).message})`); + } + } + step = "routing machines through their gateways"; await applyDefaultRoutes(scenario, byMachine, log); @@ -247,7 +274,8 @@ export async function raise( return { instanceId, scenario: scenario.scenario, - machines: [...created, ...routers], + images: registry?.pinned ?? [], + machines: [...created, ...routers, ...(registry ? [registry.machine] : [])], networks, pool, }; diff --git a/src/lifecycle/registry.ts b/src/lifecycle/registry.ts index 39b2396..4ab52c2 100644 --- a/src/lifecycle/registry.ts +++ b/src/lifecycle/registry.ts @@ -19,6 +19,10 @@ */ import { spawn } from "node:child_process"; + +import { incus, incusOk, succeeds } from "../incus/client.ts"; +import { macFor, networkName } from "./names.ts"; +import { BASE_IMAGE_ALIAS, placeImage } from "./place.ts"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -116,13 +120,29 @@ export async function stockRegistry( return { dataDir, images }; } catch (err) { - await rm(dataDir, { recursive: true, force: true }); + await discardStock({ dataDir, images: [] }); throw err; } finally { await docker(["rm", "-f", container], 60_000); } } +/** + * Remove a stocked registry's data. + * + * Through a container, because a container wrote it. The registry runs as root inside, so the + * blobs it writes into a bind mount are owned by root and an ordinary process cannot remove + * them — `rmdir` fails with EACCES on a directory that looks like ours. + * + * Whoever made the files removes them. + */ +export async function discardStock(stock: Stock): Promise { + if (!stock.dataDir) return; + await docker(["run", "--rm", "-v", `${stock.dataDir}:/stock`, REGISTRY_IMAGE, + "sh", "-c", "rm -rf /stock/* /stock/.[!.]* 2>/dev/null || true"], 120_000); + await rm(stock.dataDir, { recursive: true, force: true }).catch(() => {}); +} + /** `alpine:3.20` and `alpine` both serve from `alpine`; `foo/bar:1` from `foo/bar`. */ export function repositoryFor(reference: string): string { const withoutDigest = reference.split("@")[0] ?? reference; @@ -197,3 +217,159 @@ export function registryAddress(cidr: string): string { export function pinnedReference(address: string, image: StockedImage): string { return `${address}:${REGISTRY_PORT}/${image.repository}@${image.digest}`; } + +// --- raising it inside a scenario --------------------------------------------------------------- + +/** What a raised registry is, and what a declaration needs from it. */ +export interface RaisedRegistry { + machine: string; + segment: string; + address: string; + /** Each image, as a reference a declaration can pin. */ + pinned: string[]; +} + +/** + * Pick the segment the registry sits on. + * + * A public segment, because that is what stands in for the outside world — a first node fetches + * from upstream, and this is upstream. An IPv4 range, because a machine has to be pointed at it + * by address. + */ +export function registrySegment( + segments: Record, +): { name: string; cidr: string } | null { + for (const [name, segment] of Object.entries(segments)) { + if (segment.kind !== "public") continue; + const v4 = segment.cidr.find((c) => !c.includes(":")); + if (v4) return { name, cidr: v4 }; + } + return null; +} + +/** + * Raise a registry inside the scenario and load the stocked images into it. + * + * Scenery, in the same sense the transit router is: nothing under test runs on it, it holds no + * identity, and no assertion is made about its internals. It exists so that a machine can fetch + * an image the way a real one does — over the network, from a registry, by digest. + */ +export async function raiseRegistry( + scenario: { segments: Record }, + instanceId: string, + stock: Stock, + log: (message: string) => void = () => {}, +): Promise { + if (stock.images.length === 0) return null; + + const segment = registrySegment(scenario.segments); + if (!segment) { + throw new RegistryError( + `this scenario declares images and has no public IPv4 segment to serve them from.\n` + + ` The registry stands in for the outside world, so it sits on a public segment.`, + ); + } + + const address = registryAddress(segment.cidr); + const name = `mlab-${instanceId}-registry`; + const prefix = segment.cidr.slice(segment.cidr.lastIndexOf("/")); + + if (!(await succeeds(["config", "show", name], 15_000))) { + await incus([ + "init", BASE_IMAGE_ALIAS, name, "--vm", + "-c", "security.secureboot=false", + "-c", "limits.memory=1GiB", + "-c", `user.mesh-lab.instance=${instanceId}`, + // Tagged as a machine as well, so `destroy` finds it with one query — a router that + // carried only its own tag was left behind and held its networks open. + "-c", "user.mesh-lab.machine=registry", + "-c", "user.mesh-lab.registry=true", + ], 300_000); + await succeeds(["config", "device", "remove", name, "eth0"], 15_000); + await incus([ + "config", "device", "add", name, "eth0", "nic", + "nictype=bridged", + `parent=${networkName(instanceId, segment.name)}`, + `hwaddr=${macFor(instanceId, "registry", 0)}`, + ]); + } + await succeeds(["start", name], 60_000); + await waitForAgent(name); + + // Address it by MAC, never by interface name: a machine with a container runtime has a + // `docker0` that sorts before `enp5s0`, and naive selection configures that instead — which + // then overlaps the segment and breaks routing on the machine. + const mac = macFor(instanceId, "registry", 0); + await incus(["exec", name, "--", "sh", "-c", + `dev=$(ip -o link | awk -F': ' '/${mac}/ {print $2}' | head -1); ` + + `[ -n "$dev" ] && ip addr add ${address}${prefix} dev "$dev" 2>/dev/null; ` + + `[ -n "$dev" ] && ip link set "$dev" up`], 60_000); + + log(` registry on ${segment.name} at ${address}`); + + // The registry's own image, placed by tag — an archive keeps a tag and cannot keep a digest, + // which is the whole reason this machine exists. + await placeImage(name, "registry", REGISTRY_IMAGE, () => {}); + + // The destination must EXIST before a recursive push, or incus copies the source's contents + // rather than the source — the data lands one directory too shallow, the registry finds + // nothing where it looks, and every pull fails with `not found`. + await incus(["exec", name, "--", "mkdir", "-p", "/srv/registry"], 60_000); + await incus(["file", "push", "-r", `${stock.dataDir}/docker`, `${name}/srv/registry/`], 900_000); + + await incus(["exec", name, "--", "docker", "run", "-d", + "--name", "registry", "--restart", "unless-stopped", + "-p", `${REGISTRY_PORT}:5000`, + "-v", "/srv/registry:/var/lib/registry", + REGISTRY_IMAGE], 300_000); + + // Read back that each image is SERVED, by asking for its manifest by digest — which is + // exactly what a machine will do. + // + // Not that the catalog endpoint answers: `{"repositories":[]}` contains the word + // `repositories`, so checking for that passed on a registry holding nothing at all, and the + // failure surfaced much later as a container that could not be pulled. + let answered = false; + for (let i = 0; i < 20 && !answered; i++) { + const ping = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null", + "-w", "%{http_code}", "--max-time", "3", + `http://localhost:${REGISTRY_PORT}/v2/`], 30_000); + answered = ping?.trim() === "200"; + if (!answered) await new Promise((r) => setTimeout(r, 2_000)); + } + if (!answered) { + throw new RegistryError( + `the registry on ${name} started and never answered. Machines in this scenario cannot ` + + `fetch an image, so nothing that declares a container will work.`, + ); + } + + const pinned: string[] = []; + for (const image of stock.images) { + const code = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null", + "-w", "%{http_code}", "--max-time", "5", + "-H", "Accept: application/vnd.docker.distribution.manifest.v2+json", + `http://localhost:${REGISTRY_PORT}/v2/${image.repository}/manifests/${image.digest}`, + ], 60_000); + if (code?.trim() !== "200") { + throw new RegistryError( + `the registry on ${name} is running and does not serve ${image.repository}@${image.digest} ` + + `(it answered ${code?.trim() || "nothing"}).\n` + + ` The images were stocked on this workstation and did not arrive intact, so a ` + + `machine declaring that image would fail to pull it.`, + ); + } + const reference = pinnedReference(address, image); + pinned.push(reference); + log(` serving ${reference}`); + } + return { machine: name, segment: segment.name, address, pinned }; +} + +async function waitForAgent(name: string): Promise { + for (let i = 0; i < 90; i++) { + if (await succeeds(["exec", name, "--", "true"], 10_000)) return; + await new Promise((r) => setTimeout(r, 2_000)); + } + throw new RegistryError(`${name} started and its agent never answered.`); +}