diff --git a/scenarios/bootstrap-with-registry.yml b/scenarios/bootstrap-with-registry.yml new file mode 100644 index 0000000..83a92f0 --- /dev/null +++ b/scenarios/bootstrap-with-registry.yml @@ -0,0 +1,23 @@ +# One machine and a registry, which is the smallest scenario that can exercise a container. +# +# A sealed machine cannot reach a registry and an image placed from an archive cannot keep its +# digest (novox/hq 04-ISSUES/009), so the lab raises one inside the scenario and serves the +# images below from it. What a declaration pins is reported when this is raised — the digest +# belongs to this registry, not to the one the image came from. +scenario: bootstrap-with-registry + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + +images: + - alpine:3.20 + +place: + all: [host, runtime] diff --git a/src/cli.ts b/src/cli.ts index cf543a3..204e3e6 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -130,6 +130,12 @@ async function main(): Promise { }); const seconds = ((Date.now() - started) / 1000).toFixed(1); console.log(`\nraised ${raised.instanceId} in ${seconds}s — ${raised.machines.length} machines usable`); + if (raised.images.length > 0) { + // Printed because this is what a declaration pins, and it is not knowable until the + // scenario has been raised — the digest belongs to this registry. + console.log(`\nimages served, pinned by digest:`); + for (const image of raised.images) console.log(` ${image}`); + } if (scenario.snapshot) { const took = await snapshot(raised.instanceId, scenario.snapshot); console.log(`snapshot '${scenario.snapshot}' in ${took.toFixed(2)}s`); diff --git a/src/declaration/parse.ts b/src/declaration/parse.ts index 26714b7..d38fc24 100644 --- a/src/declaration/parse.ts +++ b/src/declaration/parse.ts @@ -96,6 +96,11 @@ export function parseScenario(text: string): Scenario { }); } + const images = raw["images"]; + if (Array.isArray(images)) { + scenario.images = images.map((i) => String(i)); + } + const place = raw["place"]; if (place && typeof place === "object") { const normalised: Record = {}; diff --git a/src/declaration/types.ts b/src/declaration/types.ts index 2721a2d..b4e5e4e 100644 --- a/src/declaration/types.ts +++ b/src/declaration/types.ts @@ -109,6 +109,14 @@ export interface Scenario { machines: Record; policy?: Policy[]; place?: Placement; + /** + * Container images this scenario needs inside it. + * + * A sealed machine cannot reach a registry, so the lab raises one on a public segment and + * serves these from it. Written as tags — the digest a declaration pins is the one THIS + * registry assigns, and it is reported when the scenario is raised. + */ + images?: string[]; /** Name the state once placement finishes, so a run can return to it. */ snapshot?: string; } diff --git a/src/declaration/validate.ts b/src/declaration/validate.ts index f705a99..20fbdd1 100644 --- a/src/declaration/validate.ts +++ b/src/declaration/validate.ts @@ -299,5 +299,29 @@ export function validate(scenario: Scenario): void { } } + for (const image of scenario.images ?? []) { + if (!image.trim()) { + problems.push("images: an empty entry names nothing"); + } else if (image.includes("@sha256:")) { + // The digest a declaration pins is the one the LAB's registry assigns, which is not + // knowable before the scenario is raised. Naming an upstream digest here would pin + // something this registry will never serve. + problems.push( + `images: '${image}' is pinned by digest. Name it by tag — the lab's registry assigns ` + + `its own digest and reports it when the scenario is raised`, + ); + } + } + if ((scenario.images ?? []).length > 0) { + const hasPublicV4 = Object.values(scenario.segments) + .some((s) => s.kind === "public" && s.cidr.some((c) => !c.includes(":"))); + if (!hasPublicV4) { + problems.push( + "images: this scenario declares images and has no public IPv4 segment to serve them " + + "from. The registry stands in for the outside world, so it sits on a public segment", + ); + } + } + if (problems.length > 0) throw new DeclarationError(problems); } diff --git a/src/lifecycle/base.ts b/src/lifecycle/base.ts index da82b2e..7f215a2 100644 --- a/src/lifecycle/base.ts +++ b/src/lifecycle/base.ts @@ -85,6 +85,23 @@ export async function buildBaseImage( } log(` runtime works (docker ${runtime})`); + // Read back that the runtime will actually pull over plain HTTP from a documentation + // range. Writing the file is not the same as the daemon honouring it, and a base image + // that looks right here fails much later — in a sealed scenario, as a container that + // cannot fetch its image, which is a long way from the cause. + const trusted = await incusOk( + ["exec", BUILDER, "--", "docker", "info", "--format", "{{.RegistryConfig.InsecureRegistryCIDRs}}"], + 60_000, + ); + if (!trusted?.includes("192.0.2.0/24")) { + throw new BaseImageError( + `the runtime in ${BUILDER} does not trust the documentation ranges as plain-HTTP ` + + `registries. It reported: ${trusted?.trim() || "nothing"}\n` + + ` Every scenario raised from this image would fail to pull from its own registry.`, + ); + } + log(" trusts the documentation ranges as registries"); + log(" publishing"); await incus(["stop", BUILDER, "--timeout", "120"], 300_000); await incus(["publish", BUILDER, "--alias", BASE_IMAGE_ALIAS, "--reuse"], 900_000); diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index f91a24f..34dcd86 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -24,6 +24,7 @@ import { planRouters, raiseRouters, raiseTransit } from "./router.ts"; import { applyHostFirewalls } from "./firewall.ts"; import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts"; import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts"; +import { discardStock, raiseRegistry, stockRegistry } from "./registry.ts"; /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */ const COW_DRIVERS = ["btrfs", "zfs"]; @@ -44,6 +45,13 @@ export interface RaisedScenario { machines: string[]; networks: string[]; pool: string; + /** + * Images the scenario's registry serves, as references a declaration can pin. + * + * Reported rather than declared, because the digest is the one this registry assigned and + * is not knowable before it was raised. + */ + images: string[]; } export class RaiseError extends Error { @@ -171,9 +179,10 @@ export async function raise( // A scenario that places a runtime or an image needs machines built from the base image, // because a sealed machine cannot install one (novox/hq ADR 0006). Chosen here rather than // declared, so a scenario says WHAT it needs and not which image provides it. - const needsRuntime = planPlacements(scenario).some(({ artifacts }) => - artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX)) - ); + const needsRuntime = (scenario.images ?? []).length > 0 || + planPlacements(scenario).some(({ artifacts }) => + artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX)) + ); if (needsRuntime && !options.image && !(await baseImageExists())) { throw new Error( `this scenario needs a container runtime inside its machines, and '${BASE_IMAGE_ALIAS}' ` + @@ -231,6 +240,24 @@ export async function raise( const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log); if (transit) routers.push(transit); + // Stocked on this workstation, where there is a network, and served from inside the + // scenario, where there is not (novox/hq 04-ISSUES/009). + step = "stocking the registry"; + const stock = await stockRegistry(scenario.images ?? [], log); + let registry: Awaited> = null; + try { + step = "raising the registry"; + registry = await raiseRegistry(scenario, instanceId, stock, log); + } finally { + // Cleaning up scratch must not fail a raise that succeeded. The scenario is standing + // and usable; a directory left behind is untidy, and saying so is the honest report. + try { + await discardStock(stock); + } catch (err) { + log(` (could not remove the registry's scratch directory: ${(err as Error).message})`); + } + } + step = "routing machines through their gateways"; await applyDefaultRoutes(scenario, byMachine, log); @@ -247,7 +274,8 @@ export async function raise( return { instanceId, scenario: scenario.scenario, - machines: [...created, ...routers], + images: registry?.pinned ?? [], + machines: [...created, ...routers, ...(registry ? [registry.machine] : [])], networks, pool, }; diff --git a/src/lifecycle/registry.ts b/src/lifecycle/registry.ts index 39b2396..4ab52c2 100644 --- a/src/lifecycle/registry.ts +++ b/src/lifecycle/registry.ts @@ -19,6 +19,10 @@ */ import { spawn } from "node:child_process"; + +import { incus, incusOk, succeeds } from "../incus/client.ts"; +import { macFor, networkName } from "./names.ts"; +import { BASE_IMAGE_ALIAS, placeImage } from "./place.ts"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -116,13 +120,29 @@ export async function stockRegistry( return { dataDir, images }; } catch (err) { - await rm(dataDir, { recursive: true, force: true }); + await discardStock({ dataDir, images: [] }); throw err; } finally { await docker(["rm", "-f", container], 60_000); } } +/** + * Remove a stocked registry's data. + * + * Through a container, because a container wrote it. The registry runs as root inside, so the + * blobs it writes into a bind mount are owned by root and an ordinary process cannot remove + * them — `rmdir` fails with EACCES on a directory that looks like ours. + * + * Whoever made the files removes them. + */ +export async function discardStock(stock: Stock): Promise { + if (!stock.dataDir) return; + await docker(["run", "--rm", "-v", `${stock.dataDir}:/stock`, REGISTRY_IMAGE, + "sh", "-c", "rm -rf /stock/* /stock/.[!.]* 2>/dev/null || true"], 120_000); + await rm(stock.dataDir, { recursive: true, force: true }).catch(() => {}); +} + /** `alpine:3.20` and `alpine` both serve from `alpine`; `foo/bar:1` from `foo/bar`. */ export function repositoryFor(reference: string): string { const withoutDigest = reference.split("@")[0] ?? reference; @@ -197,3 +217,159 @@ export function registryAddress(cidr: string): string { export function pinnedReference(address: string, image: StockedImage): string { return `${address}:${REGISTRY_PORT}/${image.repository}@${image.digest}`; } + +// --- raising it inside a scenario --------------------------------------------------------------- + +/** What a raised registry is, and what a declaration needs from it. */ +export interface RaisedRegistry { + machine: string; + segment: string; + address: string; + /** Each image, as a reference a declaration can pin. */ + pinned: string[]; +} + +/** + * Pick the segment the registry sits on. + * + * A public segment, because that is what stands in for the outside world — a first node fetches + * from upstream, and this is upstream. An IPv4 range, because a machine has to be pointed at it + * by address. + */ +export function registrySegment( + segments: Record, +): { name: string; cidr: string } | null { + for (const [name, segment] of Object.entries(segments)) { + if (segment.kind !== "public") continue; + const v4 = segment.cidr.find((c) => !c.includes(":")); + if (v4) return { name, cidr: v4 }; + } + return null; +} + +/** + * Raise a registry inside the scenario and load the stocked images into it. + * + * Scenery, in the same sense the transit router is: nothing under test runs on it, it holds no + * identity, and no assertion is made about its internals. It exists so that a machine can fetch + * an image the way a real one does — over the network, from a registry, by digest. + */ +export async function raiseRegistry( + scenario: { segments: Record }, + instanceId: string, + stock: Stock, + log: (message: string) => void = () => {}, +): Promise { + if (stock.images.length === 0) return null; + + const segment = registrySegment(scenario.segments); + if (!segment) { + throw new RegistryError( + `this scenario declares images and has no public IPv4 segment to serve them from.\n` + + ` The registry stands in for the outside world, so it sits on a public segment.`, + ); + } + + const address = registryAddress(segment.cidr); + const name = `mlab-${instanceId}-registry`; + const prefix = segment.cidr.slice(segment.cidr.lastIndexOf("/")); + + if (!(await succeeds(["config", "show", name], 15_000))) { + await incus([ + "init", BASE_IMAGE_ALIAS, name, "--vm", + "-c", "security.secureboot=false", + "-c", "limits.memory=1GiB", + "-c", `user.mesh-lab.instance=${instanceId}`, + // Tagged as a machine as well, so `destroy` finds it with one query — a router that + // carried only its own tag was left behind and held its networks open. + "-c", "user.mesh-lab.machine=registry", + "-c", "user.mesh-lab.registry=true", + ], 300_000); + await succeeds(["config", "device", "remove", name, "eth0"], 15_000); + await incus([ + "config", "device", "add", name, "eth0", "nic", + "nictype=bridged", + `parent=${networkName(instanceId, segment.name)}`, + `hwaddr=${macFor(instanceId, "registry", 0)}`, + ]); + } + await succeeds(["start", name], 60_000); + await waitForAgent(name); + + // Address it by MAC, never by interface name: a machine with a container runtime has a + // `docker0` that sorts before `enp5s0`, and naive selection configures that instead — which + // then overlaps the segment and breaks routing on the machine. + const mac = macFor(instanceId, "registry", 0); + await incus(["exec", name, "--", "sh", "-c", + `dev=$(ip -o link | awk -F': ' '/${mac}/ {print $2}' | head -1); ` + + `[ -n "$dev" ] && ip addr add ${address}${prefix} dev "$dev" 2>/dev/null; ` + + `[ -n "$dev" ] && ip link set "$dev" up`], 60_000); + + log(` registry on ${segment.name} at ${address}`); + + // The registry's own image, placed by tag — an archive keeps a tag and cannot keep a digest, + // which is the whole reason this machine exists. + await placeImage(name, "registry", REGISTRY_IMAGE, () => {}); + + // The destination must EXIST before a recursive push, or incus copies the source's contents + // rather than the source — the data lands one directory too shallow, the registry finds + // nothing where it looks, and every pull fails with `not found`. + await incus(["exec", name, "--", "mkdir", "-p", "/srv/registry"], 60_000); + await incus(["file", "push", "-r", `${stock.dataDir}/docker`, `${name}/srv/registry/`], 900_000); + + await incus(["exec", name, "--", "docker", "run", "-d", + "--name", "registry", "--restart", "unless-stopped", + "-p", `${REGISTRY_PORT}:5000`, + "-v", "/srv/registry:/var/lib/registry", + REGISTRY_IMAGE], 300_000); + + // Read back that each image is SERVED, by asking for its manifest by digest — which is + // exactly what a machine will do. + // + // Not that the catalog endpoint answers: `{"repositories":[]}` contains the word + // `repositories`, so checking for that passed on a registry holding nothing at all, and the + // failure surfaced much later as a container that could not be pulled. + let answered = false; + for (let i = 0; i < 20 && !answered; i++) { + const ping = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null", + "-w", "%{http_code}", "--max-time", "3", + `http://localhost:${REGISTRY_PORT}/v2/`], 30_000); + answered = ping?.trim() === "200"; + if (!answered) await new Promise((r) => setTimeout(r, 2_000)); + } + if (!answered) { + throw new RegistryError( + `the registry on ${name} started and never answered. Machines in this scenario cannot ` + + `fetch an image, so nothing that declares a container will work.`, + ); + } + + const pinned: string[] = []; + for (const image of stock.images) { + const code = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null", + "-w", "%{http_code}", "--max-time", "5", + "-H", "Accept: application/vnd.docker.distribution.manifest.v2+json", + `http://localhost:${REGISTRY_PORT}/v2/${image.repository}/manifests/${image.digest}`, + ], 60_000); + if (code?.trim() !== "200") { + throw new RegistryError( + `the registry on ${name} is running and does not serve ${image.repository}@${image.digest} ` + + `(it answered ${code?.trim() || "nothing"}).\n` + + ` The images were stocked on this workstation and did not arrive intact, so a ` + + `machine declaring that image would fail to pull it.`, + ); + } + const reference = pinnedReference(address, image); + pinned.push(reference); + log(` serving ${reference}`); + } + return { machine: name, segment: segment.name, address, pinned }; +} + +async function waitForAgent(name: string): Promise { + for (let i = 0; i < 90; i++) { + if (await succeeds(["exec", name, "--", "true"], 10_000)) return; + await new Promise((r) => setTimeout(r, 2_000)); + } + throw new RegistryError(`${name} started and its agent never answered.`); +}