From d6eef25590f788e086a8f1dc3399d75389cd0557 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 28 Aug 2026 01:56:32 +0200 Subject: [PATCH] The lab can give a sealed machine a container runtime ADR 0046's open consequence: "the lab needs a way to place images, and the machine it places them into needs a container runtime, which a sealed scenario cannot install either." The runtime half is done, and it is research 012's reframing applied literally -- fetch at build time on a machine with a network, apply on a target that needs nothing. `mesh-lab base build` launches a machine WITH a network, installs a runtime, verifies it by asking the runtime rather than the package manager, and publishes the result. Measured: ~30s to install, ~60s to publish, ~700MiB, paid once per lab rather than per scenario. A scenario that places `runtime` or an image is then raised from that base image, chosen rather than declared -- a scenario says what it needs, not which image provides it. If the base does not exist it says so and how to build it. Verified in a genuinely sealed machine (no route out, confirmed by ping): package, service including the new `boot: enabled`, and action all applied, were idempotent on a second run, and read back correctly. Those three had never run anywhere but a workstation. The image half is NOT done, and testing found why: a digest-pinned image cannot be placed from an archive. `docker save alpine@sha256:...` produces an archive with no repo tag, because a repo digest only exists for an image a registry served -- so it loads dangling and a container declaring that digest reaches for a registry the machine cannot see. That collides with ADR 0046, which has the host REFUSE an unpinned image. Tag refused by the host, digest unusable in the lab: there is currently no declaration the lab can raise that exercises the container shape at all. Filed as 04-ISSUES/009, whose resolution is a registry inside the scenario -- which is what the real mesh does rather than a workaround for the lab. Also fixed a weak check of my own, which is the same fault in miniature: the load was tested with `includes("Loaded image")`, a prefix of both `Loaded image:` and `Loaded image ID:`. So an unusable dangling load reported success and the failure surfaced later as a container that would not start. --- src/cli.ts | 10 ++ src/lifecycle/base.ts | 113 +++++++++++++++++++++ src/lifecycle/place.ts | 195 ++++++++++++++++++++++++++++++++++++- src/lifecycle/raise.ts | 18 +++- src/lifecycle/supported.ts | 14 +-- test/place.test.ts | 44 ++++++++- 6 files changed, 379 insertions(+), 15 deletions(-) create mode 100644 src/lifecycle/base.ts diff --git a/src/cli.ts b/src/cli.ts index 2cb9eaa..be96ec1 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -91,6 +91,16 @@ async function main(): Promise { case "check": return check(); + case "base": { + // `base build` exists because a sealed scenario cannot install a container runtime, and + // the runtime has to come from somewhere with a network (novox/hq ADR 0046). + if (rest[0] !== "build") fail("base needs a subcommand: build"); + const { buildBaseImage } = await import("./lifecycle/base.ts"); + const built = await buildBaseImage((line) => console.log(line)); + console.log(`${built.alias}: built, with docker ${built.runtime}`); + return; + } + case "validate": { const path = rest[0] ?? fail("validate needs a scenario file"); const scenario = loadScenario(path); diff --git a/src/lifecycle/base.ts b/src/lifecycle/base.ts new file mode 100644 index 0000000..168b24c --- /dev/null +++ b/src/lifecycle/base.ts @@ -0,0 +1,113 @@ +/** + * Building the base image a scenario's machines are raised from. + * + * A sealed scenario cannot install a container runtime: its segments use documentation ranges + * and there is no route out (novox/hq ADR 0032). ADR 0046 records the consequence — *the lab + * needs a way to place images, and the machine it places them into needs a container runtime, + * which a sealed scenario cannot install either.* + * + * This is that, and it is research 012's reframing applied literally: **fetch at build time on + * a machine that has a network, apply on a target that then needs nothing.** The build happens + * here, once per lab, on a machine with a network. What a scenario raises afterwards needs + * neither. + * + * Measured while writing it: installing the runtime takes about 30 seconds, publishing about a + * minute, and the result is roughly 700 MiB. + */ + +import { incus, incusOk, succeeds } from "../incus/client.ts"; +import { BASE_IMAGE_ALIAS } from "./place.ts"; + +/** The stock image the base is built FROM. */ +export const UPSTREAM_IMAGE = "images:archlinux/current"; + +const BUILDER = "mesh-lab-base-builder"; + +export class BaseImageError extends Error { + constructor(message: string) { + super(message); + this.name = "BaseImageError"; + } +} + +/** + * Build the base image, replacing any previous one. + * + * Every step is read back. A published image that turns out not to have a working runtime is + * worse than no image, because every scenario raised from it fails somewhere else. + */ +export async function buildBaseImage( + log: (message: string) => void = () => {}, +): Promise<{ alias: string; runtime: string }> { + await succeeds(["delete", "-f", BUILDER], 120_000); + + log(` launching ${BUILDER} from ${UPSTREAM_IMAGE}, with a network`); + await incus([ + "launch", UPSTREAM_IMAGE, BUILDER, "--vm", + "-c", "security.secureboot=false", + "-c", "limits.memory=2GiB", + "-c", "limits.cpu=2", + ], 300_000); + + try { + await waitForAgent(BUILDER); + + log(" installing a container runtime"); + await incus(["exec", BUILDER, "--", "pacman", "-Sy", "--noconfirm", "docker"], 600_000); + await incus(["exec", BUILDER, "--", "systemctl", "enable", "docker"], 60_000); + await incus(["exec", BUILDER, "--", "systemctl", "start", "docker"], 120_000); + + // Read back from the runtime, not from the package manager. An installed package is not a + // capability (novox/hq 04-ISSUES/007), and this is the one place to catch that — after + // publishing, every scenario pays for it instead. + const runtime = (await incusOk( + ["exec", BUILDER, "--", "docker", "info", "--format", "{{.ServerVersion}}"], + 120_000, + ))?.trim(); + if (!runtime) { + throw new BaseImageError( + `the runtime was installed in ${BUILDER} and does not answer. Publishing this would ` + + `give every scenario an image that looks right and is not.`, + ); + } + log(` runtime works (docker ${runtime})`); + + log(" publishing"); + await incus(["stop", BUILDER, "--timeout", "120"], 300_000); + await incus(["publish", BUILDER, "--alias", BASE_IMAGE_ALIAS, "--reuse"], 900_000); + + const listed = await incusOk(["image", "list", BASE_IMAGE_ALIAS, "--format", "csv", "-c", "l"], 60_000); + if (!listed?.includes(BASE_IMAGE_ALIAS)) { + throw new BaseImageError( + `publishing reported success and '${BASE_IMAGE_ALIAS}' is not in the image list.`, + ); + } + + log(` published ${BASE_IMAGE_ALIAS}`); + return { alias: BASE_IMAGE_ALIAS, runtime }; + } finally { + // The builder is scaffolding. Leaving it standing would be a machine with a network in a + // lab whose whole point is that scenarios do not have one. + await succeeds(["delete", "-f", BUILDER], 120_000); + } +} + +/** Whether the base image exists, for a scenario to check before it raises. */ +export async function baseImageExists(): Promise { + const listed = await incusOk( + ["image", "list", BASE_IMAGE_ALIAS, "--format", "csv", "-c", "l"], 30_000, + ); + return Boolean(listed?.includes(BASE_IMAGE_ALIAS)); +} + +/** + * Wait for the guest agent, because `launch` returning means the VM started, not that anything + * inside it will answer. + */ +async function waitForAgent(name: string): Promise { + for (let i = 0; i < 90; i++) { + if (await succeeds(["exec", name, "--", "true"], 10_000)) return; + await new Promise((r) => setTimeout(r, 2_000)); + } + throw new BaseImageError(`${name} started and its agent never answered.`); +} diff --git a/src/lifecycle/place.ts b/src/lifecycle/place.ts index 7d5d321..ea542c1 100644 --- a/src/lifecycle/place.ts +++ b/src/lifecycle/place.ts @@ -12,12 +12,31 @@ */ import type { Scenario } from "../declaration/types.ts"; +import { spawn } from "node:child_process"; +import { unlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { incus, incusOk } from "../incus/client.ts"; -/** What this stage can put inside a machine. */ -export const PLACEABLE = ["host"] as const; +/** + * What this stage can put inside a machine. + * + * `image:` is the one that takes an argument — `image:alpine@sha256:...` — because + * WHICH image is the point of placing one. + */ +export const PLACEABLE = ["host", "runtime"] as const; export type Placeable = (typeof PLACEABLE)[number]; +/** The prefix that marks a container image, and what follows it is passed through unchanged. */ +export const IMAGE_PREFIX = "image:"; + +/** Whether an artifact names something this stage can place. */ +export function isPlaceable(artifact: string): boolean { + return (PLACEABLE as readonly string[]).includes(artifact) || + (artifact.startsWith(IMAGE_PREFIX) && artifact.slice(IMAGE_PREFIX.length).trim() !== ""); +} + /** Where the host binary lives on a machine once placed. */ export const HOST_PATH = "/usr/local/bin/mesh-host"; @@ -134,8 +153,12 @@ export async function applyPlacements( const placements = planPlacements(scenario); if (placements.length === 0) return []; + // Only demanded when something actually needs it. A scenario placing a runtime and an image + // and no host is a legitimate thing to want, and asking it for a host binary would be + // refusing a scenario for not doing something it never said it would. + const wantsHost = placements.some((p) => p.artifacts.includes("host")); const binary = hostBinaryPath(); - if (!binary) { + if (wantsHost && !binary) { throw new Error( `this scenario places the host, and no host binary was given. Set ` + `MESH_LAB_HOST_BINARY to a built mesh-host. Guessing at a path would place ` + @@ -147,10 +170,172 @@ export async function applyPlacements( for (const { machine, artifacts } of placements) { const name = machineNames.get(machine); if (!name) continue; + + // Order within a machine is the order declared, because a scenario placing an image + // before a runtime means something different from the reverse and the lab should not + // silently reorder it into working. for (const artifact of artifacts) { - if (artifact !== "host") continue; // refused earlier; belt and braces - placed.push(await placeHost(name, machine, binary, log)); + if (artifact === "host") { + placed.push(await placeHost(name, machine, binary as string, log)); + } else if (artifact === "runtime") { + await placeRuntime(name, machine, log); + } else if (artifact.startsWith(IMAGE_PREFIX)) { + await placeImage(name, machine, artifact.slice(IMAGE_PREFIX.length).trim(), log); + } + // Anything else was refused by the validator; reaching here would be a validator bug. } } return placed; } + +/** + * The base image a scenario's machines are built from, when they need a container runtime. + * + * A sealed scenario cannot install one: its segments use documentation ranges and there is no + * route out (novox/hq ADR 0032). So the runtime arrives the way research 012 says everything + * awkward should — **fetched at build time on a machine that has a network, applied on a target + * that then needs nothing.** Building this image is that build time. + * + * Measured before this was written: an Arch VM installs docker in about 30 seconds, and + * publishing the result takes about a minute and yields roughly 700 MiB. That is a one-time + * cost paid once per lab, not per scenario. + */ +export const BASE_IMAGE_ALIAS = "mesh-lab/base"; + +/** How to build it, said in one place so an error can point at it. */ +export const BASE_IMAGE_HOWTO = + `Build it with: mesh-lab base build\n` + + ` It launches a machine WITH a network, installs a container runtime, and publishes the\n` + + ` result as '${BASE_IMAGE_ALIAS}'. A scenario is sealed and cannot do that for itself.`; + +/** + * Confirm a machine has a working container runtime. + * + * Asks the runtime, never the filesystem. A binary being present is not a runtime working — + * that is 04-ISSUES/007, and it is the reason this runs `info` rather than checking a path. + * + * It does not INSTALL one. A sealed machine cannot fetch, so an absent runtime is a scenario + * built on the wrong image, and saying that is more useful than failing inside a package + * manager with no network. + */ +export async function placeRuntime( + instanceName: string, + machine: string, + log: (message: string) => void = () => {}, +): Promise { + const version = (await incusOk( + ["exec", instanceName, "--", "docker", "info", "--format", "{{.ServerVersion}}"], + 60_000, + ))?.trim(); + + if (!version) { + throw new PlacementError( + machine, + `${machine} has no working container runtime, and a sealed scenario cannot install one.\n` + + ` This machine needs to be built from '${BASE_IMAGE_ALIAS}'.\n` + + BASE_IMAGE_HOWTO, + ); + } + log(` ${machine} has a container runtime (docker ${version})`); + return version; +} + +/** + * Put a container image inside a machine. + * + * Exported from this workstation and loaded in the machine, because the machine cannot reach a + * registry. The reference is passed through unchanged — including its digest — so what runs in + * the lab is the image the declaration names rather than whatever a tag pointed at. + */ +export async function placeImage( + instanceName: string, + machine: string, + reference: string, + log: (message: string) => void = () => {}, +): Promise { + // A digest reference cannot be placed this way, and finding that out here is much better + // than finding it out when a container fails to start. + // + // `docker save alpine@sha256:...` produces an archive with NO repo tag — only an image ID — + // because a repo digest exists only for an image a registry served. Loading it gives a + // dangling image, so `docker run ` falls through to the registry, which a + // sealed machine cannot reach. Measured, not assumed: the load says `Loaded image ID:` + // instead of `Loaded image:`, and `docker images` then lists nothing. + // + // This collides with novox/hq ADR 0046, which pins bundle images BY DIGEST and has the host + // refuse anything else. Reconciling the two needs a registry inside the scenario, which is + // real design work — see 04-ISSUES/009. + if (reference.includes("@sha256:")) { + throw new PlacementError( + machine, + `${reference} is pinned by digest, and an image placed from an archive cannot keep its ` + + `digest — a repo digest only exists for an image a registry served.\n` + + ` Placing it would load an image with no name, and a container declaring that digest ` + + `would try to reach a registry the machine cannot see.\n` + + ` Place it by tag, or give the scenario a registry (novox/hq 04-ISSUES/009).`, + ); + } + + const tar = join(tmpdir(), `mesh-lab-image-${process.pid}-${Date.now()}.tar`); + + // Exported from whatever this workstation has. The lab places images; it does not fetch + // them, so an image nobody pulled here is an error rather than a download. + const saved = await local("docker", ["save", reference, "-o", tar], 600_000); + if (!saved.ok) { + throw new PlacementError( + machine, + `cannot export ${reference} from this workstation: ${saved.stderr.trim()}\n` + + ` The lab places images it already has, and never fetches on a scenario's behalf.\n` + + ` Pull it here first, then raise.`, + ); + } + + try { + await incus(["file", "push", tar, `${instanceName}/tmp/image.tar`], 900_000); + + // Read back what the runtime says, not that the push returned. A file arriving is not an + // image present, and `docker load` names what it loaded — so that is what is checked. + const loaded = await incusOk( + ["exec", instanceName, "--", "docker", "load", "-i", "/tmp/image.tar"], + 600_000, + ); + // `Loaded image:` and `Loaded image ID:` are different outcomes and only one is useful. + // Matching the shorter string accepted both, so an archive that loaded as a dangling + // image reported success — a check that passes on the wrong thing. + if (!loaded?.includes("Loaded image:")) { + throw new PlacementError( + machine, + `${reference} was pushed to ${machine} and did not arrive as a usable image.\n` + + ` The runtime said: ${loaded?.trim() || "nothing"}\n` + + ` 'Loaded image ID:' means it loaded with no name, which nothing can run by name.`, + ); + } + log(` placed ${reference} on ${machine}`); + } finally { + await unlink(tar).catch(() => {}); + } +} + +/** Run something on THIS workstation. The incus client only speaks to incus. */ +function local( + command: string, + args: string[], + timeoutMs: number, +): Promise<{ ok: boolean; stdout: string; stderr: string }> { + return new Promise((resolve) => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs); + child.stdout.on("data", (d) => (stdout += d)); + child.stderr.on("data", (d) => (stderr += d)); + child.on("error", (err) => { + clearTimeout(timer); + resolve({ ok: false, stdout, stderr: err.message }); + }); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ ok: code === 0, stdout, stderr }); + }); + }); +} diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index 5f8d517..a7888c4 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -22,7 +22,8 @@ import { applyAddresses, applyDefaultRoutes } from "./address.ts"; import { assertSupported } from "./supported.ts"; import { planRouters, raiseRouters, raiseTransit } from "./router.ts"; import { applyHostFirewalls } from "./firewall.ts"; -import { applyPlacements } from "./place.ts"; +import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts"; +import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts"; /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */ const COW_DRIVERS = ["btrfs", "zfs"]; @@ -166,7 +167,20 @@ export async function raise( options: RaiseOptions = {}, ): Promise { const log = options.onProgress ?? (() => {}); - const image = options.image ?? "images:archlinux/current"; + + // A scenario that places a runtime or an image needs machines built from the base image, + // because a sealed machine cannot install one (novox/hq ADR 0046). Chosen here rather than + // declared, so a scenario says WHAT it needs and not which image provides it. + const needsRuntime = planPlacements(scenario).some(({ artifacts }) => + artifacts.some((a) => a === "runtime" || a.startsWith(IMAGE_PREFIX)) + ); + if (needsRuntime && !options.image && !(await baseImageExists())) { + throw new Error( + `this scenario needs a container runtime inside its machines, and '${BASE_IMAGE_ALIAS}' ` + + `does not exist.\n${BASE_IMAGE_HOWTO}`, + ); + } + const image = options.image ?? (needsRuntime ? BASE_IMAGE_ALIAS : UPSTREAM_IMAGE); const readyTimeout = options.readyTimeoutSeconds ?? 180; const instanceId = options.instanceId ?? newInstanceId(scenario.scenario, new Date()); diff --git a/src/lifecycle/supported.ts b/src/lifecycle/supported.ts index c78d132..55b07d8 100644 --- a/src/lifecycle/supported.ts +++ b/src/lifecycle/supported.ts @@ -14,7 +14,7 @@ */ import type { Scenario } from "../declaration/types.ts"; -import { PLACEABLE, planPlacements } from "./place.ts"; +import { IMAGE_PREFIX, isPlaceable, PLACEABLE, planPlacements } from "./place.ts"; export class UnsupportedError extends Error { readonly missing: string[]; @@ -34,19 +34,19 @@ export class UnsupportedError extends Error { export function assertSupported(scenario: Scenario): void { const missing: string[] = []; - // `place: [host]` works. Everything else in the vocabulary is named individually rather - // than refused as a whole, so a scenario that places a host and a substrate is told exactly - // which half the lab cannot do. + // `host`, `runtime` and `image:` work. Everything else in the vocabulary is named + // individually rather than refused as a whole, so a scenario that places a host and a + // substrate is told exactly which half the lab cannot do. const unplaceable = new Set(); for (const { artifacts } of planPlacements(scenario)) { for (const artifact of artifacts) { - if (!(PLACEABLE as readonly string[]).includes(artifact)) unplaceable.add(artifact); + if (!isPlaceable(artifact)) unplaceable.add(artifact); } } for (const artifact of [...unplaceable].sort()) { missing.push( - `place: ${artifact} — only ${PLACEABLE.join(", ")} can be placed; the tiers above ` + - `tier 0 do not exist yet`, + `place: ${artifact} — only ${PLACEABLE.join(", ")} and ${IMAGE_PREFIX} can ` + + `be placed; the tiers above tier 0 do not exist yet`, ); } diff --git a/test/place.test.ts b/test/place.test.ts index 29cb3d0..acb5bc6 100644 --- a/test/place.test.ts +++ b/test/place.test.ts @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { parseScenario } from "../src/declaration/parse.ts"; -import { planPlacements, PLACEABLE } from "../src/lifecycle/place.ts"; +import { planPlacements, PLACEABLE, isPlaceable } from "../src/lifecycle/place.ts"; import { assertSupported, UnsupportedError } from "../src/lifecycle/supported.ts"; /** @@ -87,3 +87,45 @@ test("the refusal says what CAN be placed", () => { } } }); + +// --- runtime and image placement (novox/hq ADR 0046: the lab places what a sealed scenario +// cannot fetch) --- + +test("an image reference is placeable, and a bare 'image:' is not", () => { + assert.ok(isPlaceable("image:alpine@sha256:abc"), "a reference should be placeable"); + assert.ok(isPlaceable("image:postgres:17"), "a tag is the scenario's business, not the lab's"); + assert.ok(!isPlaceable("image:"), "there is nothing to place"); + assert.ok(!isPlaceable("image: "), "whitespace is not a reference"); +}); + +test("the placeables are host, runtime and an image", () => { + assert.ok(isPlaceable("host")); + assert.ok(isPlaceable("runtime")); + assert.ok(!isPlaceable("substrate"), "the tiers above tier 0 do not exist yet"); + assert.ok(!isPlaceable("control-plane")); +}); + +test("an unplaceable artifact is named, not refused as a whole", () => { + // A scenario placing a host and a substrate is told which half the lab cannot do — refusing + // wholesale would send somebody looking for the wrong problem. + const scenario = { + name: "s", + segments: {}, + machines: { a: {} as never }, + place: { a: ["host", "runtime", "image:alpine@sha256:x", "substrate"] }, + } as unknown as Parameters[0]; + + assert.throws( + () => assertSupported(scenario), + (err: Error) => { + // Checked as `place: —`, which is how an artifact is REPORTED as + // unplaceable. Searching for the bare word matched the message's own list of what CAN + // be placed, which mentions runtime — so the test failed on a correct message. + assert.ok(err.message.includes("place: substrate —"), "the unplaceable one is named"); + assert.ok(!err.message.includes("place: image:alpine"), "a placeable one is not"); + assert.ok(!err.message.includes("place: runtime —"), "nor is runtime"); + assert.ok(!err.message.includes("place: host —"), "nor is host"); + return true; + }, + ); +});