Merge pull request 'The run rebuilds the module runtimes its beds stock (075); two mechanism beds read the catalogue's redis (074)' (#43) from feat/mesh-tests-and-runtimes into main

README.md

@@ -165,10 +165,15 @@ export MESH_LAB_ROUTE_PROXY=<somewhere>/route-proxy
 
 `MESH_LAB_HOST_BINARY` and `MESH_LAB_MODULES` do double duty: the repository each sits in is what
 `suite` rebuilds and what the receipt claims. Point the run at a repository and it is built and
-claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built: a bed
-installs a catalogue module by reading its manifest from that checkout when it runs, so the
-receipt names the catalogue's commit too, and a run taken before a manifest changed says so
-(novox/hq 04-ISSUES/073).
+claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built as a
+repository: a bed installs a catalogue module by reading its manifest from that checkout when it
+runs, so the receipt names the catalogue's commit too, and a run taken before a manifest changed
+says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds'
+scenarios stock (`mesh-runtime-<module>:development`): each is compared against the module's
+source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts
+beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh`
+where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075).
+`--no-build` skips this too, and then the bed runs whatever image the store holds.
 
 Check before running a long suite — it says which of these are missing rather than skipping
 quietly:

scenarios/redis-node.yml

@@ -24,6 +24,9 @@ images:
   # Redis's tool+provisioner runtime, built by scripts/build-module-runtime.sh redis into the local
   # daemon and loaded onto the machine, which holds it by its own image ID.
   - mesh-runtime-redis:development
+  # The vault's, for the beds that install the catalogue's redis: its own password is a secret the
+  # vault provides (novox/hq ADR 0085).
+  - mesh-runtime-mesh-vault:development
 
 place:
   all: [host, runtime]

src/rebuild.ts

@@ -16,6 +16,7 @@
 
 import { spawnSync } from "node:child_process";
 import { repositories } from "./repos.ts";
+import { plannedRuntimes } from "./runtimes.ts";
 
 export interface Build {
   /** What it produces, for the log. */
@@ -112,10 +113,16 @@ export function carriedImage(env: NodeJS.ProcessEnv = process.env): string {
   return env["MESH_LAB_CARRIED_IMAGE"] ?? "mesh-builder:development";
 }
 
-/** rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. */
-export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] {
+/**
+ * rebuild runs the plan, and throws on the first failure rather than testing a stale artifact.
+ *
+ * The plan is what the run was pointed at, plus the module runtimes the named beds stock where
+ * those are older than their source (novox/hq 04-ISSUES/075) — so a bed never again passes against
+ * a runtime built two weeks before the manifest it serves.
+ */
+export function rebuild(env: NodeJS.ProcessEnv = process.env, testFiles: string[] = []): string[] {
   const built: string[] = [];
-  for (const build of planned(env)) {
+  for (const build of [...planned(env), ...plannedRuntimes(testFiles, env)]) {
     const [command, ...args] = build.argv;
     const ran = spawnSync(command!, args, {
       cwd: build.in,

src/runtimes.ts

@@ -0,0 +1,144 @@
+/**
+ * The module runtimes a run stocks are rebuilt by the run, like everything else it tests.
+ *
+ * A per-module bed stocks the module's runtime image — the tool runtime carrying that module's
+ * code — from the workstation's image store, by tag. It was built by hand, by a script the suite
+ * never called, and on the day this was written the images for six modules about to run dated
+ * from two weeks before the manifests they were installed with (novox/hq 04-ISSUES/075). The
+ * suite's own rule, *the run rebuilds what it tests*, was held for the host and the control plane
+ * and not for these.
+ *
+ * So: for the beds about to run, every `mesh-runtime-<module>:development` their scenarios stock
+ * is compared against the source it is built from — the module in the catalogue, and the tool
+ * runtime and SDK it is built on — and rebuilt where the image is older, missing, or the source is
+ * uncommitted. A rebuild that fails stops the suite, the way a stale host binary would.
+ */
+
+import { readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { spawnSync } from "node:child_process";
+import { parse } from "yaml";
+import type { Build } from "./rebuild.ts";
+import { catalogueRoot } from "./repos.ts";
+
+/** A runtime image a scenario stocks by tag, and the module it is built from. */
+export interface StockedRuntime {
+  tag: string;
+  module: string;
+}
+
+/**
+ * Tags whose name is not the module's. The audit logger's runtime was the first, built before the
+ * script was generalised, and the scenario still stocks it under the module's slug.
+ */
+const NAMED_OTHERWISE: Record<string, string> = { "mesh-runtime-audit": "audit-logger" };
+
+const RUNTIME_TAG = /^(mesh-runtime-[a-z0-9-]+):development$/;
+
+/** The runtimes the scenarios of these beds stock, each named once. */
+export function runtimesStockedBy(testFiles: string[], root: string = process.cwd()): StockedRuntime[] {
+  const seen = new Map<string, StockedRuntime>();
+  for (const file of testFiles) {
+    const text = readFileSync(resolve(root, file), "utf8");
+    const named = /const SCENARIO = "([^"]+)"/.exec(text);
+    if (!named) continue;
+    const scenario = parse(readFileSync(join(root, "scenarios", `${named[1]}.yml`), "utf8")) as { images?: unknown };
+    for (const image of Array.isArray(scenario.images) ? scenario.images : []) {
+      const m = RUNTIME_TAG.exec(String(image));
+      if (!m) continue;
+      const repository = m[1]!;
+      seen.set(repository, { tag: String(image), module: NAMED_OTHERWISE[repository] ?? repository.slice("mesh-runtime-".length) });
+    }
+  }
+  return [...seen.values()];
+}
+
+/** When an image was made and when its source last changed, in seconds; null for no image. */
+export interface Ages {
+  image: number | null;
+  /** Infinity where the source has uncommitted changes: what is on disk is newer than any commit. */
+  source: number;
+}
+
+/** stale is whether the image predates its source, or is not there at all. */
+export function isStale(a: Ages): boolean {
+  return a.image === null || a.image < a.source;
+}
+
+/** A command runner, for the two questions asked below; injected so the rules can be tested. */
+export type Ask = (command: string, args: string[]) => { status: number | null; stdout: string };
+
+const ask: Ask = (command, args) => {
+  const ran = spawnSync(command, args, { encoding: "utf8" });
+  return { status: ran.status, stdout: ran.stdout ?? "" };
+};
+
+/** ageOfImage is when the local image store made this tag, or null when it holds no such image. */
+export function ageOfImage(tag: string, run: Ask = ask): number | null {
+  const ran = run("docker", ["image", "inspect", "--format", "{{.Created}}", tag]);
+  if (ran.status !== 0) return null;
+  const at = Date.parse(ran.stdout.trim());
+  return Number.isNaN(at) ? null : Math.floor(at / 1000);
+}
+
+/**
+ * ageOfSource is the newest commit touching what the runtime is built from: the module's directory
+ * in the catalogue, and the whole of each repository it is built on top of. Uncommitted changes in
+ * any of them are newer than every commit.
+ */
+export function ageOfSource(catalogue: string, module: string, builtOn: string[], run: Ask = ask): number {
+  let newest = 0;
+  const at = (dir: string, path?: string): number => {
+    const args = ["-C", dir, "log", "-1", "--format=%ct"];
+    if (path) args.push("--", path);
+    const ran = run("git", args);
+    const t = Number.parseInt(ran.stdout.trim(), 10);
+    return ran.status === 0 && Number.isFinite(t) ? t : 0;
+  };
+  const dirty = (dir: string, path?: string): boolean => {
+    const args = ["-C", dir, "status", "--porcelain"];
+    if (path) args.push("--", path);
+    const ran = run("git", args);
+    return ran.status === 0 && ran.stdout.trim() !== "";
+  };
+  if (dirty(catalogue, `modules/${module}`)) return Infinity;
+  newest = Math.max(newest, at(catalogue, `modules/${module}`));
+  for (const dir of builtOn) {
+    if (dirty(dir)) return Infinity;
+    newest = Math.max(newest, at(dir));
+  }
+  return newest;
+}
+
+/**
+ * plannedRuntimes is the runtime builds these beds need before they run, given where the run was
+ * pointed: nothing where no catalogue was named (the beds skip), one build per stale image
+ * otherwise. The repositories the runtime is built on are the siblings the build script itself
+ * reads (`MESH_TOOLS`, `MESH_SDK`, or the checkouts beside this one).
+ */
+export function plannedRuntimes(testFiles: string[], env: NodeJS.ProcessEnv = process.env,
+  root: string = process.cwd(), run: Ask = ask): Build[] {
+  const named = env["MESH_LAB_CATALOG"];
+  if (!named) return [];
+  const catalogue = catalogueRoot(named);
+  const builtOn = [
+    env["MESH_TOOLS"] ?? resolve(root, "..", "mesh-tools"),
+    env["MESH_SDK"] ?? resolve(root, "..", "mesh-sdk"),
+  ];
+  const builds: Build[] = [];
+  for (const runtime of runtimesStockedBy(testFiles, root)) {
+    const ages: Ages = {
+      image: ageOfImage(runtime.tag, run),
+      source: ageOfSource(catalogue, runtime.module, builtOn, run),
+    };
+    if (!isStale(ages)) continue;
+    builds.push({
+      what: `runtime ${runtime.tag}`,
+      in: root,
+      argv: ["scripts/build-module-runtime.sh", runtime.module, join(tmpdir(), `mesh-lab-${runtime.module}.tar`)],
+      env: { MESH_CATALOG: catalogue, RUNTIME_TAG: runtime.tag },
+    });
+  }
+  return builds;
+}

src/suite.ts

@@ -40,7 +40,7 @@ export async function runSuite(args: string[]): Promise<number> {
   if (!args.includes("--no-build")) {
     // Before the run, always. The artifacts are built from two other repositories, and a suite
     // that tests yesterday's binary reports on code nobody is looking at (novox/hq 04-ISSUES/005).
-    const built = rebuild();
+    const built = rebuild(process.env, files);
     if (built.length > 0) console.log(`built: ${built.join(", ")}\n`);
   }
   // Read now, while it is true. The receipt names these, and reading them when the run ends

test/beds-read-the-catalogue.test.ts

@@ -34,8 +34,11 @@ import { catalogueIsPresent, catalogueDir } from "./integration/harness.ts";
  *   WEARING  the bed proves a mesh mechanism (a grant, a credential, a restart, a route) with a
  *            module cut down to the shape the mechanism needs — no upstream server, a secret in the
  *            environment, a requirement edge removed — and gives it a catalogue name. It is a mesh
- *            test wearing a catalogue module's name. It should carry a name of its own, or read the
- *            catalogue and meet the module's real requirements.
+ *            test wearing a catalogue module's name. It cannot simply be renamed: a module's name
+ *            is its tool namespace and its broker scope, so a fixture running the module's runtime
+ *            must carry the module's name (novox/hq ADR 0093). It reads the catalogue and installs
+ *            what the module requires — the vault for a secret, the route module for a route — or
+ *            it runs no real runtime and carries a name of its own.
  *   DIFFERS  a module bed whose copy differs from the catalogue in more than the lab may rewrite
  *            (an image, a port, an address). Reading the catalogue is the fix and needs a run.
  */
@@ -55,10 +58,8 @@ const STILL_CARRIED: Record<string, { modules: string[]; why: string }> = {
   "assigned-plex.test.ts": { modules: ["plex"], why: "WEARING: the sidecar alone, no Plex, the token in the environment" },
   "assigned-redis.test.ts": { modules: ["redis"], why: "WEARING: its own secret, a lab seal key in the environment" },
   "assigned-sonarr.test.ts": { modules: ["sonarr"], why: "WEARING: the sidecar alone against a forged config.xml" },
-  "mesh-grant-end-to-end.test.ts": { modules: ["redis"], why: "WEARING: a grant mechanism test" },
   "minio-grant-end-to-end.test.ts": { modules: ["minio"], why: "WEARING: a grant mechanism test, the root password by env-file" },
   "postgres-grant-end-to-end.test.ts": { modules: ["postgres"], why: "WEARING: a grant mechanism test, the superuser by env-file" },
-  "provider-on-backend-network.test.ts": { modules: ["redis"], why: "WEARING: a network mechanism test" },
   "provider-uses-mesh-credential.test.ts": { modules: ["redis"], why: "WEARING: a credential mechanism test" },
   "runtime-restart-on-config.test.ts": { modules: ["grafana"], why: "WEARING: a restart mechanism test" },
   "route-forwarding.test.ts": { modules: ["route-proxy", "hello-web"],

test/integration/mesh-grant-end-to-end.test.ts

@@ -23,7 +23,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
 import { raise } from "../../src/lifecycle/raise.ts";
 import { destroy, exec } from "../../src/lifecycle/operate.ts";
 import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
-import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
+import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
 import type { HeldImage } from "../../src/pinning.ts";
 
 const capability = await labIsUsable();
@@ -36,7 +36,7 @@ const skip = !capability.usable
     ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
     : !bundle || !existsSync(bundle)
       ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
-      : false;
+      : catalogueIsPresent();
 
 const SCENARIO = "redis-node";
 const MACHINE = "anchor";
@@ -139,51 +139,12 @@ after(async () => {
 test("the mesh grants a consumer redis's cache, and the credential it delivers authenticates", {
   skip, timeout: 900_000,
 }, async () => {
-  // The PROVIDER: redis in its committed shape — server and a broker-bound runtime on the private
-  // redis network, the runtime running the provisioner.
-  const redisManifest = JSON.stringify({
-    module: "redis",
-    version: "1",
-    provides: [{ name: "redis-cache", scope: "mesh" }],
-    serves: { "redis-cache": {} },
-    emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
-    consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
-    receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
-    grants: { "redis-cache": "/var/lib/redis-module/grants" },
-    "own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
-    resources: [
-      { id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
-      { id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
-      { id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
-      { id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
-      {
-        id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
-        content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
-      },
-      { id: "net", type: "network", name: "redis" },
-      {
-        id: "server", type: "container", name: "redis", image: pinned("redis"), network: "redis",
-        ports: ["6379"],
-        volumes: ["/services/redis/data:/data", "/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro"],
-        args: ["/etc/redis/redis.conf"],
-      },
-      {
-        id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
-        network: "redis",
-        volumes: [
-          "/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
-          "/var/lib/redis-module/grants:/var/lib/redis-module/grants:ro",
-          "/var/lib/redis-module/default.secret:/run/secrets/default:ro",
-        ],
-        env: {
-          MESH_BROKER_FILE: "/run/secrets/broker",
-          MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
-          MESH_PROVISION_REDIS: "redis:6379",
-          MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
-        },
-      },
-    ],
-  });
+  // The PROVIDER: the catalogue's redis (novox/hq 04-ISSUES/074) — server and a broker-bound
+  // runtime on the private redis network, the runtime running the provisioner. It requires a
+  // `secret` for its own password, so the vault that provides one is installed beside it, exactly
+  // as the vault bed does.
+  const vaultManifest = catalogueModule("mesh-vault", held);
+  const redisManifest = catalogueModule("redis", held);
 
   // The CONSUMER: a module that requires redis-cache and no more. It runs no code here — the mesh
   // delivers it a bound file (where redis is, and the login to present) and its sealed password,
@@ -191,6 +152,9 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
   const consumerManifest = JSON.stringify({
     module: "cacheuser",
     version: "1",
+    // `mesh_anchor_cacheuser` is 21 characters, over the 20 a backend keeps (ADR 0049); the slug
+    // makes the consumer identity `mesh_anchor_cache`.
+    slug: "cache",
     requires: ["redis-cache"],
     // `contributes` (not just `requires`) is what makes a consumer *ask* — the grant forms from a
     // contribution. It must be non-empty; redis's provisioner ignores the value (it uses the login
@@ -201,6 +165,10 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
     resources: [{ id: "state", type: "directory", path: "/var/lib/cacheuser", mode: "0700" }],
   });
 
+  await must(`printf %s ${quote(vaultManifest)} > /tmp/mesh-vault.json && docker cp /tmp/mesh-vault.json mesh-controller:/mesh-vault.json`);
+  await mesh("module add /mesh-vault.json");
+  await mesh(`module issue mesh-vault --node ${MACHINE}`);
+  await mesh(`assign ${MACHINE} mesh-vault`);
   await must(`printf %s ${quote(redisManifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
   await mesh("module add /redis.json");
   await mesh(`module issue redis --node ${MACHINE}`);

test/integration/provider-on-backend-network.test.ts

@@ -20,7 +20,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
 import { raise } from "../../src/lifecycle/raise.ts";
 import { destroy, exec } from "../../src/lifecycle/operate.ts";
 import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
-import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
+import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
 import type { HeldImage } from "../../src/pinning.ts";
 
 const capability = await labIsUsable();
@@ -33,7 +33,7 @@ const skip = !capability.usable
     ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
     : !bundle || !existsSync(bundle)
       ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
-      : false;
+      : catalogueIsPresent();
 
 const SCENARIO = "redis-node";
 const MACHINE = "anchor";
@@ -136,54 +136,15 @@ after(async () => {
 test("redis's runtime, on the backend's private network, binds the broker and provisions with the mesh's credential", {
   skip, timeout: 900_000,
 }, async () => {
-  // Exactly the committed redis shape: a private `redis` network, the server on it with a published
-  // port, and the runtime on it too — reaching redis by name and the broker by NAT.
-  const manifest = JSON.stringify({
-    module: "redis",
-    version: "1",
-    provides: [{ name: "redis-cache", scope: "mesh" }],
-    serves: { "redis-cache": {} },
-    emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
-    consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
-    receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
-    grants: { "redis-cache": "/var/lib/redis-module/grants" },
-    "own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
-    resources: [
-      { id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
-      { id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
-      { id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
-      { id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
-      {
-        id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
-        content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
-      },
-      { id: "net", type: "network", name: "redis" },
-      {
-        id: "server", type: "container", name: "redis", image: pinned("redis"), network: "redis",
-        ports: ["6379"],
-        volumes: [
-          "/services/redis/data:/data",
-          "/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
-        ],
-        args: ["/etc/redis/redis.conf"],
-      },
-      {
-        id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
-        network: "redis",
-        volumes: [
-          "/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
-          "/var/lib/redis-module/grants:/var/lib/redis-module/grants:ro",
-          "/var/lib/redis-module/default.secret:/run/secrets/default:ro",
-        ],
-        env: {
-          MESH_BROKER_FILE: "/run/secrets/broker",
-          MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
-          MESH_PROVISION_REDIS: "redis:6379",
-          MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
-        },
-      },
-    ],
-  });
+  // The catalogue's redis (novox/hq 04-ISSUES/074): a private `redis` network, the server on it
+  // with a published port, and the runtime on it too — reaching redis by name and the broker by
+  // NAT. Its own password is a `secret` the vault provides, so the vault is installed beside it.
+  const vaultManifest = catalogueModule("mesh-vault", held);
+  await must(`printf %s ${quote(vaultManifest)} > /tmp/mesh-vault.json && docker cp /tmp/mesh-vault.json mesh-controller:/mesh-vault.json`);
+  await mesh("module add /mesh-vault.json");
+  await mesh(`module issue mesh-vault --node ${MACHINE}`);
+  await mesh(`assign ${MACHINE} mesh-vault`);
+  const manifest = catalogueModule("redis", held);
   await must(`printf %s ${quote(manifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
   await mesh("module add /redis.json");
   await mesh(`module issue redis --node ${MACHINE}`);

test/runtimes.test.ts

@@ -0,0 +1,88 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { ageOfImage, ageOfSource, isStale, plannedRuntimes, runtimesStockedBy, type Ask } from "../src/runtimes.ts";
+
+// The module runtimes a run stocks are rebuilt by the run (novox/hq 04-ISSUES/075): what a bed's
+// scenario stocks is found, compared against its source, and built where older.
+
+function aLabWith(beds: Record<string, string>, scenarios: Record<string, string[]>): { root: string; done: () => void } {
+  const root = mkdtempSync(join(tmpdir(), "mesh-lab-runtimes-"));
+  mkdirSync(join(root, "scenarios"));
+  mkdirSync(join(root, "test", "integration"), { recursive: true });
+  for (const [name, images] of Object.entries(scenarios)) {
+    writeFileSync(join(root, "scenarios", `${name}.yml`), `scenario: ${name}\nimages:\n${images.map((i) => `  - ${i}\n`).join("")}`);
+  }
+  for (const [file, scenario] of Object.entries(beds)) {
+    writeFileSync(join(root, "test", "integration", file), `const SCENARIO = "${scenario}";\n`);
+  }
+  return { root, done: () => rmSync(root, { recursive: true, force: true }) };
+}
+
+test("what a bed's scenario stocks is found, by module, once", () => {
+  const lab = aLabWith(
+    { "a.test.ts": "one", "b.test.ts": "two", "c.test.ts": "one" },
+    { one: ["mesh-controller:development", "mesh-runtime-gitlab:development", "postgres:16"],
+      two: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
+  try {
+    const found = runtimesStockedBy(["test/integration/a.test.ts", "test/integration/b.test.ts", "test/integration/c.test.ts"], lab.root);
+    assert.deepEqual(found, [
+      { tag: "mesh-runtime-gitlab:development", module: "gitlab" },
+      // The audit logger's runtime is stocked under its slug, and the module is named for it.
+      { tag: "mesh-runtime-audit:development", module: "audit-logger" },
+    ]);
+  } finally { lab.done(); }
+});
+
+test("an image is stale when missing, older than its source, or when the source is uncommitted", () => {
+  assert.equal(isStale({ image: null, source: 0 }), true);
+  assert.equal(isStale({ image: 100, source: 200 }), true);
+  assert.equal(isStale({ image: 200, source: 100 }), false);
+  assert.equal(isStale({ image: 200, source: Infinity }), true);
+});
+
+test("the image's age comes from the store and the source's from the newest commit in any part", () => {
+  const answers: Ask = (command, args) => {
+    if (command === "docker") return { status: 0, stdout: "2026-09-21T12:00:00.000000000Z\n" };
+    if (args.includes("status")) return { status: 0, stdout: "" };
+    if (args.includes("modules/gitlab")) return { status: 0, stdout: "1000\n" };
+    return { status: 0, stdout: args[1] === "/tools" ? "3000\n" : "2000\n" };
+  };
+  assert.equal(ageOfImage("mesh-runtime-gitlab:development", answers), Date.parse("2026-09-21T12:00:00Z") / 1000);
+  assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], answers), 3000);
+  // No such image is null, not zero: "never built" and "built at the epoch" are different answers.
+  assert.equal(ageOfImage("mesh-runtime-nothing:development", () => ({ status: 1, stdout: "" })), null);
+  // Uncommitted changes anywhere in the source are newer than every commit.
+  const dirtyTools: Ask = (command, args) =>
+    args.includes("status") && args[1] === "/tools" ? { status: 0, stdout: " M src/x.ts\n" } : answers(command, args);
+  assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], dirtyTools), Infinity);
+});
+
+test("only a stale runtime is planned, built by the lab's own script under the tag the scenario stocks", () => {
+  const lab = aLabWith({ "a.test.ts": "one" },
+    { one: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
+  try {
+    const answers: Ask = (command, args) => {
+      if (command === "docker") {
+        // gitlab's image is from yesterday; the audit logger has none.
+        return args.includes("mesh-runtime-gitlab:development")
+          ? { status: 0, stdout: "2026-09-21T12:00:00Z\n" } : { status: 1, stdout: "" };
+      }
+      if (args.includes("status")) return { status: 0, stdout: "" };
+      return { status: 0, stdout: `${Date.parse("2026-09-20T00:00:00Z") / 1000}\n` };
+    };
+    const env = { MESH_LAB_CATALOG: "/catalogue/modules", MESH_TOOLS: "/tools", MESH_SDK: "/sdk" };
+    const builds = plannedRuntimes(["test/integration/a.test.ts"], env, lab.root, answers);
+    assert.equal(builds.length, 1);
+    assert.equal(builds[0]!.what, "runtime mesh-runtime-audit:development");
+    assert.equal(builds[0]!.argv[0], "scripts/build-module-runtime.sh");
+    assert.equal(builds[0]!.argv[1], "audit-logger");
+    assert.equal(builds[0]!.env?.["MESH_CATALOG"], "/catalogue");
+    assert.equal(builds[0]!.env?.["RUNTIME_TAG"], "mesh-runtime-audit:development");
+    // No catalogue named: nothing is planned, because no bed will read one either.
+    assert.deepEqual(plannedRuntimes(["test/integration/a.test.ts"], {}, lab.root, answers), []);
+  } finally { lab.done(); }
+});