README.md

@@ -165,10 +165,15 @@ export MESH_LAB_ROUTE_PROXY=<somewhere>/route-proxy
 
 `MESH_LAB_HOST_BINARY` and `MESH_LAB_MODULES` do double duty: the repository each sits in is what
 `suite` rebuilds and what the receipt claims. Point the run at a repository and it is built and
-claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built: a bed
-installs a catalogue module by reading its manifest from that checkout when it runs, so the
-receipt names the catalogue's commit too, and a run taken before a manifest changed says so
-(novox/hq 04-ISSUES/073).
+claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built as a
+repository: a bed installs a catalogue module by reading its manifest from that checkout when it
+runs, so the receipt names the catalogue's commit too, and a run taken before a manifest changed
+says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds'
+scenarios stock (`mesh-runtime-<module>:development`): each is compared against the module's
+source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts
+beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh`
+where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075).
+`--no-build` skips this too, and then the bed runs whatever image the store holds.
 
 Check before running a long suite — it says which of these are missing rather than skipping
 quietly:

src/rebuild.ts

@@ -16,6 +16,7 @@
 
 import { spawnSync } from "node:child_process";
 import { repositories } from "./repos.ts";
+import { plannedRuntimes } from "./runtimes.ts";
 
 export interface Build {
   /** What it produces, for the log. */
@@ -112,10 +113,16 @@ export function carriedImage(env: NodeJS.ProcessEnv = process.env): string {
   return env["MESH_LAB_CARRIED_IMAGE"] ?? "mesh-builder:development";
 }
 
-/** rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. */
-export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] {
+/**
+ * rebuild runs the plan, and throws on the first failure rather than testing a stale artifact.
+ *
+ * The plan is what the run was pointed at, plus the module runtimes the named beds stock where
+ * those are older than their source (novox/hq 04-ISSUES/075) — so a bed never again passes against
+ * a runtime built two weeks before the manifest it serves.
+ */
+export function rebuild(env: NodeJS.ProcessEnv = process.env, testFiles: string[] = []): string[] {
   const built: string[] = [];
-  for (const build of planned(env)) {
+  for (const build of [...planned(env), ...plannedRuntimes(testFiles, env)]) {
     const [command, ...args] = build.argv;
     const ran = spawnSync(command!, args, {
       cwd: build.in,

src/runtimes.ts

@@ -0,0 +1,144 @@
+/**
+ * The module runtimes a run stocks are rebuilt by the run, like everything else it tests.
+ *
+ * A per-module bed stocks the module's runtime image — the tool runtime carrying that module's
+ * code — from the workstation's image store, by tag. It was built by hand, by a script the suite
+ * never called, and on the day this was written the images for six modules about to run dated
+ * from two weeks before the manifests they were installed with (novox/hq 04-ISSUES/075). The
+ * suite's own rule, *the run rebuilds what it tests*, was held for the host and the control plane
+ * and not for these.
+ *
+ * So: for the beds about to run, every `mesh-runtime-<module>:development` their scenarios stock
+ * is compared against the source it is built from — the module in the catalogue, and the tool
+ * runtime and SDK it is built on — and rebuilt where the image is older, missing, or the source is
+ * uncommitted. A rebuild that fails stops the suite, the way a stale host binary would.
+ */
+
+import { readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { spawnSync } from "node:child_process";
+import { parse } from "yaml";
+import type { Build } from "./rebuild.ts";
+import { catalogueRoot } from "./repos.ts";
+
+/** A runtime image a scenario stocks by tag, and the module it is built from. */
+export interface StockedRuntime {
+  tag: string;
+  module: string;
+}
+
+/**
+ * Tags whose name is not the module's. The audit logger's runtime was the first, built before the
+ * script was generalised, and the scenario still stocks it under the module's slug.
+ */
+const NAMED_OTHERWISE: Record<string, string> = { "mesh-runtime-audit": "audit-logger" };
+
+const RUNTIME_TAG = /^(mesh-runtime-[a-z0-9-]+):development$/;
+
+/** The runtimes the scenarios of these beds stock, each named once. */
+export function runtimesStockedBy(testFiles: string[], root: string = process.cwd()): StockedRuntime[] {
+  const seen = new Map<string, StockedRuntime>();
+  for (const file of testFiles) {
+    const text = readFileSync(resolve(root, file), "utf8");
+    const named = /const SCENARIO = "([^"]+)"/.exec(text);
+    if (!named) continue;
+    const scenario = parse(readFileSync(join(root, "scenarios", `${named[1]}.yml`), "utf8")) as { images?: unknown };
+    for (const image of Array.isArray(scenario.images) ? scenario.images : []) {
+      const m = RUNTIME_TAG.exec(String(image));
+      if (!m) continue;
+      const repository = m[1]!;
+      seen.set(repository, { tag: String(image), module: NAMED_OTHERWISE[repository] ?? repository.slice("mesh-runtime-".length) });
+    }
+  }
+  return [...seen.values()];
+}
+
+/** When an image was made and when its source last changed, in seconds; null for no image. */
+export interface Ages {
+  image: number | null;
+  /** Infinity where the source has uncommitted changes: what is on disk is newer than any commit. */
+  source: number;
+}
+
+/** stale is whether the image predates its source, or is not there at all. */
+export function isStale(a: Ages): boolean {
+  return a.image === null || a.image < a.source;
+}
+
+/** A command runner, for the two questions asked below; injected so the rules can be tested. */
+export type Ask = (command: string, args: string[]) => { status: number | null; stdout: string };
+
+const ask: Ask = (command, args) => {
+  const ran = spawnSync(command, args, { encoding: "utf8" });
+  return { status: ran.status, stdout: ran.stdout ?? "" };
+};
+
+/** ageOfImage is when the local image store made this tag, or null when it holds no such image. */
+export function ageOfImage(tag: string, run: Ask = ask): number | null {
+  const ran = run("docker", ["image", "inspect", "--format", "{{.Created}}", tag]);
+  if (ran.status !== 0) return null;
+  const at = Date.parse(ran.stdout.trim());
+  return Number.isNaN(at) ? null : Math.floor(at / 1000);
+}
+
+/**
+ * ageOfSource is the newest commit touching what the runtime is built from: the module's directory
+ * in the catalogue, and the whole of each repository it is built on top of. Uncommitted changes in
+ * any of them are newer than every commit.
+ */
+export function ageOfSource(catalogue: string, module: string, builtOn: string[], run: Ask = ask): number {
+  let newest = 0;
+  const at = (dir: string, path?: string): number => {
+    const args = ["-C", dir, "log", "-1", "--format=%ct"];
+    if (path) args.push("--", path);
+    const ran = run("git", args);
+    const t = Number.parseInt(ran.stdout.trim(), 10);
+    return ran.status === 0 && Number.isFinite(t) ? t : 0;
+  };
+  const dirty = (dir: string, path?: string): boolean => {
+    const args = ["-C", dir, "status", "--porcelain"];
+    if (path) args.push("--", path);
+    const ran = run("git", args);
+    return ran.status === 0 && ran.stdout.trim() !== "";
+  };
+  if (dirty(catalogue, `modules/${module}`)) return Infinity;
+  newest = Math.max(newest, at(catalogue, `modules/${module}`));
+  for (const dir of builtOn) {
+    if (dirty(dir)) return Infinity;
+    newest = Math.max(newest, at(dir));
+  }
+  return newest;
+}
+
+/**
+ * plannedRuntimes is the runtime builds these beds need before they run, given where the run was
+ * pointed: nothing where no catalogue was named (the beds skip), one build per stale image
+ * otherwise. The repositories the runtime is built on are the siblings the build script itself
+ * reads (`MESH_TOOLS`, `MESH_SDK`, or the checkouts beside this one).
+ */
+export function plannedRuntimes(testFiles: string[], env: NodeJS.ProcessEnv = process.env,
+  root: string = process.cwd(), run: Ask = ask): Build[] {
+  const named = env["MESH_LAB_CATALOG"];
+  if (!named) return [];
+  const catalogue = catalogueRoot(named);
+  const builtOn = [
+    env["MESH_TOOLS"] ?? resolve(root, "..", "mesh-tools"),
+    env["MESH_SDK"] ?? resolve(root, "..", "mesh-sdk"),
+  ];
+  const builds: Build[] = [];
+  for (const runtime of runtimesStockedBy(testFiles, root)) {
+    const ages: Ages = {
+      image: ageOfImage(runtime.tag, run),
+      source: ageOfSource(catalogue, runtime.module, builtOn, run),
+    };
+    if (!isStale(ages)) continue;
+    builds.push({
+      what: `runtime ${runtime.tag}`,
+      in: root,
+      argv: ["scripts/build-module-runtime.sh", runtime.module, join(tmpdir(), `mesh-lab-${runtime.module}.tar`)],
+      env: { MESH_CATALOG: catalogue, RUNTIME_TAG: runtime.tag },
+    });
+  }
+  return builds;
+}

src/suite.ts

@@ -40,7 +40,7 @@ export async function runSuite(args: string[]): Promise<number> {
   if (!args.includes("--no-build")) {
     // Before the run, always. The artifacts are built from two other repositories, and a suite
     // that tests yesterday's binary reports on code nobody is looking at (novox/hq 04-ISSUES/005).
-    const built = rebuild();
+    const built = rebuild(process.env, files);
     if (built.length > 0) console.log(`built: ${built.join(", ")}\n`);
   }
   // Read now, while it is true. The receipt names these, and reading them when the run ends

test/runtimes.test.ts

@@ -0,0 +1,88 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { ageOfImage, ageOfSource, isStale, plannedRuntimes, runtimesStockedBy, type Ask } from "../src/runtimes.ts";
+
+// The module runtimes a run stocks are rebuilt by the run (novox/hq 04-ISSUES/075): what a bed's
+// scenario stocks is found, compared against its source, and built where older.
+
+function aLabWith(beds: Record<string, string>, scenarios: Record<string, string[]>): { root: string; done: () => void } {
+  const root = mkdtempSync(join(tmpdir(), "mesh-lab-runtimes-"));
+  mkdirSync(join(root, "scenarios"));
+  mkdirSync(join(root, "test", "integration"), { recursive: true });
+  for (const [name, images] of Object.entries(scenarios)) {
+    writeFileSync(join(root, "scenarios", `${name}.yml`), `scenario: ${name}\nimages:\n${images.map((i) => `  - ${i}\n`).join("")}`);
+  }
+  for (const [file, scenario] of Object.entries(beds)) {
+    writeFileSync(join(root, "test", "integration", file), `const SCENARIO = "${scenario}";\n`);
+  }
+  return { root, done: () => rmSync(root, { recursive: true, force: true }) };
+}
+
+test("what a bed's scenario stocks is found, by module, once", () => {
+  const lab = aLabWith(
+    { "a.test.ts": "one", "b.test.ts": "two", "c.test.ts": "one" },
+    { one: ["mesh-controller:development", "mesh-runtime-gitlab:development", "postgres:16"],
+      two: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
+  try {
+    const found = runtimesStockedBy(["test/integration/a.test.ts", "test/integration/b.test.ts", "test/integration/c.test.ts"], lab.root);
+    assert.deepEqual(found, [
+      { tag: "mesh-runtime-gitlab:development", module: "gitlab" },
+      // The audit logger's runtime is stocked under its slug, and the module is named for it.
+      { tag: "mesh-runtime-audit:development", module: "audit-logger" },
+    ]);
+  } finally { lab.done(); }
+});
+
+test("an image is stale when missing, older than its source, or when the source is uncommitted", () => {
+  assert.equal(isStale({ image: null, source: 0 }), true);
+  assert.equal(isStale({ image: 100, source: 200 }), true);
+  assert.equal(isStale({ image: 200, source: 100 }), false);
+  assert.equal(isStale({ image: 200, source: Infinity }), true);
+});
+
+test("the image's age comes from the store and the source's from the newest commit in any part", () => {
+  const answers: Ask = (command, args) => {
+    if (command === "docker") return { status: 0, stdout: "2026-09-21T12:00:00.000000000Z\n" };
+    if (args.includes("status")) return { status: 0, stdout: "" };
+    if (args.includes("modules/gitlab")) return { status: 0, stdout: "1000\n" };
+    return { status: 0, stdout: args[1] === "/tools" ? "3000\n" : "2000\n" };
+  };
+  assert.equal(ageOfImage("mesh-runtime-gitlab:development", answers), Date.parse("2026-09-21T12:00:00Z") / 1000);
+  assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], answers), 3000);
+  // No such image is null, not zero: "never built" and "built at the epoch" are different answers.
+  assert.equal(ageOfImage("mesh-runtime-nothing:development", () => ({ status: 1, stdout: "" })), null);
+  // Uncommitted changes anywhere in the source are newer than every commit.
+  const dirtyTools: Ask = (command, args) =>
+    args.includes("status") && args[1] === "/tools" ? { status: 0, stdout: " M src/x.ts\n" } : answers(command, args);
+  assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], dirtyTools), Infinity);
+});
+
+test("only a stale runtime is planned, built by the lab's own script under the tag the scenario stocks", () => {
+  const lab = aLabWith({ "a.test.ts": "one" },
+    { one: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
+  try {
+    const answers: Ask = (command, args) => {
+      if (command === "docker") {
+        // gitlab's image is from yesterday; the audit logger has none.
+        return args.includes("mesh-runtime-gitlab:development")
+          ? { status: 0, stdout: "2026-09-21T12:00:00Z\n" } : { status: 1, stdout: "" };
+      }
+      if (args.includes("status")) return { status: 0, stdout: "" };
+      return { status: 0, stdout: `${Date.parse("2026-09-20T00:00:00Z") / 1000}\n` };
+    };
+    const env = { MESH_LAB_CATALOG: "/catalogue/modules", MESH_TOOLS: "/tools", MESH_SDK: "/sdk" };
+    const builds = plannedRuntimes(["test/integration/a.test.ts"], env, lab.root, answers);
+    assert.equal(builds.length, 1);
+    assert.equal(builds[0]!.what, "runtime mesh-runtime-audit:development");
+    assert.equal(builds[0]!.argv[0], "scripts/build-module-runtime.sh");
+    assert.equal(builds[0]!.argv[1], "audit-logger");
+    assert.equal(builds[0]!.env?.["MESH_CATALOG"], "/catalogue");
+    assert.equal(builds[0]!.env?.["RUNTIME_TAG"], "mesh-runtime-audit:development");
+    // No catalogue named: nothing is planned, because no bed will read one either.
+    assert.deepEqual(plannedRuntimes(["test/integration/a.test.ts"], {}, lab.root, answers), []);
+  } finally { lab.done(); }
+});