The run rebuilds the module runtimes its beds stock

A per-module bed stocks mesh-runtime-<module>:development from the workstation's
image store, built by hand by a script the suite never called; six were two weeks
older than the manifests they served. For the beds named, every runtime their
scenarios stock is compared against the module's source and the tool runtime and
SDK it is built on, and rebuilt where older, missing or uncommitted; a failed
build stops the suite (novox/hq 04-ISSUES/075).

README.md

@@ -165,10 +165,15 @@ export MESH_LAB_ROUTE_PROXY=<somewhere>/route-proxy
 
 `MESH_LAB_HOST_BINARY` and `MESH_LAB_MODULES` do double duty: the repository each sits in is what
 `suite` rebuilds and what the receipt claims. Point the run at a repository and it is built and
-claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built: a bed
-installs a catalogue module by reading its manifest from that checkout when it runs, so the
-receipt names the catalogue's commit too, and a run taken before a manifest changed says so
-(novox/hq 04-ISSUES/073).
+claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built as a
+repository: a bed installs a catalogue module by reading its manifest from that checkout when it
+runs, so the receipt names the catalogue's commit too, and a run taken before a manifest changed
+says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds'
+scenarios stock (`mesh-runtime-<module>:development`): each is compared against the module's
+source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts
+beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh`
+where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075).
+`--no-build` skips this too, and then the bed runs whatever image the store holds.
 
 Check before running a long suite — it says which of these are missing rather than skipping
 quietly:

src/rebuild.ts

@@ -16,6 +16,7 @@
 
 import { spawnSync } from "node:child_process";
 import { repositories } from "./repos.ts";
+import { plannedRuntimes } from "./runtimes.ts";
 
 export interface Build {
   /** What it produces, for the log. */
@@ -112,10 +113,16 @@ export function carriedImage(env: NodeJS.ProcessEnv = process.env): string {
   return env["MESH_LAB_CARRIED_IMAGE"] ?? "mesh-builder:development";
 }
 
-/** rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. */
-export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] {
+/**
+ * rebuild runs the plan, and throws on the first failure rather than testing a stale artifact.
+ *
+ * The plan is what the run was pointed at, plus the module runtimes the named beds stock where
+ * those are older than their source (novox/hq 04-ISSUES/075) — so a bed never again passes against
+ * a runtime built two weeks before the manifest it serves.
+ */
+export function rebuild(env: NodeJS.ProcessEnv = process.env, testFiles: string[] = []): string[] {
   const built: string[] = [];
-  for (const build of planned(env)) {
+  for (const build of [...planned(env), ...plannedRuntimes(testFiles, env)]) {
     const [command, ...args] = build.argv;
     const ran = spawnSync(command!, args, {
       cwd: build.in,

src/runtimes.ts

@@ -0,0 +1,144 @@
+/**
+ * The module runtimes a run stocks are rebuilt by the run, like everything else it tests.
+ *
+ * A per-module bed stocks the module's runtime image — the tool runtime carrying that module's
+ * code — from the workstation's image store, by tag. It was built by hand, by a script the suite
+ * never called, and on the day this was written the images for six modules about to run dated
+ * from two weeks before the manifests they were installed with (novox/hq 04-ISSUES/075). The
+ * suite's own rule, *the run rebuilds what it tests*, was held for the host and the control plane
+ * and not for these.
+ *
+ * So: for the beds about to run, every `mesh-runtime-<module>:development` their scenarios stock
+ * is compared against the source it is built from — the module in the catalogue, and the tool
+ * runtime and SDK it is built on — and rebuilt where the image is older, missing, or the source is
+ * uncommitted. A rebuild that fails stops the suite, the way a stale host binary would.
+ */
+
+import { readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { spawnSync } from "node:child_process";
+import { parse } from "yaml";
+import type { Build } from "./rebuild.ts";
+import { catalogueRoot } from "./repos.ts";
+
+/** A runtime image a scenario stocks by tag, and the module it is built from. */
+export interface StockedRuntime {
+  tag: string;
+  module: string;
+}
+
+/**
+ * Tags whose name is not the module's. The audit logger's runtime was the first, built before the
+ * script was generalised, and the scenario still stocks it under the module's slug.
+ */
+const NAMED_OTHERWISE: Record<string, string> = { "mesh-runtime-audit": "audit-logger" };
+
+const RUNTIME_TAG = /^(mesh-runtime-[a-z0-9-]+):development$/;
+
+/** The runtimes the scenarios of these beds stock, each named once. */
+export function runtimesStockedBy(testFiles: string[], root: string = process.cwd()): StockedRuntime[] {
+  const seen = new Map<string, StockedRuntime>();
+  for (const file of testFiles) {
+    const text = readFileSync(resolve(root, file), "utf8");
+    const named = /const SCENARIO = "([^"]+)"/.exec(text);
+    if (!named) continue;
+    const scenario = parse(readFileSync(join(root, "scenarios", `${named[1]}.yml`), "utf8")) as { images?: unknown };
+    for (const image of Array.isArray(scenario.images) ? scenario.images : []) {
+      const m = RUNTIME_TAG.exec(String(image));
+      if (!m) continue;
+      const repository = m[1]!;
+      seen.set(repository, { tag: String(image), module: NAMED_OTHERWISE[repository] ?? repository.slice("mesh-runtime-".length) });
+    }
+  }
+  return [...seen.values()];
+}
+
+/** When an image was made and when its source last changed, in seconds; null for no image. */
+export interface Ages {
+  image: number | null;
+  /** Infinity where the source has uncommitted changes: what is on disk is newer than any commit. */
+  source: number;
+}
+
+/** stale is whether the image predates its source, or is not there at all. */
+export function isStale(a: Ages): boolean {
+  return a.image === null || a.image < a.source;
+}
+
+/** A command runner, for the two questions asked below; injected so the rules can be tested. */
+export type Ask = (command: string, args: string[]) => { status: number | null; stdout: string };
+
+const ask: Ask = (command, args) => {
+  const ran = spawnSync(command, args, { encoding: "utf8" });
+  return { status: ran.status, stdout: ran.stdout ?? "" };
+};
+
+/** ageOfImage is when the local image store made this tag, or null when it holds no such image. */
+export function ageOfImage(tag: string, run: Ask = ask): number | null {
+  const ran = run("docker", ["image", "inspect", "--format", "{{.Created}}", tag]);
+  if (ran.status !== 0) return null;
+  const at = Date.parse(ran.stdout.trim());
+  return Number.isNaN(at) ? null : Math.floor(at / 1000);
+}
+
+/**
+ * ageOfSource is the newest commit touching what the runtime is built from: the module's directory
+ * in the catalogue, and the whole of each repository it is built on top of. Uncommitted changes in
+ * any of them are newer than every commit.
+ */
+export function ageOfSource(catalogue: string, module: string, builtOn: string[], run: Ask = ask): number {
+  let newest = 0;
+  const at = (dir: string, path?: string): number => {
+    const args = ["-C", dir, "log", "-1", "--format=%ct"];
+    if (path) args.push("--", path);
+    const ran = run("git", args);
+    const t = Number.parseInt(ran.stdout.trim(), 10);
+    return ran.status === 0 && Number.isFinite(t) ? t : 0;
+  };
+  const dirty = (dir: string, path?: string): boolean => {
+    const args = ["-C", dir, "status", "--porcelain"];
+    if (path) args.push("--", path);
+    const ran = run("git", args);
+    return ran.status === 0 && ran.stdout.trim() !== "";
+  };
+  if (dirty(catalogue, `modules/${module}`)) return Infinity;
+  newest = Math.max(newest, at(catalogue, `modules/${module}`));
+  for (const dir of builtOn) {
+    if (dirty(dir)) return Infinity;
+    newest = Math.max(newest, at(dir));
+  }
+  return newest;
+}
+
+/**
+ * plannedRuntimes is the runtime builds these beds need before they run, given where the run was
+ * pointed: nothing where no catalogue was named (the beds skip), one build per stale image
+ * otherwise. The repositories the runtime is built on are the siblings the build script itself
+ * reads (`MESH_TOOLS`, `MESH_SDK`, or the checkouts beside this one).
+ */
+export function plannedRuntimes(testFiles: string[], env: NodeJS.ProcessEnv = process.env,
+  root: string = process.cwd(), run: Ask = ask): Build[] {
+  const named = env["MESH_LAB_CATALOG"];
+  if (!named) return [];
+  const catalogue = catalogueRoot(named);
+  const builtOn = [
+    env["MESH_TOOLS"] ?? resolve(root, "..", "mesh-tools"),
+    env["MESH_SDK"] ?? resolve(root, "..", "mesh-sdk"),
+  ];
+  const builds: Build[] = [];
+  for (const runtime of runtimesStockedBy(testFiles, root)) {
+    const ages: Ages = {
+      image: ageOfImage(runtime.tag, run),
+      source: ageOfSource(catalogue, runtime.module, builtOn, run),
+    };
+    if (!isStale(ages)) continue;
+    builds.push({
+      what: `runtime ${runtime.tag}`,
+      in: root,
+      argv: ["scripts/build-module-runtime.sh", runtime.module, join(tmpdir(), `mesh-lab-${runtime.module}.tar`)],
+      env: { MESH_CATALOG: catalogue, RUNTIME_TAG: runtime.tag },
+    });
+  }
+  return builds;
+}

src/suite.ts

@@ -40,7 +40,7 @@ export async function runSuite(args: string[]): Promise<number> {
   if (!args.includes("--no-build")) {
     // Before the run, always. The artifacts are built from two other repositories, and a suite
     // that tests yesterday's binary reports on code nobody is looking at (novox/hq 04-ISSUES/005).
-    const built = rebuild();
+    const built = rebuild(process.env, files);
     if (built.length > 0) console.log(`built: ${built.join(", ")}\n`);
   }
   // Read now, while it is true. The receipt names these, and reading them when the run ends

test/runtimes.test.ts

@@ -0,0 +1,88 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { ageOfImage, ageOfSource, isStale, plannedRuntimes, runtimesStockedBy, type Ask } from "../src/runtimes.ts";
+
+// The module runtimes a run stocks are rebuilt by the run (novox/hq 04-ISSUES/075): what a bed's
+// scenario stocks is found, compared against its source, and built where older.
+
+function aLabWith(beds: Record<string, string>, scenarios: Record<string, string[]>): { root: string; done: () => void } {
+  const root = mkdtempSync(join(tmpdir(), "mesh-lab-runtimes-"));
+  mkdirSync(join(root, "scenarios"));
+  mkdirSync(join(root, "test", "integration"), { recursive: true });
+  for (const [name, images] of Object.entries(scenarios)) {
+    writeFileSync(join(root, "scenarios", `${name}.yml`), `scenario: ${name}\nimages:\n${images.map((i) => `  - ${i}\n`).join("")}`);
+  }
+  for (const [file, scenario] of Object.entries(beds)) {
+    writeFileSync(join(root, "test", "integration", file), `const SCENARIO = "${scenario}";\n`);
+  }
+  return { root, done: () => rmSync(root, { recursive: true, force: true }) };
+}
+
+test("what a bed's scenario stocks is found, by module, once", () => {
+  const lab = aLabWith(
+    { "a.test.ts": "one", "b.test.ts": "two", "c.test.ts": "one" },
+    { one: ["mesh-controller:development", "mesh-runtime-gitlab:development", "postgres:16"],
+      two: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
+  try {
+    const found = runtimesStockedBy(["test/integration/a.test.ts", "test/integration/b.test.ts", "test/integration/c.test.ts"], lab.root);
+    assert.deepEqual(found, [
+      { tag: "mesh-runtime-gitlab:development", module: "gitlab" },
+      // The audit logger's runtime is stocked under its slug, and the module is named for it.
+      { tag: "mesh-runtime-audit:development", module: "audit-logger" },
+    ]);
+  } finally { lab.done(); }
+});
+
+test("an image is stale when missing, older than its source, or when the source is uncommitted", () => {
+  assert.equal(isStale({ image: null, source: 0 }), true);
+  assert.equal(isStale({ image: 100, source: 200 }), true);
+  assert.equal(isStale({ image: 200, source: 100 }), false);
+  assert.equal(isStale({ image: 200, source: Infinity }), true);
+});
+
+test("the image's age comes from the store and the source's from the newest commit in any part", () => {
+  const answers: Ask = (command, args) => {
+    if (command === "docker") return { status: 0, stdout: "2026-09-21T12:00:00.000000000Z\n" };
+    if (args.includes("status")) return { status: 0, stdout: "" };
+    if (args.includes("modules/gitlab")) return { status: 0, stdout: "1000\n" };
+    return { status: 0, stdout: args[1] === "/tools" ? "3000\n" : "2000\n" };
+  };
+  assert.equal(ageOfImage("mesh-runtime-gitlab:development", answers), Date.parse("2026-09-21T12:00:00Z") / 1000);
+  assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], answers), 3000);
+  // No such image is null, not zero: "never built" and "built at the epoch" are different answers.
+  assert.equal(ageOfImage("mesh-runtime-nothing:development", () => ({ status: 1, stdout: "" })), null);
+  // Uncommitted changes anywhere in the source are newer than every commit.
+  const dirtyTools: Ask = (command, args) =>
+    args.includes("status") && args[1] === "/tools" ? { status: 0, stdout: " M src/x.ts\n" } : answers(command, args);
+  assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], dirtyTools), Infinity);
+});
+
+test("only a stale runtime is planned, built by the lab's own script under the tag the scenario stocks", () => {
+  const lab = aLabWith({ "a.test.ts": "one" },
+    { one: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
+  try {
+    const answers: Ask = (command, args) => {
+      if (command === "docker") {
+        // gitlab's image is from yesterday; the audit logger has none.
+        return args.includes("mesh-runtime-gitlab:development")
+          ? { status: 0, stdout: "2026-09-21T12:00:00Z\n" } : { status: 1, stdout: "" };
+      }
+      if (args.includes("status")) return { status: 0, stdout: "" };
+      return { status: 0, stdout: `${Date.parse("2026-09-20T00:00:00Z") / 1000}\n` };
+    };
+    const env = { MESH_LAB_CATALOG: "/catalogue/modules", MESH_TOOLS: "/tools", MESH_SDK: "/sdk" };
+    const builds = plannedRuntimes(["test/integration/a.test.ts"], env, lab.root, answers);
+    assert.equal(builds.length, 1);
+    assert.equal(builds[0]!.what, "runtime mesh-runtime-audit:development");
+    assert.equal(builds[0]!.argv[0], "scripts/build-module-runtime.sh");
+    assert.equal(builds[0]!.argv[1], "audit-logger");
+    assert.equal(builds[0]!.env?.["MESH_CATALOG"], "/catalogue");
+    assert.equal(builds[0]!.env?.["RUNTIME_TAG"], "mesh-runtime-audit:development");
+    // No catalogue named: nothing is planned, because no bed will read one either.
+    assert.deepEqual(plannedRuntimes(["test/integration/a.test.ts"], {}, lab.root, answers), []);
+  } finally { lab.done(); }
+});