lab: name the base image's age, refuse a stale one, and rebuild it (issue 698)

A bed machine carries the package database its base image was built with,
and the mirrors stop serving it within weeks: genesis-keeps-its-bus failed
on a 404 from an image 44 days old, with nothing saying its age and no mesh
tool to rebuild it. lab_check and lab_run now say the age, lab_run refuses
an image older than 14 days and names the remedy, and lab_base reads the
image and rebuilds it with mesh-lab's base build when asked.

modules/lab/test/base.test.ts

@@ -0,0 +1,98 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, describeBase, runRefusal, staleBase } from "../tools/base.ts";
+import { baseScript, buildingBase, listBaseBuilds, listRuns } from "../tools/runs.ts";
+
+// Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a
+// package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for
+// postgresql-libs. Nothing said how old the image was, nothing refused it, and no mesh tool rebuilt it.
+
+const now = new Date("2026-10-14T01:00:00Z");
+const listed = (uploaded: string, alias = BASE_IMAGE_ALIAS) => JSON.stringify([
+  { aliases: [{ name: "images:other" }], uploaded_at: "2026-10-13T00:00:00Z" },
+  { aliases: [{ name: alias }], uploaded_at: uploaded, created_at: "2026-08-01T00:00:00Z" },
+]);
+
+test("the base image's age is read from when this machine made it", () => {
+  const b = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
+  assert.equal(b.exists, true);
+  assert.equal(b.built, "2026-08-31T09:00:00.000Z");
+  assert.equal(b.ageDays, 43);
+  assert.match(describeBase(b), /built 2026-08-31T09:00:00.000Z \(43 day\(s\) ago; the bound is 14\)/);
+});
+
+test("an image older than the bound refuses a run, naming the tool that rebuilds it", () => {
+  const stale = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
+  const reason = runRefusal(stale);
+  assert.ok(reason, "a 43-day-old base image was not refused");
+  assert.match(reason, /older than the lab's bound of 14 days/);
+  assert.match(reason, /lab_base \{"rebuild": true\}/);
+  assert.equal(staleBase(stale), reason);
+});
+
+test("an image within the bound runs", () => {
+  const fresh = baseImageFrom(listed(new Date(now.getTime() - BASE_MAX_AGE_DAYS * 86_400_000).toISOString()), now);
+  assert.equal(fresh.ageDays, BASE_MAX_AGE_DAYS);
+  assert.equal(runRefusal(fresh), undefined);
+});
+
+test("a missing image, or one whose age cannot be read, is said and not refused", () => {
+  const none = baseImageFrom(listed("2026-10-01T00:00:00Z", "something-else"), now);
+  assert.equal(none.exists, false);
+  assert.equal(runRefusal(none), undefined);
+  assert.match(describeBase(none), /is not on this machine; rebuild it with lab_base/);
+  const unread = baseImageFrom(undefined, now);
+  assert.equal(runRefusal(unread), undefined);
+  assert.match(describeBase(unread), /its age is not known/);
+});
+
+test("no run starts while the base image is rebuilt", () => {
+  const fresh = baseImageFrom(listed("2026-10-13T00:00:00Z"), now);
+  const reason = runRefusal(fresh, { id: "base-x", started: "2026-10-14T00:59:00Z" });
+  assert.match(reason ?? "", /is being rebuilt \(base-x/);
+});
+
+test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => {
+  const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials");
+  assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/);
+  assert.match(s, /src\/cli.ts base build/);
+  assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone");
+});
+
+test("a rebuild is known by its status, and is not a run", () => {
+  const work = mkdtempSync(join(tmpdir(), "lab-base-"));
+  try {
+    mkdirSync(join(work, "base-1"));
+    writeFileSync(join(work, "base-1", "status.json"), JSON.stringify({ id: "base-1", state: "built", started: "2026-10-13T00:00:00Z" }));
+    mkdirSync(join(work, "base-2"));
+    // A build whose process is gone while it says building was lost, and holds nothing up.
+    writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: 2 ** 22 + 7 }));
+    assert.deepEqual(listBaseBuilds(work).map((b) => `${b.id} ${b.state}`), ["base-2 lost", "base-1 built"]);
+    assert.equal(buildingBase(work), undefined);
+    writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: process.pid }));
+    assert.equal(buildingBase(work)?.id, "base-2");
+    assert.deepEqual(listRuns(work), []);
+    assert.ok(readFileSync(join(work, "base-1", "status.json"), "utf8"));
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+// The tools themselves, as written in index.ts (it registers with the SDK when imported, so it is read).
+test("lab_run refuses on the base before it starts anything, lab_check names its age, lab_base reads unless asked", () => {
+  const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8");
+  const run = index.slice(index.indexOf('name: "lab_run"'), index.indexOf('name: "lab_base"'));
+  assert.ok(run.indexOf("runRefusal(base, buildingBase(work))") > 0, "lab_run does not ask about the base");
+  assert.ok(run.indexOf("runRefusal(") < run.indexOf("start(work"), "lab_run starts before it asks about the base");
+  const check = index.slice(index.indexOf('name: "lab_check"'), index.indexOf('name: "lab_run"'));
+  assert.match(check, /if \(stale\) problems\.push\(stale\)/);
+  assert.match(check, /describeBase\(base\)/);
+  const base = index.slice(index.indexOf('name: "lab_base"'), index.indexOf('name: "lab_status"'));
+  assert.ok(base.indexOf("if (args.rebuild !== true) return reading;") > 0, "lab_base does not read by default");
+  assert.ok(base.indexOf("if (args.rebuild !== true) return reading;") < base.indexOf("startBaseBuild("), "lab_base rebuilds unasked");
+  assert.ok(base.indexOf("running(work)") < base.indexOf("startBaseBuild("), "lab_base rebuilds under a run");
+});

modules/lab/tools/base.ts

@@ -0,0 +1,91 @@
+// The lab's base image: how old it is, and rebuilding it (novox/hq issue 698).
+//
+// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which
+// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the
+// package database the image was built with. On a rolling distribution the mirrors stop serving what an old
+// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror
+// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a
+// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image
+// older than the bound, and rebuilds the image when asked to.
+//
+// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is
+// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds
+// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little;
+// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test.
+
+import { spawnSync } from "node:child_process";
+
+/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */
+export const BASE_IMAGE_ALIAS = "mesh-lab/base";
+
+/** How old the base image may be, in days, before a run is refused (see the head of this file). */
+export const BASE_MAX_AGE_DAYS = 14;
+
+const DAY = 24 * 60 * 60 * 1000;
+
+export interface BaseImage {
+  alias: string;
+  /** Whether the image is there at all. */
+  exists: boolean;
+  /** When this machine made it, ISO. */
+  built?: string;
+  /** Its age in whole days. */
+  ageDays?: number;
+  /** Why its age could not be read, when it could not. */
+  problem?: string;
+}
+
+/**
+ * baseImageFrom reads `incus image list <alias> --format json`: the image whose aliases hold the base's
+ * name, and when it was made here (its upload, which a publish sets).
+ */
+export function baseImageFrom(listed: string | undefined, now: Date): BaseImage {
+  const alias = BASE_IMAGE_ALIAS;
+  if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" };
+  let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>;
+  try {
+    images = JSON.parse(listed);
+  } catch {
+    return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" };
+  }
+  const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias));
+  if (!image) return { alias, exists: false };
+  const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN))
+    .find((t) => Number.isFinite(t) && t > 0);
+  if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" };
+  return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) };
+}
+
+/** readBaseImage asks incus, on this machine, for the base image. */
+export function readBaseImage(now = new Date()): BaseImage {
+  const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" });
+  return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now);
+}
+
+/** The base image's age in one sentence, as lab_check and lab_run say it. */
+export function describeBase(b: BaseImage): string {
+  if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`;
+  if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`;
+  return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`;
+}
+
+/**
+ * staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the
+ * bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says
+ * so itself, and a bed that does not need it runs.
+ */
+export function staleBase(b: BaseImage): string | undefined {
+  if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined;
+  return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` +
+    `${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` +
+    `(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`;
+}
+
+/**
+ * runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which
+ * replaces the image a bed raises its machines from, or an image older than the bound.
+ */
+export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined {
+  if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`;
+  return staleBase(base);
+}

modules/lab/tools/index.ts

@@ -9,8 +9,9 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
   checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start,
-  stop, tail, writeCredential,
+  stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild,
 } from "./runs.js";
+import { BASE_MAX_AGE_DAYS, describeBase, readBaseImage, runRefusal, staleBase } from "./base.js";
 
 export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
   const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs";
@@ -64,7 +65,12 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
         const busy = running(work);
         const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined);
         const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : "";
-        const output = [`${check.stdout}${check.stderr}`.trim(), removed, ...leftovers, ...problems].filter(Boolean).join("\n");
+        // **And how old the base image is** (novox/hq issue 698): one past the bound is a problem, as lab_run refuses it.
+        const base = readBaseImage();
+        const stale = staleBase(base);
+        if (stale) problems.push(stale);
+        const output = [`${check.stdout}${check.stderr}`.trim(), describeBase(base), removed, ...leftovers, ...problems]
+          .filter(Boolean).join("\n");
         return { ok: check.status === 0 && problems.length === 0, output };
       },
     },
@@ -98,9 +104,44 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
         if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` };
         const busy = running(work);
         if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy };
+        // **Not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698).
+        const base = readBaseImage();
+        const refused = runRefusal(base, buildingBase(work));
+        if (refused) return { started: false, reason: refused, base };
         const registry = await registryOf(forge);
         return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry),
-          registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) };
+          base: describeBase(base), registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) };
+      },
+    },
+    {
+      name: "lab_base",
+      description:
+        "The lab's base image, which every bed machine with a container runtime is raised from: when it was built, its " +
+        `age, the bound lab_run refuses past (${BASE_MAX_AGE_DAYS} days), and the last rebuilds with the newest one's log. ` +
+        "Reads only, unless rebuild is true: then it rebuilds the image with mesh-lab's `base build` from the forge's " +
+        "main, in the background, and answers at once; ask again for the build's state and the image's new age.",
+      input: {
+        rebuild: { type: "boolean", description: "rebuild the image (default false: only read)" },
+      },
+      run: async (args) => {
+        const before = readBaseImage();
+        const builds = listBaseBuilds(work).slice(0, 3);
+        const newest = builds[0];
+        const reading = {
+          base: before, said: describeBase(before), bound: BASE_MAX_AGE_DAYS,
+          ...(staleBase(before) ? { stale: staleBase(before) } : {}),
+          builds, ...(newest ? { log: baseBuildLog(work, newest.id) } : {}),
+        };
+        if (args.rebuild !== true) return reading;
+        const forge = forgeOf();
+        if (!forge) return { ...reading, started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"<url>\"}" };
+        const rebuilding = buildingBase(work);
+        if (rebuilding) return { ...reading, started: false, reason: `${rebuilding.id} is still rebuilding it` };
+        // A run raises its machines from the image; replacing it under one would change what the run tests.
+        const busy = running(work);
+        if (busy) return { ...reading, started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image` };
+        const registry = await registryOf(forge);
+        return { ...reading, started: true, build: startBaseBuild(work, forge, credentialOf(forge), registry.registry) };
       },
     },
     {

modules/lab/tools/runs.ts

@@ -446,3 +446,91 @@ export function tail(work: string, id: string, lines: number): string {
     return "";
   }
 }
+
+// ---- the base image's rebuild (novox/hq issue 698): mesh-lab's `base build`, in the background, as a run is.
+
+export interface BaseBuild {
+  id: string;
+  state: "building" | "built" | "failed" | "lost";
+  started: string;
+  ended?: string;
+  exit?: number;
+  pid?: number;
+}
+
+function buildStatusPath(work: string, id: string): string {
+  return join(work, id, "status.json");
+}
+
+/** The base builds, newest first. A build whose process is gone while it says building was lost. */
+export function listBaseBuilds(work: string): BaseBuild[] {
+  if (!existsSync(work)) return [];
+  const builds: BaseBuild[] = [];
+  for (const id of readdirSync(work).filter((d) => d.startsWith("base-"))) {
+    try {
+      const b = JSON.parse(readFileSync(buildStatusPath(work, id), "utf8")) as BaseBuild;
+      if (b.state === "building" && b.pid && !alive(b.pid)) b.state = "lost";
+      builds.push(b);
+    } catch {
+      // A directory without a status is not a build.
+    }
+  }
+  return builds.sort((a, b) => b.started.localeCompare(a.started));
+}
+
+/** The base build still going, if any. */
+export function buildingBase(work: string): BaseBuild | undefined {
+  return listBaseBuilds(work).find((b) => b.state === "building");
+}
+
+/** The last lines of a base build's log. */
+export function baseBuildLog(work: string, id: string, lines = 40): string {
+  try {
+    return readFileSync(join(work, id, "build.log"), "utf8").split("\n").slice(-Math.max(1, lines)).join("\n");
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it,
+ * then its `base build`, which replaces the image and reads back that it works before publishing it.
+ */
+export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string {
+  const lab = join(dir, "mesh-lab");
+  return `set -euo pipefail
+export GIT_TERMINAL_PROMPT=0
+${credentialGuard(credential)}
+cd ${shellQuote(dir)}
+git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)}
+${credential ? `rm -f ${shellQuote(credential)}` : ""}
+echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)"
+printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))}
+export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))}
+${installStep(lab, false)}
+cd ${shellQuote(lab)}
+node --experimental-strip-types src/cli.ts base build
+`;
+}
+
+/** startBaseBuild rebuilds the base image in the background and answers at once with the build. */
+export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild {
+  const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`;
+  const dir = join(work, id);
+  mkdirSync(dir, { recursive: true });
+  const build: BaseBuild = { id, state: "building", started: new Date().toISOString() };
+  writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
+  const credentials = writeCredential(join(work, `.${id}.git-credentials`), credential);
+  writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 });
+  const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$?
+node -e ${shellQuote(
+    `const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
+  )} "$code"
+rm -f ${shellQuote(join(work, `.${id}.git-credentials`))}
+rm -rf ${shellQuote(join(dir, "mesh-lab"))}`;
+  const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
+  child.unref();
+  build.pid = child.pid;
+  writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
+  return build;
+}

modules/lab/tsconfig.json

@@ -8,5 +8,5 @@
     "skipLibCheck": true,
     "noEmit": true
   },
-  "include": ["tools/index.ts", "tools/runs.ts"]
+  "include": ["tools/index.ts", "tools/runs.ts", "tools/base.ts"]
 }