lab: one job at a time with no await between check and start, and no rebuild left holding anything (issue 698, review 256)

A run and a rebuild each asked the package registry between checking for
the other and writing their status, so both could start and a bed could
raise machines from an image being replaced. The registry is asked first
now. A hard-killed rebuild's credential is cleaned like a run's; a job that
never recorded its process is lost rather than blocking every run; and
lab_check reports a missing or unreadable base image.

modules/lab/tools/base.ts

@@ -1,91 +0,0 @@
-// The lab's base image: how old it is, and rebuilding it (novox/hq issue 698).
-//
-// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which
-// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the
-// package database the image was built with. On a rolling distribution the mirrors stop serving what an old
-// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror
-// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a
-// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image
-// older than the bound, and rebuilds the image when asked to.
-//
-// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is
-// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds
-// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little;
-// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test.
-
-import { spawnSync } from "node:child_process";
-
-/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */
-export const BASE_IMAGE_ALIAS = "mesh-lab/base";
-
-/** How old the base image may be, in days, before a run is refused (see the head of this file). */
-export const BASE_MAX_AGE_DAYS = 14;
-
-const DAY = 24 * 60 * 60 * 1000;
-
-export interface BaseImage {
-  alias: string;
-  /** Whether the image is there at all. */
-  exists: boolean;
-  /** When this machine made it, ISO. */
-  built?: string;
-  /** Its age in whole days. */
-  ageDays?: number;
-  /** Why its age could not be read, when it could not. */
-  problem?: string;
-}
-
-/**
- * baseImageFrom reads `incus image list <alias> --format json`: the image whose aliases hold the base's
- * name, and when it was made here (its upload, which a publish sets).
- */
-export function baseImageFrom(listed: string | undefined, now: Date): BaseImage {
-  const alias = BASE_IMAGE_ALIAS;
-  if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" };
-  let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>;
-  try {
-    images = JSON.parse(listed);
-  } catch {
-    return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" };
-  }
-  const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias));
-  if (!image) return { alias, exists: false };
-  const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN))
-    .find((t) => Number.isFinite(t) && t > 0);
-  if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" };
-  return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) };
-}
-
-/** readBaseImage asks incus, on this machine, for the base image. */
-export function readBaseImage(now = new Date()): BaseImage {
-  const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" });
-  return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now);
-}
-
-/** The base image's age in one sentence, as lab_check and lab_run say it. */
-export function describeBase(b: BaseImage): string {
-  if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`;
-  if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`;
-  return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`;
-}
-
-/**
- * staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the
- * bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says
- * so itself, and a bed that does not need it runs.
- */
-export function staleBase(b: BaseImage): string | undefined {
-  if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined;
-  return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` +
-    `${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` +
-    `(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`;
-}
-
-/**
- * runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which
- * replaces the image a bed raises its machines from, or an image older than the bound.
- */
-export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined {
-  if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`;
-  return staleBase(base);
-}

modules/lab/tools/index.ts

@@ -9,9 +9,9 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
   checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start,
-  stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild,
+  stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun,
+  launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem,
 } from "./runs.js";
-import { BASE_MAX_AGE_DAYS, describeBase, readBaseImage, runRefusal, staleBase } from "./base.js";
 
 export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
   const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs";
@@ -60,15 +60,16 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
         // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every
         // repository it clones answers on the forge, and every one it installs has a lock file where the run
         // installs from.
-        // A credential a stopped, crashed or rebooted run left behind is removed, and said; one still in use
-        // by a running run is not.
-        const busy = running(work);
-        const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined);
-        const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : "";
-        // **And how old the base image is** (novox/hq issue 698): one past the bound is a problem, as lab_run refuses it.
+        // A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in
+        // use by a running run or rebuild is not.
+        const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id);
+        const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id)));
+        const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : "";
+        // **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot
+        // be read is a problem here, though lab_run refuses only the first.
         const base = readBaseImage();
-        const stale = staleBase(base);
-        if (stale) problems.push(stale);
+        const unfit = baseProblem(base);
+        if (unfit) problems.push(unfit);
         const output = [`${check.stdout}${check.stderr}`.trim(), describeBase(base), removed, ...leftovers, ...problems]
           .filter(Boolean).join("\n");
         return { ok: check.status === 0 && problems.length === 0, output };
@@ -102,15 +103,13 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
         }
         const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r));
         if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` };
-        const busy = running(work);
-        if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy };
-        // **Not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698).
-        const base = readBaseImage();
-        const refused = runRefusal(base, buildingBase(work));
-        if (refused) return { started: false, reason: refused, base };
-        const registry = await registryOf(forge);
-        return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry),
-          base: describeBase(base), registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) };
+        // **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698,
+        // review 256): the registry asked first, then the check and the start with nothing awaited between.
+        const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() },
+          (registry) => start(work, forge, tests, refs, credentialOf(forge), registry));
+        if (!launched.started) return launched;
+        return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry,
+          ...(launched.problem ? { problem: launched.problem } : {}) };
       },
     },
     {
@@ -129,19 +128,18 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
         const newest = builds[0];
         const reading = {
           base: before, said: describeBase(before), bound: BASE_MAX_AGE_DAYS,
-          ...(staleBase(before) ? { stale: staleBase(before) } : {}),
+          ...(baseProblem(before) ? { problem: baseProblem(before) } : {}),
           builds, ...(newest ? { log: baseBuildLog(work, newest.id) } : {}),
         };
         if (args.rebuild !== true) return reading;
         const forge = forgeOf();
         if (!forge) return { ...reading, started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"<url>\"}" };
-        const rebuilding = buildingBase(work);
-        if (rebuilding) return { ...reading, started: false, reason: `${rebuilding.id} is still rebuilding it` };
-        // A run raises its machines from the image; replacing it under one would change what the run tests.
-        const busy = running(work);
-        if (busy) return { ...reading, started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image` };
-        const registry = await registryOf(forge);
-        return { ...reading, started: true, build: startBaseBuild(work, forge, credentialOf(forge), registry.registry) };
+        // Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with
+        // nothing awaited between (review 256).
+        const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before },
+          (registry) => startBaseBuild(work, forge, credentialOf(forge), registry));
+        if (launched.started === false) return { ...reading, started: false, reason: launched.reason };
+        return { ...reading, started: true, build: launched.job };
       },
     },
     {

modules/lab/tools/runs.ts

@@ -6,7 +6,7 @@
 // and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers
 // for it, and says it was lost rather than pretending it is still going.
 
-import { spawn } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
@@ -223,16 +223,18 @@ trap ${shellQuote(`${rm}; exit 129`)} HUP`;
 }
 
 /**
- * cleanCredentials removes every run's credential file left in the work directory but the one named — what
- * a crash or a reboot left behind, which no trap could remove. Answers what it removed.
+ * cleanCredentials removes every run's and every base rebuild's credential file left in the work directory
+ * but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it
+ * removed.
  */
-export function cleanCredentials(work: string, keep?: string): string[] {
+export function cleanCredentials(work: string, keep?: string | string[]): string[] {
   if (!existsSync(work)) return [];
+  const kept = new Set([keep ?? []].flat());
   const removed: string[] = [];
   for (const name of readdirSync(work)) {
-    if (!/^\.run-.*\.git-credentials$/.test(name)) continue;
+    if (!/^\.(run|base)-.*\.git-credentials$/.test(name)) continue;
     const path = join(work, name);
-    if (path === keep) continue;
+    if (kept.has(path)) continue;
     rmSync(path, { force: true });
     removed.push(path);
   }
@@ -300,8 +302,10 @@ export function readStatus(work: string, id: string): RunStatus | undefined {
   try {
     const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus;
     // A run whose process is gone while its status still says it is going was lost — the runtime or
-    // the machine restarted under it. Said, rather than left reading as running for ever.
-    if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) {
+    // the machine restarted under it. Said, rather than left reading as running for ever. One that never
+    // recorded a process (the runtime died between writing its status and starting it) is lost too: it
+    // would otherwise hold up every run after it (review 256).
+    if (!["passed", "failed", "stopped", "lost"].includes(s.state) && gone(s.pid)) {
       s.state = "lost";
     }
     return s;
@@ -310,6 +314,11 @@ export function readStatus(work: string, id: string): RunStatus | undefined {
   }
 }
 
+/** gone is whether a job's process is not running: none recorded, or none by that id. */
+function gone(pid: number | undefined): boolean {
+  return !pid || !alive(pid);
+}
+
 function alive(pid: number): boolean {
   try {
     process.kill(pid, 0);
@@ -447,7 +456,106 @@ export function tail(work: string, id: string, lines: number): string {
   }
 }
 
-// ---- the base image's rebuild (novox/hq issue 698): mesh-lab's `base build`, in the background, as a run is.
+// ---- the base image (novox/hq issue 698)
+//
+// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which
+// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the
+// package database the image was built with. On a rolling distribution the mirrors stop serving what an old
+// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror
+// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a
+// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image
+// older than the bound, and rebuilds the image when asked to.
+//
+// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is
+// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds
+// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little;
+// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test.
+
+/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */
+export const BASE_IMAGE_ALIAS = "mesh-lab/base";
+
+/** How old the base image may be, in days, before a run is refused (see the head of this file). */
+export const BASE_MAX_AGE_DAYS = 14;
+
+const DAY = 24 * 60 * 60 * 1000;
+
+export interface BaseImage {
+  alias: string;
+  /** Whether the image is there at all. */
+  exists: boolean;
+  /** When this machine made it, ISO. */
+  built?: string;
+  /** Its age in whole days. */
+  ageDays?: number;
+  /** Why its age could not be read, when it could not. */
+  problem?: string;
+}
+
+/**
+ * baseImageFrom reads `incus image list <alias> --format json`: the image whose aliases hold the base's
+ * name, and when it was made here (its upload, which a publish sets).
+ */
+export function baseImageFrom(listed: string | undefined, now: Date): BaseImage {
+  const alias = BASE_IMAGE_ALIAS;
+  if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" };
+  let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>;
+  try {
+    images = JSON.parse(listed);
+  } catch {
+    return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" };
+  }
+  const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias));
+  if (!image) return { alias, exists: false };
+  const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN))
+    .find((t) => Number.isFinite(t) && t > 0);
+  if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" };
+  return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) };
+}
+
+/** readBaseImage asks incus, on this machine, for the base image. */
+export function readBaseImage(now = new Date()): BaseImage {
+  const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" });
+  return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now);
+}
+
+/** The base image's age in one sentence, as lab_check and lab_run say it. */
+export function describeBase(b: BaseImage): string {
+  if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`;
+  if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`;
+  return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`;
+}
+
+/**
+ * staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the
+ * bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says
+ * so itself, and a bed that does not need it runs.
+ */
+export function staleBase(b: BaseImage): string | undefined {
+  if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined;
+  return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` +
+    `${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` +
+    `(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`;
+}
+
+/**
+ * baseProblem is what lab_check and lab_base say is wrong with the base image, or undefined: past the bound, not
+ * on this machine, or of an age that cannot be read (review 256). Only the first refuses a run.
+ */
+export function baseProblem(b: BaseImage): string | undefined {
+  if (b.problem || !b.exists) return describeBase(b);
+  return staleBase(b);
+}
+
+/**
+ * runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which
+ * replaces the image a bed raises its machines from, or an image older than the bound.
+ */
+export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined {
+  if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`;
+  return staleBase(base);
+}
+
+// ---- its rebuild: mesh-lab's `base build`, in the background, as a run is.
 
 export interface BaseBuild {
   id: string;
@@ -469,7 +577,7 @@ export function listBaseBuilds(work: string): BaseBuild[] {
   for (const id of readdirSync(work).filter((d) => d.startsWith("base-"))) {
     try {
       const b = JSON.parse(readFileSync(buildStatusPath(work, id), "utf8")) as BaseBuild;
-      if (b.state === "building" && b.pid && !alive(b.pid)) b.state = "lost";
+      if (b.state === "building" && gone(b.pid)) b.state = "lost";
       builds.push(b);
     } catch {
       // A directory without a status is not a build.
@@ -520,13 +628,15 @@ export function startBaseBuild(work: string, forge: string, credential?: string,
   mkdirSync(dir, { recursive: true });
   const build: BaseBuild = { id, state: "building", started: new Date().toISOString() };
   writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
-  const credentials = writeCredential(join(work, `.${id}.git-credentials`), credential);
+  // What a crashed or rebooted run or rebuild left behind goes before this one writes its own.
+  cleanCredentials(work);
+  const credentials = writeCredential(credentialPath(work, id), credential);
   writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 });
   const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$?
 node -e ${shellQuote(
     `const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
   )} "$code"
-rm -f ${shellQuote(join(work, `.${id}.git-credentials`))}
+rm -f ${shellQuote(credentialPath(work, id))}
 rm -rf ${shellQuote(join(dir, "mesh-lab"))}`;
   const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
   child.unref();
@@ -534,3 +644,49 @@ rm -rf ${shellQuote(join(dir, "mesh-lab"))}`;
   writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
   return build;
 }
+
+// ---- starting a run or a rebuild: one job on the lab's machine at a time (review 256).
+//
+// **Everything that waits is done first; the check and the start are one step.** A run and a rebuild each
+// refused while the other was going, but each asked the package registry — an await — between its check and
+// writing its status, so a run and a rebuild asked together could both pass their checks and both start: a
+// bed raising machines from an image being replaced under it. Here the registry is asked first, and nothing
+// is awaited between reading what is going and writing the new job's status, so in the one runtime that
+// serves the lab's tools no other call runs in between.
+
+export interface Launch {
+  /** The address a run or a rebuild names the package registry by, asked of the forge (npmRegistry). */
+  registry: () => Promise<{ registry: string; problem?: string }>;
+  /** The base image as it is now. */
+  base: () => BaseImage;
+}
+
+export type Launched<T> = { started: true; registry: string; problem?: string; base: BaseImage; job: T }
+  | { started: false; reason: string; base: BaseImage };
+
+/** launchRun starts a run unless a run or a rebuild is going, or the base image is past its bound. */
+export async function launchRun(work: string, launch: Launch, begin: (registry: string) => RunStatus):
+  Promise<Launched<RunStatus>> {
+  const registry = await launch.registry();
+  // From here to begin(): nothing awaited.
+  const base = launch.base();
+  const busy = running(work);
+  if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, base };
+  const refused = runRefusal(base, buildingBase(work));
+  if (refused) return { started: false, reason: refused, base };
+  return { started: true, ...registry, base, job: begin(registry.registry) };
+}
+
+/** launchRebuild starts a rebuild of the base image unless a run or another rebuild is going. */
+export async function launchRebuild(work: string, launch: Launch, begin: (registry: string) => BaseBuild):
+  Promise<Launched<BaseBuild>> {
+  const registry = await launch.registry();
+  // From here to begin(): nothing awaited.
+  const base = launch.base();
+  const rebuilding = buildingBase(work);
+  if (rebuilding) return { started: false, reason: `${rebuilding.id} is still rebuilding it`, base };
+  // A run raises its machines from the image; replacing it under one would change what the run tests.
+  const busy = running(work);
+  if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image`, base };
+  return { started: true, ...registry, base, job: begin(registry.registry) };
+}