lab: one job at a time with no await between check and start, and no rebuild left holding anything (issue 698, review 256)

A run and a rebuild each asked the package registry between checking for
the other and writing their status, so both could start and a bed could
raise machines from an image being replaced. The registry is asked first
now. A hard-killed rebuild's credential is cleaned like a run's; a job that
never recorded its process is lost rather than blocking every run; and
lab_check reports a missing or unreadable base image.

modules/lab/test/base.test.ts

@@ -4,8 +4,11 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
-import { BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, describeBase, runRefusal, staleBase } from "../tools/base.ts";
-import { baseScript, buildingBase, listBaseBuilds, listRuns } from "../tools/runs.ts";
+import {
+  BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, baseProblem, baseScript, buildingBase, cleanCredentials, credentialPath,
+  describeBase, launchRebuild, launchRun, listBaseBuilds, listRuns, readStatus, runRefusal, running, staleBase,
+  type BaseImage,
+} from "../tools/runs.ts";
 
 // Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a
 // package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for
@@ -82,17 +85,114 @@ test("a rebuild is known by its status, and is not a run", () => {
   }
 });
 
-// The tools themselves, as written in index.ts (it registers with the SDK when imported, so it is read).
-test("lab_run refuses on the base before it starts anything, lab_check names its age, lab_base reads unless asked", () => {
+// **A run and a rebuild asked together: one starts** (review 256). Each asks the package registry, which
+// waits, and each refused while the other was going; asked at once, both passed their checks before either
+// wrote its status. The registry is asked first now, and the check and the start are one step.
+function aLab(): string {
+  return mkdtempSync(join(tmpdir(), "lab-launch-"));
+}
+const fresh: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-10-13T00:00:00.000Z", ageDays: 1 };
+const slowRegistry = () => new Promise<{ registry: string }>((ok) => setTimeout(() => ok({ registry: "https://forge.invalid/r/" }), 20));
+const launch = { registry: slowRegistry, base: () => fresh };
+function beginRun(work: string, id: string) {
+  return () => {
+    mkdirSync(join(work, id));
+    const s = { id, state: "running" as const, started: new Date().toISOString(), tests: [], refs: {}, pid: process.pid };
+    writeFileSync(join(work, id, "status.json"), JSON.stringify(s));
+    return s;
+  };
+}
+function beginBuild(work: string, id: string) {
+  return () => {
+    mkdirSync(join(work, id));
+    const b = { id, state: "building" as const, started: new Date().toISOString(), pid: process.pid };
+    writeFileSync(join(work, id, "status.json"), JSON.stringify(b));
+    return b;
+  };
+}
+
+test("a run and a rebuild asked at once: exactly one starts", async () => {
+  for (const runFirst of [true, false]) {
+    const work = aLab();
+    try {
+      // Both asked before either answers, in one order and then the other.
+      const ask = { run: () => launchRun(work, launch, beginRun(work, "run-a")), build: () => launchRebuild(work, launch, beginBuild(work, "base-a")) };
+      const [r, b] = runFirst ? await Promise.all([ask.run(), ask.build()])
+        : (await Promise.all([ask.build(), ask.run()])).reverse() as [Awaited<ReturnType<typeof ask.run>>, Awaited<ReturnType<typeof ask.build>>];
+      assert.equal([r.started, b.started].filter(Boolean).length, 1, `both or neither started: run ${r.started}, rebuild ${b.started}`);
+      assert.equal(listRuns(work).length + listBaseBuilds(work).length, 1);
+    } finally {
+      rmSync(work, { recursive: true, force: true });
+    }
+  }
+});
+
+test("two runs, or two rebuilds, asked at once: one of each starts", async () => {
+  const work = aLab();
+  try {
+    const runs = await Promise.all([launchRun(work, launch, beginRun(work, "run-a")), launchRun(work, launch, beginRun(work, "run-b"))]);
+    assert.deepEqual(runs.map((r) => r.started).sort(), [false, true]);
+    const other = aLab();
+    const builds = await Promise.all([launchRebuild(other, launch, beginBuild(other, "base-a")), launchRebuild(other, launch, beginBuild(other, "base-b"))]);
+    assert.deepEqual(builds.map((b) => b.started).sort(), [false, true]);
+    rmSync(other, { recursive: true, force: true });
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("a run is refused on a stale base, and starts nothing", async () => {
+  const work = aLab();
+  try {
+    const stale: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-08-31T09:00:00.000Z", ageDays: 43 };
+    const r = await launchRun(work, { registry: slowRegistry, base: () => stale }, beginRun(work, "run-a"));
+    assert.equal(r.started, false);
+    assert.match(r.started ? "" : r.reason, /lab_base \{"rebuild": true\}/);
+    assert.deepEqual(listRuns(work), []);
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("a rebuild or a run that never recorded its process is lost, and holds nothing up", () => {
+  const work = aLab();
+  try {
+    mkdirSync(join(work, "base-x"));
+    writeFileSync(join(work, "base-x", "status.json"), JSON.stringify({ id: "base-x", state: "building", started: "2026-10-14T00:00:00Z" }));
+    mkdirSync(join(work, "run-x"));
+    writeFileSync(join(work, "run-x", "status.json"), JSON.stringify({ id: "run-x", state: "running", started: "2026-10-14T00:00:00Z", tests: [], refs: {} }));
+    assert.equal(listBaseBuilds(work)[0]?.state, "lost");
+    assert.equal(buildingBase(work), undefined);
+    assert.equal(readStatus(work, "run-x")?.state, "lost");
+    assert.equal(running(work), undefined);
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("a rebuild's credential left behind is cleaned, and a running one's is kept", () => {
+  const work = aLab();
+  try {
+    for (const id of ["base-old", "base-now", "run-old", "run-now"]) writeFileSync(credentialPath(work, id), "x");
+    const removed = cleanCredentials(work, [credentialPath(work, "base-now"), credentialPath(work, "run-now")]).sort();
+    assert.deepEqual(removed, [credentialPath(work, "base-old"), credentialPath(work, "run-old")].sort());
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("lab_check's problem: a stale, missing or unread base image; a fresh one is none", () => {
+  assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false }) ?? "", /is not on this machine/);
+  assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false, problem: "incus could not list the lab's images" }) ?? "", /age is not known/);
+  assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: true, built: "x", ageDays: 30 }) ?? "", /older than the lab's bound/);
+  assert.equal(baseProblem(fresh), undefined);
+});
+
+// The tools use the gate and the check above (index.ts registers with the SDK when imported, so it is read).
+test("lab_run and lab_base start through the gate, and lab_check names what is wrong with the base", () => {
   const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8");
-  const run = index.slice(index.indexOf('name: "lab_run"'), index.indexOf('name: "lab_base"'));
-  assert.ok(run.indexOf("runRefusal(base, buildingBase(work))") > 0, "lab_run does not ask about the base");
-  assert.ok(run.indexOf("runRefusal(") < run.indexOf("start(work"), "lab_run starts before it asks about the base");
-  const check = index.slice(index.indexOf('name: "lab_check"'), index.indexOf('name: "lab_run"'));
-  assert.match(check, /if \(stale\) problems\.push\(stale\)/);
-  assert.match(check, /describeBase\(base\)/);
-  const base = index.slice(index.indexOf('name: "lab_base"'), index.indexOf('name: "lab_status"'));
-  assert.ok(base.indexOf("if (args.rebuild !== true) return reading;") > 0, "lab_base does not read by default");
-  assert.ok(base.indexOf("if (args.rebuild !== true) return reading;") < base.indexOf("startBaseBuild("), "lab_base rebuilds unasked");
-  assert.ok(base.indexOf("running(work)") < base.indexOf("startBaseBuild("), "lab_base rebuilds under a run");
+  assert.match(index, /await launchRun\(work,/);
+  assert.match(index, /await launchRebuild\(work,/);
+  assert.match(index, /const unfit = baseProblem\(base\);\s*if \(unfit\) problems\.push\(unfit\)/);
+  assert.match(index, /if \(args\.rebuild !== true\) return reading;/);
 });