import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, baseProblem, baseScript, buildingBase, cleanCredentials, credentialPath, describeBase, launchRebuild, launchRun, listBaseBuilds, listRuns, readStatus, runRefusal, running, staleBase, type BaseImage, } from "../tools/runs.ts"; // Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a // package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for // postgresql-libs. Nothing said how old the image was, nothing refused it, and no mesh tool rebuilt it. const now = new Date("2026-10-14T01:00:00Z"); const listed = (uploaded: string, alias = BASE_IMAGE_ALIAS) => JSON.stringify([ { aliases: [{ name: "images:other" }], uploaded_at: "2026-10-13T00:00:00Z" }, { aliases: [{ name: alias }], uploaded_at: uploaded, created_at: "2026-08-01T00:00:00Z" }, ]); test("the base image's age is read from when this machine made it", () => { const b = baseImageFrom(listed("2026-08-31T09:00:00Z"), now); assert.equal(b.exists, true); assert.equal(b.built, "2026-08-31T09:00:00.000Z"); assert.equal(b.ageDays, 43); assert.match(describeBase(b), /built 2026-08-31T09:00:00.000Z \(43 day\(s\) ago; the bound is 14\)/); }); test("an image older than the bound refuses a run, naming the tool that rebuilds it", () => { const stale = baseImageFrom(listed("2026-08-31T09:00:00Z"), now); const reason = runRefusal(stale); assert.ok(reason, "a 43-day-old base image was not refused"); assert.match(reason, /older than the lab's bound of 14 days/); assert.match(reason, /lab_base \{"rebuild": true\}/); assert.equal(staleBase(stale), reason); }); test("an image within the bound runs", () => { const fresh = baseImageFrom(listed(new Date(now.getTime() - BASE_MAX_AGE_DAYS * 86_400_000).toISOString()), now); assert.equal(fresh.ageDays, BASE_MAX_AGE_DAYS); assert.equal(runRefusal(fresh), undefined); }); test("a missing image, or one whose age cannot be read, is said and not refused", () => { const none = baseImageFrom(listed("2026-10-01T00:00:00Z", "something-else"), now); assert.equal(none.exists, false); assert.equal(runRefusal(none), undefined); assert.match(describeBase(none), /is not on this machine; rebuild it with lab_base/); const unread = baseImageFrom(undefined, now); assert.equal(runRefusal(unread), undefined); assert.match(describeBase(unread), /its age is not known/); }); test("no run starts while the base image is rebuilt", () => { const fresh = baseImageFrom(listed("2026-10-13T00:00:00Z"), now); const reason = runRefusal(fresh, { id: "base-x", started: "2026-10-14T00:59:00Z" }); assert.match(reason ?? "", /is being rebuilt \(base-x/); }); test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => { const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials"); assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/); assert.match(s, /src\/cli.ts base build/); assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone"); }); test("a rebuild is known by its status, and is not a run", () => { const work = mkdtempSync(join(tmpdir(), "lab-base-")); try { mkdirSync(join(work, "base-1")); writeFileSync(join(work, "base-1", "status.json"), JSON.stringify({ id: "base-1", state: "built", started: "2026-10-13T00:00:00Z" })); mkdirSync(join(work, "base-2")); // A build whose process is gone while it says building was lost, and holds nothing up. writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: 2 ** 22 + 7 })); assert.deepEqual(listBaseBuilds(work).map((b) => `${b.id} ${b.state}`), ["base-2 lost", "base-1 built"]); assert.equal(buildingBase(work), undefined); writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: process.pid })); assert.equal(buildingBase(work)?.id, "base-2"); assert.deepEqual(listRuns(work), []); assert.ok(readFileSync(join(work, "base-1", "status.json"), "utf8")); } finally { rmSync(work, { recursive: true, force: true }); } }); // **A run and a rebuild asked together: one starts** (review 256). Each asks the package registry, which // waits, and each refused while the other was going; asked at once, both passed their checks before either // wrote its status. The registry is asked first now, and the check and the start are one step. function aLab(): string { return mkdtempSync(join(tmpdir(), "lab-launch-")); } const fresh: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-10-13T00:00:00.000Z", ageDays: 1 }; const slowRegistry = () => new Promise<{ registry: string }>((ok) => setTimeout(() => ok({ registry: "https://forge.invalid/r/" }), 20)); const launch = { registry: slowRegistry, base: () => fresh }; function beginRun(work: string, id: string) { return () => { mkdirSync(join(work, id)); const s = { id, state: "running" as const, started: new Date().toISOString(), tests: [], refs: {}, pid: process.pid }; writeFileSync(join(work, id, "status.json"), JSON.stringify(s)); return s; }; } function beginBuild(work: string, id: string) { return () => { mkdirSync(join(work, id)); const b = { id, state: "building" as const, started: new Date().toISOString(), pid: process.pid }; writeFileSync(join(work, id, "status.json"), JSON.stringify(b)); return b; }; } test("a run and a rebuild asked at once: exactly one starts", async () => { for (const runFirst of [true, false]) { const work = aLab(); try { // Both asked before either answers, in one order and then the other. const ask = { run: () => launchRun(work, launch, beginRun(work, "run-a")), build: () => launchRebuild(work, launch, beginBuild(work, "base-a")) }; const [r, b] = runFirst ? await Promise.all([ask.run(), ask.build()]) : (await Promise.all([ask.build(), ask.run()])).reverse() as [Awaited>, Awaited>]; assert.equal([r.started, b.started].filter(Boolean).length, 1, `both or neither started: run ${r.started}, rebuild ${b.started}`); assert.equal(listRuns(work).length + listBaseBuilds(work).length, 1); } finally { rmSync(work, { recursive: true, force: true }); } } }); test("two runs, or two rebuilds, asked at once: one of each starts", async () => { const work = aLab(); try { const runs = await Promise.all([launchRun(work, launch, beginRun(work, "run-a")), launchRun(work, launch, beginRun(work, "run-b"))]); assert.deepEqual(runs.map((r) => r.started).sort(), [false, true]); const other = aLab(); const builds = await Promise.all([launchRebuild(other, launch, beginBuild(other, "base-a")), launchRebuild(other, launch, beginBuild(other, "base-b"))]); assert.deepEqual(builds.map((b) => b.started).sort(), [false, true]); rmSync(other, { recursive: true, force: true }); } finally { rmSync(work, { recursive: true, force: true }); } }); test("a run is refused on a stale base, and starts nothing", async () => { const work = aLab(); try { const stale: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-08-31T09:00:00.000Z", ageDays: 43 }; const r = await launchRun(work, { registry: slowRegistry, base: () => stale }, beginRun(work, "run-a")); assert.equal(r.started, false); assert.match(r.started ? "" : r.reason, /lab_base \{"rebuild": true\}/); assert.deepEqual(listRuns(work), []); } finally { rmSync(work, { recursive: true, force: true }); } }); test("a rebuild or a run that never recorded its process is lost, and holds nothing up", () => { const work = aLab(); try { mkdirSync(join(work, "base-x")); writeFileSync(join(work, "base-x", "status.json"), JSON.stringify({ id: "base-x", state: "building", started: "2026-10-14T00:00:00Z" })); mkdirSync(join(work, "run-x")); writeFileSync(join(work, "run-x", "status.json"), JSON.stringify({ id: "run-x", state: "running", started: "2026-10-14T00:00:00Z", tests: [], refs: {} })); assert.equal(listBaseBuilds(work)[0]?.state, "lost"); assert.equal(buildingBase(work), undefined); assert.equal(readStatus(work, "run-x")?.state, "lost"); assert.equal(running(work), undefined); } finally { rmSync(work, { recursive: true, force: true }); } }); test("a rebuild's credential left behind is cleaned, and a running one's is kept", () => { const work = aLab(); try { for (const id of ["base-old", "base-now", "run-old", "run-now"]) writeFileSync(credentialPath(work, id), "x"); const removed = cleanCredentials(work, [credentialPath(work, "base-now"), credentialPath(work, "run-now")]).sort(); assert.deepEqual(removed, [credentialPath(work, "base-old"), credentialPath(work, "run-old")].sort()); } finally { rmSync(work, { recursive: true, force: true }); } }); test("lab_check's problem: a stale, missing or unread base image; a fresh one is none", () => { assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false }) ?? "", /is not on this machine/); assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false, problem: "incus could not list the lab's images" }) ?? "", /age is not known/); assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: true, built: "x", ageDays: 30 }) ?? "", /older than the lab's bound/); assert.equal(baseProblem(fresh), undefined); }); // The tools use the gate and the check above (index.ts registers with the SDK when imported, so it is read). test("lab_run and lab_base start through the gate, and lab_check names what is wrong with the base", () => { const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8"); assert.match(index, /await launchRun\(work,/); assert.match(index, /await launchRebuild\(work,/); assert.match(index, /const unfit = baseProblem\(base\);\s*if \(unfit\) problems\.push\(unfit\)/); assert.match(index, /if \(args\.rebuild !== true\) return reading;/); });