diff --git a/modules/lab/test/base.test.ts b/modules/lab/test/base.test.ts index f05bc8a3..93a6f8ec 100644 --- a/modules/lab/test/base.test.ts +++ b/modules/lab/test/base.test.ts @@ -4,8 +4,11 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod import { tmpdir } from "node:os"; import { join } from "node:path"; -import { BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, describeBase, runRefusal, staleBase } from "../tools/base.ts"; -import { baseScript, buildingBase, listBaseBuilds, listRuns } from "../tools/runs.ts"; +import { + BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, baseProblem, baseScript, buildingBase, cleanCredentials, credentialPath, + describeBase, launchRebuild, launchRun, listBaseBuilds, listRuns, readStatus, runRefusal, running, staleBase, + type BaseImage, +} from "../tools/runs.ts"; // Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a // package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for @@ -82,17 +85,114 @@ test("a rebuild is known by its status, and is not a run", () => { } }); -// The tools themselves, as written in index.ts (it registers with the SDK when imported, so it is read). -test("lab_run refuses on the base before it starts anything, lab_check names its age, lab_base reads unless asked", () => { +// **A run and a rebuild asked together: one starts** (review 256). Each asks the package registry, which +// waits, and each refused while the other was going; asked at once, both passed their checks before either +// wrote its status. The registry is asked first now, and the check and the start are one step. +function aLab(): string { + return mkdtempSync(join(tmpdir(), "lab-launch-")); +} +const fresh: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-10-13T00:00:00.000Z", ageDays: 1 }; +const slowRegistry = () => new Promise<{ registry: string }>((ok) => setTimeout(() => ok({ registry: "https://forge.invalid/r/" }), 20)); +const launch = { registry: slowRegistry, base: () => fresh }; +function beginRun(work: string, id: string) { + return () => { + mkdirSync(join(work, id)); + const s = { id, state: "running" as const, started: new Date().toISOString(), tests: [], refs: {}, pid: process.pid }; + writeFileSync(join(work, id, "status.json"), JSON.stringify(s)); + return s; + }; +} +function beginBuild(work: string, id: string) { + return () => { + mkdirSync(join(work, id)); + const b = { id, state: "building" as const, started: new Date().toISOString(), pid: process.pid }; + writeFileSync(join(work, id, "status.json"), JSON.stringify(b)); + return b; + }; +} + +test("a run and a rebuild asked at once: exactly one starts", async () => { + for (const runFirst of [true, false]) { + const work = aLab(); + try { + // Both asked before either answers, in one order and then the other. + const ask = { run: () => launchRun(work, launch, beginRun(work, "run-a")), build: () => launchRebuild(work, launch, beginBuild(work, "base-a")) }; + const [r, b] = runFirst ? await Promise.all([ask.run(), ask.build()]) + : (await Promise.all([ask.build(), ask.run()])).reverse() as [Awaited>, Awaited>]; + assert.equal([r.started, b.started].filter(Boolean).length, 1, `both or neither started: run ${r.started}, rebuild ${b.started}`); + assert.equal(listRuns(work).length + listBaseBuilds(work).length, 1); + } finally { + rmSync(work, { recursive: true, force: true }); + } + } +}); + +test("two runs, or two rebuilds, asked at once: one of each starts", async () => { + const work = aLab(); + try { + const runs = await Promise.all([launchRun(work, launch, beginRun(work, "run-a")), launchRun(work, launch, beginRun(work, "run-b"))]); + assert.deepEqual(runs.map((r) => r.started).sort(), [false, true]); + const other = aLab(); + const builds = await Promise.all([launchRebuild(other, launch, beginBuild(other, "base-a")), launchRebuild(other, launch, beginBuild(other, "base-b"))]); + assert.deepEqual(builds.map((b) => b.started).sort(), [false, true]); + rmSync(other, { recursive: true, force: true }); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +test("a run is refused on a stale base, and starts nothing", async () => { + const work = aLab(); + try { + const stale: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-08-31T09:00:00.000Z", ageDays: 43 }; + const r = await launchRun(work, { registry: slowRegistry, base: () => stale }, beginRun(work, "run-a")); + assert.equal(r.started, false); + assert.match(r.started ? "" : r.reason, /lab_base \{"rebuild": true\}/); + assert.deepEqual(listRuns(work), []); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +test("a rebuild or a run that never recorded its process is lost, and holds nothing up", () => { + const work = aLab(); + try { + mkdirSync(join(work, "base-x")); + writeFileSync(join(work, "base-x", "status.json"), JSON.stringify({ id: "base-x", state: "building", started: "2026-10-14T00:00:00Z" })); + mkdirSync(join(work, "run-x")); + writeFileSync(join(work, "run-x", "status.json"), JSON.stringify({ id: "run-x", state: "running", started: "2026-10-14T00:00:00Z", tests: [], refs: {} })); + assert.equal(listBaseBuilds(work)[0]?.state, "lost"); + assert.equal(buildingBase(work), undefined); + assert.equal(readStatus(work, "run-x")?.state, "lost"); + assert.equal(running(work), undefined); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +test("a rebuild's credential left behind is cleaned, and a running one's is kept", () => { + const work = aLab(); + try { + for (const id of ["base-old", "base-now", "run-old", "run-now"]) writeFileSync(credentialPath(work, id), "x"); + const removed = cleanCredentials(work, [credentialPath(work, "base-now"), credentialPath(work, "run-now")]).sort(); + assert.deepEqual(removed, [credentialPath(work, "base-old"), credentialPath(work, "run-old")].sort()); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +test("lab_check's problem: a stale, missing or unread base image; a fresh one is none", () => { + assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false }) ?? "", /is not on this machine/); + assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false, problem: "incus could not list the lab's images" }) ?? "", /age is not known/); + assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: true, built: "x", ageDays: 30 }) ?? "", /older than the lab's bound/); + assert.equal(baseProblem(fresh), undefined); +}); + +// The tools use the gate and the check above (index.ts registers with the SDK when imported, so it is read). +test("lab_run and lab_base start through the gate, and lab_check names what is wrong with the base", () => { const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8"); - const run = index.slice(index.indexOf('name: "lab_run"'), index.indexOf('name: "lab_base"')); - assert.ok(run.indexOf("runRefusal(base, buildingBase(work))") > 0, "lab_run does not ask about the base"); - assert.ok(run.indexOf("runRefusal(") < run.indexOf("start(work"), "lab_run starts before it asks about the base"); - const check = index.slice(index.indexOf('name: "lab_check"'), index.indexOf('name: "lab_run"')); - assert.match(check, /if \(stale\) problems\.push\(stale\)/); - assert.match(check, /describeBase\(base\)/); - const base = index.slice(index.indexOf('name: "lab_base"'), index.indexOf('name: "lab_status"')); - assert.ok(base.indexOf("if (args.rebuild !== true) return reading;") > 0, "lab_base does not read by default"); - assert.ok(base.indexOf("if (args.rebuild !== true) return reading;") < base.indexOf("startBaseBuild("), "lab_base rebuilds unasked"); - assert.ok(base.indexOf("running(work)") < base.indexOf("startBaseBuild("), "lab_base rebuilds under a run"); + assert.match(index, /await launchRun\(work,/); + assert.match(index, /await launchRebuild\(work,/); + assert.match(index, /const unfit = baseProblem\(base\);\s*if \(unfit\) problems\.push\(unfit\)/); + assert.match(index, /if \(args\.rebuild !== true\) return reading;/); }); diff --git a/modules/lab/tools/base.ts b/modules/lab/tools/base.ts deleted file mode 100644 index ade888e5..00000000 --- a/modules/lab/tools/base.ts +++ /dev/null @@ -1,91 +0,0 @@ -// The lab's base image: how old it is, and rebuilding it (novox/hq issue 698). -// -// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which -// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the -// package database the image was built with. On a rolling distribution the mirrors stop serving what an old -// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror -// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a -// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image -// older than the bound, and rebuilds the image when asked to. -// -// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is -// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds -// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little; -// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test. - -import { spawnSync } from "node:child_process"; - -/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */ -export const BASE_IMAGE_ALIAS = "mesh-lab/base"; - -/** How old the base image may be, in days, before a run is refused (see the head of this file). */ -export const BASE_MAX_AGE_DAYS = 14; - -const DAY = 24 * 60 * 60 * 1000; - -export interface BaseImage { - alias: string; - /** Whether the image is there at all. */ - exists: boolean; - /** When this machine made it, ISO. */ - built?: string; - /** Its age in whole days. */ - ageDays?: number; - /** Why its age could not be read, when it could not. */ - problem?: string; -} - -/** - * baseImageFrom reads `incus image list --format json`: the image whose aliases hold the base's - * name, and when it was made here (its upload, which a publish sets). - */ -export function baseImageFrom(listed: string | undefined, now: Date): BaseImage { - const alias = BASE_IMAGE_ALIAS; - if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" }; - let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>; - try { - images = JSON.parse(listed); - } catch { - return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" }; - } - const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias)); - if (!image) return { alias, exists: false }; - const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN)) - .find((t) => Number.isFinite(t) && t > 0); - if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" }; - return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) }; -} - -/** readBaseImage asks incus, on this machine, for the base image. */ -export function readBaseImage(now = new Date()): BaseImage { - const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" }); - return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now); -} - -/** The base image's age in one sentence, as lab_check and lab_run say it. */ -export function describeBase(b: BaseImage): string { - if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`; - if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`; - return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`; -} - -/** - * staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the - * bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says - * so itself, and a bed that does not need it runs. - */ -export function staleBase(b: BaseImage): string | undefined { - if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined; - return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` + - `${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` + - `(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`; -} - -/** - * runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which - * replaces the image a bed raises its machines from, or an image older than the bound. - */ -export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined { - if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`; - return staleBase(base); -} diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts index 1b7b071b..63c7c41f 100644 --- a/modules/lab/tools/index.ts +++ b/modules/lab/tools/index.ts @@ -9,9 +9,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start, - stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, + stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun, + launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem, } from "./runs.js"; -import { BASE_MAX_AGE_DAYS, describeBase, readBaseImage, runRefusal, staleBase } from "./base.js"; export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs"; @@ -60,15 +60,16 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every // repository it clones answers on the forge, and every one it installs has a lock file where the run // installs from. - // A credential a stopped, crashed or rebooted run left behind is removed, and said; one still in use - // by a running run is not. - const busy = running(work); - const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined); - const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : ""; - // **And how old the base image is** (novox/hq issue 698): one past the bound is a problem, as lab_run refuses it. + // A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in + // use by a running run or rebuild is not. + const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id); + const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id))); + const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : ""; + // **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot + // be read is a problem here, though lab_run refuses only the first. const base = readBaseImage(); - const stale = staleBase(base); - if (stale) problems.push(stale); + const unfit = baseProblem(base); + if (unfit) problems.push(unfit); const output = [`${check.stdout}${check.stderr}`.trim(), describeBase(base), removed, ...leftovers, ...problems] .filter(Boolean).join("\n"); return { ok: check.status === 0 && problems.length === 0, output }; @@ -102,15 +103,13 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { } const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r)); if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` }; - const busy = running(work); - if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy }; - // **Not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698). - const base = readBaseImage(); - const refused = runRefusal(base, buildingBase(work)); - if (refused) return { started: false, reason: refused, base }; - const registry = await registryOf(forge); - return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry), - base: describeBase(base), registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) }; + // **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698, + // review 256): the registry asked first, then the check and the start with nothing awaited between. + const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() }, + (registry) => start(work, forge, tests, refs, credentialOf(forge), registry)); + if (!launched.started) return launched; + return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry, + ...(launched.problem ? { problem: launched.problem } : {}) }; }, }, { @@ -129,19 +128,18 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const newest = builds[0]; const reading = { base: before, said: describeBase(before), bound: BASE_MAX_AGE_DAYS, - ...(staleBase(before) ? { stale: staleBase(before) } : {}), + ...(baseProblem(before) ? { problem: baseProblem(before) } : {}), builds, ...(newest ? { log: baseBuildLog(work, newest.id) } : {}), }; if (args.rebuild !== true) return reading; const forge = forgeOf(); if (!forge) return { ...reading, started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; - const rebuilding = buildingBase(work); - if (rebuilding) return { ...reading, started: false, reason: `${rebuilding.id} is still rebuilding it` }; - // A run raises its machines from the image; replacing it under one would change what the run tests. - const busy = running(work); - if (busy) return { ...reading, started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image` }; - const registry = await registryOf(forge); - return { ...reading, started: true, build: startBaseBuild(work, forge, credentialOf(forge), registry.registry) }; + // Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with + // nothing awaited between (review 256). + const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before }, + (registry) => startBaseBuild(work, forge, credentialOf(forge), registry)); + if (launched.started === false) return { ...reading, started: false, reason: launched.reason }; + return { ...reading, started: true, build: launched.job }; }, }, { diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts index 14ce72d3..408db9d7 100644 --- a/modules/lab/tools/runs.ts +++ b/modules/lab/tools/runs.ts @@ -6,7 +6,7 @@ // and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers // for it, and says it was lost rather than pretending it is still going. -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -223,16 +223,18 @@ trap ${shellQuote(`${rm}; exit 129`)} HUP`; } /** - * cleanCredentials removes every run's credential file left in the work directory but the one named — what - * a crash or a reboot left behind, which no trap could remove. Answers what it removed. + * cleanCredentials removes every run's and every base rebuild's credential file left in the work directory + * but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it + * removed. */ -export function cleanCredentials(work: string, keep?: string): string[] { +export function cleanCredentials(work: string, keep?: string | string[]): string[] { if (!existsSync(work)) return []; + const kept = new Set([keep ?? []].flat()); const removed: string[] = []; for (const name of readdirSync(work)) { - if (!/^\.run-.*\.git-credentials$/.test(name)) continue; + if (!/^\.(run|base)-.*\.git-credentials$/.test(name)) continue; const path = join(work, name); - if (path === keep) continue; + if (kept.has(path)) continue; rmSync(path, { force: true }); removed.push(path); } @@ -300,8 +302,10 @@ export function readStatus(work: string, id: string): RunStatus | undefined { try { const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus; // A run whose process is gone while its status still says it is going was lost — the runtime or - // the machine restarted under it. Said, rather than left reading as running for ever. - if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) { + // the machine restarted under it. Said, rather than left reading as running for ever. One that never + // recorded a process (the runtime died between writing its status and starting it) is lost too: it + // would otherwise hold up every run after it (review 256). + if (!["passed", "failed", "stopped", "lost"].includes(s.state) && gone(s.pid)) { s.state = "lost"; } return s; @@ -310,6 +314,11 @@ export function readStatus(work: string, id: string): RunStatus | undefined { } } +/** gone is whether a job's process is not running: none recorded, or none by that id. */ +function gone(pid: number | undefined): boolean { + return !pid || !alive(pid); +} + function alive(pid: number): boolean { try { process.kill(pid, 0); @@ -447,7 +456,106 @@ export function tail(work: string, id: string, lines: number): string { } } -// ---- the base image's rebuild (novox/hq issue 698): mesh-lab's `base build`, in the background, as a run is. +// ---- the base image (novox/hq issue 698) +// +// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which +// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the +// package database the image was built with. On a rolling distribution the mirrors stop serving what an old +// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror +// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a +// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image +// older than the bound, and rebuilds the image when asked to. +// +// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is +// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds +// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little; +// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test. + +/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */ +export const BASE_IMAGE_ALIAS = "mesh-lab/base"; + +/** How old the base image may be, in days, before a run is refused (see the head of this file). */ +export const BASE_MAX_AGE_DAYS = 14; + +const DAY = 24 * 60 * 60 * 1000; + +export interface BaseImage { + alias: string; + /** Whether the image is there at all. */ + exists: boolean; + /** When this machine made it, ISO. */ + built?: string; + /** Its age in whole days. */ + ageDays?: number; + /** Why its age could not be read, when it could not. */ + problem?: string; +} + +/** + * baseImageFrom reads `incus image list --format json`: the image whose aliases hold the base's + * name, and when it was made here (its upload, which a publish sets). + */ +export function baseImageFrom(listed: string | undefined, now: Date): BaseImage { + const alias = BASE_IMAGE_ALIAS; + if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" }; + let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>; + try { + images = JSON.parse(listed); + } catch { + return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" }; + } + const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias)); + if (!image) return { alias, exists: false }; + const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN)) + .find((t) => Number.isFinite(t) && t > 0); + if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" }; + return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) }; +} + +/** readBaseImage asks incus, on this machine, for the base image. */ +export function readBaseImage(now = new Date()): BaseImage { + const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" }); + return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now); +} + +/** The base image's age in one sentence, as lab_check and lab_run say it. */ +export function describeBase(b: BaseImage): string { + if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`; + if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`; + return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`; +} + +/** + * staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the + * bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says + * so itself, and a bed that does not need it runs. + */ +export function staleBase(b: BaseImage): string | undefined { + if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined; + return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` + + `${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` + + `(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`; +} + +/** + * baseProblem is what lab_check and lab_base say is wrong with the base image, or undefined: past the bound, not + * on this machine, or of an age that cannot be read (review 256). Only the first refuses a run. + */ +export function baseProblem(b: BaseImage): string | undefined { + if (b.problem || !b.exists) return describeBase(b); + return staleBase(b); +} + +/** + * runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which + * replaces the image a bed raises its machines from, or an image older than the bound. + */ +export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined { + if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`; + return staleBase(base); +} + +// ---- its rebuild: mesh-lab's `base build`, in the background, as a run is. export interface BaseBuild { id: string; @@ -469,7 +577,7 @@ export function listBaseBuilds(work: string): BaseBuild[] { for (const id of readdirSync(work).filter((d) => d.startsWith("base-"))) { try { const b = JSON.parse(readFileSync(buildStatusPath(work, id), "utf8")) as BaseBuild; - if (b.state === "building" && b.pid && !alive(b.pid)) b.state = "lost"; + if (b.state === "building" && gone(b.pid)) b.state = "lost"; builds.push(b); } catch { // A directory without a status is not a build. @@ -520,13 +628,15 @@ export function startBaseBuild(work: string, forge: string, credential?: string, mkdirSync(dir, { recursive: true }); const build: BaseBuild = { id, state: "building", started: new Date().toISOString() }; writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2)); - const credentials = writeCredential(join(work, `.${id}.git-credentials`), credential); + // What a crashed or rebooted run or rebuild left behind goes before this one writes its own. + cleanCredentials(work); + const credentials = writeCredential(credentialPath(work, id), credential); writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 }); const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$? node -e ${shellQuote( `const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, )} "$code" -rm -f ${shellQuote(join(work, `.${id}.git-credentials`))} +rm -f ${shellQuote(credentialPath(work, id))} rm -rf ${shellQuote(join(dir, "mesh-lab"))}`; const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); child.unref(); @@ -534,3 +644,49 @@ rm -rf ${shellQuote(join(dir, "mesh-lab"))}`; writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2)); return build; } + +// ---- starting a run or a rebuild: one job on the lab's machine at a time (review 256). +// +// **Everything that waits is done first; the check and the start are one step.** A run and a rebuild each +// refused while the other was going, but each asked the package registry — an await — between its check and +// writing its status, so a run and a rebuild asked together could both pass their checks and both start: a +// bed raising machines from an image being replaced under it. Here the registry is asked first, and nothing +// is awaited between reading what is going and writing the new job's status, so in the one runtime that +// serves the lab's tools no other call runs in between. + +export interface Launch { + /** The address a run or a rebuild names the package registry by, asked of the forge (npmRegistry). */ + registry: () => Promise<{ registry: string; problem?: string }>; + /** The base image as it is now. */ + base: () => BaseImage; +} + +export type Launched = { started: true; registry: string; problem?: string; base: BaseImage; job: T } + | { started: false; reason: string; base: BaseImage }; + +/** launchRun starts a run unless a run or a rebuild is going, or the base image is past its bound. */ +export async function launchRun(work: string, launch: Launch, begin: (registry: string) => RunStatus): + Promise> { + const registry = await launch.registry(); + // From here to begin(): nothing awaited. + const base = launch.base(); + const busy = running(work); + if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, base }; + const refused = runRefusal(base, buildingBase(work)); + if (refused) return { started: false, reason: refused, base }; + return { started: true, ...registry, base, job: begin(registry.registry) }; +} + +/** launchRebuild starts a rebuild of the base image unless a run or another rebuild is going. */ +export async function launchRebuild(work: string, launch: Launch, begin: (registry: string) => BaseBuild): + Promise> { + const registry = await launch.registry(); + // From here to begin(): nothing awaited. + const base = launch.base(); + const rebuilding = buildingBase(work); + if (rebuilding) return { started: false, reason: `${rebuilding.id} is still rebuilding it`, base }; + // A run raises its machines from the image; replacing it under one would change what the run tests. + const busy = running(work); + if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image`, base }; + return { started: true, ...registry, base, job: begin(registry.registry) }; +} diff --git a/modules/lab/tsconfig.json b/modules/lab/tsconfig.json index 002e8e27..f4f306fe 100644 --- a/modules/lab/tsconfig.json +++ b/modules/lab/tsconfig.json @@ -8,5 +8,5 @@ "skipLibCheck": true, "noEmit": true }, - "include": ["tools/index.ts", "tools/runs.ts", "tools/base.ts"] + "include": ["tools/index.ts", "tools/runs.ts"] }