diff --git a/modules/lab/test/base.test.ts b/modules/lab/test/base.test.ts new file mode 100644 index 00000000..93a6f8ec --- /dev/null +++ b/modules/lab/test/base.test.ts @@ -0,0 +1,198 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, baseProblem, baseScript, buildingBase, cleanCredentials, credentialPath, + describeBase, launchRebuild, launchRun, listBaseBuilds, listRuns, readStatus, runRefusal, running, staleBase, + type BaseImage, +} from "../tools/runs.ts"; + +// Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a +// package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for +// postgresql-libs. Nothing said how old the image was, nothing refused it, and no mesh tool rebuilt it. + +const now = new Date("2026-10-14T01:00:00Z"); +const listed = (uploaded: string, alias = BASE_IMAGE_ALIAS) => JSON.stringify([ + { aliases: [{ name: "images:other" }], uploaded_at: "2026-10-13T00:00:00Z" }, + { aliases: [{ name: alias }], uploaded_at: uploaded, created_at: "2026-08-01T00:00:00Z" }, +]); + +test("the base image's age is read from when this machine made it", () => { + const b = baseImageFrom(listed("2026-08-31T09:00:00Z"), now); + assert.equal(b.exists, true); + assert.equal(b.built, "2026-08-31T09:00:00.000Z"); + assert.equal(b.ageDays, 43); + assert.match(describeBase(b), /built 2026-08-31T09:00:00.000Z \(43 day\(s\) ago; the bound is 14\)/); +}); + +test("an image older than the bound refuses a run, naming the tool that rebuilds it", () => { + const stale = baseImageFrom(listed("2026-08-31T09:00:00Z"), now); + const reason = runRefusal(stale); + assert.ok(reason, "a 43-day-old base image was not refused"); + assert.match(reason, /older than the lab's bound of 14 days/); + assert.match(reason, /lab_base \{"rebuild": true\}/); + assert.equal(staleBase(stale), reason); +}); + +test("an image within the bound runs", () => { + const fresh = baseImageFrom(listed(new Date(now.getTime() - BASE_MAX_AGE_DAYS * 86_400_000).toISOString()), now); + assert.equal(fresh.ageDays, BASE_MAX_AGE_DAYS); + assert.equal(runRefusal(fresh), undefined); +}); + +test("a missing image, or one whose age cannot be read, is said and not refused", () => { + const none = baseImageFrom(listed("2026-10-01T00:00:00Z", "something-else"), now); + assert.equal(none.exists, false); + assert.equal(runRefusal(none), undefined); + assert.match(describeBase(none), /is not on this machine; rebuild it with lab_base/); + const unread = baseImageFrom(undefined, now); + assert.equal(runRefusal(unread), undefined); + assert.match(describeBase(unread), /its age is not known/); +}); + +test("no run starts while the base image is rebuilt", () => { + const fresh = baseImageFrom(listed("2026-10-13T00:00:00Z"), now); + const reason = runRefusal(fresh, { id: "base-x", started: "2026-10-14T00:59:00Z" }); + assert.match(reason ?? "", /is being rebuilt \(base-x/); +}); + +test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => { + const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials"); + assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/); + assert.match(s, /src\/cli.ts base build/); + assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone"); +}); + +test("a rebuild is known by its status, and is not a run", () => { + const work = mkdtempSync(join(tmpdir(), "lab-base-")); + try { + mkdirSync(join(work, "base-1")); + writeFileSync(join(work, "base-1", "status.json"), JSON.stringify({ id: "base-1", state: "built", started: "2026-10-13T00:00:00Z" })); + mkdirSync(join(work, "base-2")); + // A build whose process is gone while it says building was lost, and holds nothing up. + writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: 2 ** 22 + 7 })); + assert.deepEqual(listBaseBuilds(work).map((b) => `${b.id} ${b.state}`), ["base-2 lost", "base-1 built"]); + assert.equal(buildingBase(work), undefined); + writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: process.pid })); + assert.equal(buildingBase(work)?.id, "base-2"); + assert.deepEqual(listRuns(work), []); + assert.ok(readFileSync(join(work, "base-1", "status.json"), "utf8")); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +// **A run and a rebuild asked together: one starts** (review 256). Each asks the package registry, which +// waits, and each refused while the other was going; asked at once, both passed their checks before either +// wrote its status. The registry is asked first now, and the check and the start are one step. +function aLab(): string { + return mkdtempSync(join(tmpdir(), "lab-launch-")); +} +const fresh: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-10-13T00:00:00.000Z", ageDays: 1 }; +const slowRegistry = () => new Promise<{ registry: string }>((ok) => setTimeout(() => ok({ registry: "https://forge.invalid/r/" }), 20)); +const launch = { registry: slowRegistry, base: () => fresh }; +function beginRun(work: string, id: string) { + return () => { + mkdirSync(join(work, id)); + const s = { id, state: "running" as const, started: new Date().toISOString(), tests: [], refs: {}, pid: process.pid }; + writeFileSync(join(work, id, "status.json"), JSON.stringify(s)); + return s; + }; +} +function beginBuild(work: string, id: string) { + return () => { + mkdirSync(join(work, id)); + const b = { id, state: "building" as const, started: new Date().toISOString(), pid: process.pid }; + writeFileSync(join(work, id, "status.json"), JSON.stringify(b)); + return b; + }; +} + +test("a run and a rebuild asked at once: exactly one starts", async () => { + for (const runFirst of [true, false]) { + const work = aLab(); + try { + // Both asked before either answers, in one order and then the other. + const ask = { run: () => launchRun(work, launch, beginRun(work, "run-a")), build: () => launchRebuild(work, launch, beginBuild(work, "base-a")) }; + const [r, b] = runFirst ? await Promise.all([ask.run(), ask.build()]) + : (await Promise.all([ask.build(), ask.run()])).reverse() as [Awaited>, Awaited>]; + assert.equal([r.started, b.started].filter(Boolean).length, 1, `both or neither started: run ${r.started}, rebuild ${b.started}`); + assert.equal(listRuns(work).length + listBaseBuilds(work).length, 1); + } finally { + rmSync(work, { recursive: true, force: true }); + } + } +}); + +test("two runs, or two rebuilds, asked at once: one of each starts", async () => { + const work = aLab(); + try { + const runs = await Promise.all([launchRun(work, launch, beginRun(work, "run-a")), launchRun(work, launch, beginRun(work, "run-b"))]); + assert.deepEqual(runs.map((r) => r.started).sort(), [false, true]); + const other = aLab(); + const builds = await Promise.all([launchRebuild(other, launch, beginBuild(other, "base-a")), launchRebuild(other, launch, beginBuild(other, "base-b"))]); + assert.deepEqual(builds.map((b) => b.started).sort(), [false, true]); + rmSync(other, { recursive: true, force: true }); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +test("a run is refused on a stale base, and starts nothing", async () => { + const work = aLab(); + try { + const stale: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-08-31T09:00:00.000Z", ageDays: 43 }; + const r = await launchRun(work, { registry: slowRegistry, base: () => stale }, beginRun(work, "run-a")); + assert.equal(r.started, false); + assert.match(r.started ? "" : r.reason, /lab_base \{"rebuild": true\}/); + assert.deepEqual(listRuns(work), []); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +test("a rebuild or a run that never recorded its process is lost, and holds nothing up", () => { + const work = aLab(); + try { + mkdirSync(join(work, "base-x")); + writeFileSync(join(work, "base-x", "status.json"), JSON.stringify({ id: "base-x", state: "building", started: "2026-10-14T00:00:00Z" })); + mkdirSync(join(work, "run-x")); + writeFileSync(join(work, "run-x", "status.json"), JSON.stringify({ id: "run-x", state: "running", started: "2026-10-14T00:00:00Z", tests: [], refs: {} })); + assert.equal(listBaseBuilds(work)[0]?.state, "lost"); + assert.equal(buildingBase(work), undefined); + assert.equal(readStatus(work, "run-x")?.state, "lost"); + assert.equal(running(work), undefined); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +test("a rebuild's credential left behind is cleaned, and a running one's is kept", () => { + const work = aLab(); + try { + for (const id of ["base-old", "base-now", "run-old", "run-now"]) writeFileSync(credentialPath(work, id), "x"); + const removed = cleanCredentials(work, [credentialPath(work, "base-now"), credentialPath(work, "run-now")]).sort(); + assert.deepEqual(removed, [credentialPath(work, "base-old"), credentialPath(work, "run-old")].sort()); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +test("lab_check's problem: a stale, missing or unread base image; a fresh one is none", () => { + assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false }) ?? "", /is not on this machine/); + assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false, problem: "incus could not list the lab's images" }) ?? "", /age is not known/); + assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: true, built: "x", ageDays: 30 }) ?? "", /older than the lab's bound/); + assert.equal(baseProblem(fresh), undefined); +}); + +// The tools use the gate and the check above (index.ts registers with the SDK when imported, so it is read). +test("lab_run and lab_base start through the gate, and lab_check names what is wrong with the base", () => { + const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8"); + assert.match(index, /await launchRun\(work,/); + assert.match(index, /await launchRebuild\(work,/); + assert.match(index, /const unfit = baseProblem\(base\);\s*if \(unfit\) problems\.push\(unfit\)/); + assert.match(index, /if \(args\.rebuild !== true\) return reading;/); +}); diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts index 4511164a..63c7c41f 100644 --- a/modules/lab/tools/index.ts +++ b/modules/lab/tools/index.ts @@ -9,7 +9,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start, - stop, tail, writeCredential, + stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun, + launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem, } from "./runs.js"; export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { @@ -59,12 +60,18 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every // repository it clones answers on the forge, and every one it installs has a lock file where the run // installs from. - // A credential a stopped, crashed or rebooted run left behind is removed, and said; one still in use - // by a running run is not. - const busy = running(work); - const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined); - const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : ""; - const output = [`${check.stdout}${check.stderr}`.trim(), removed, ...leftovers, ...problems].filter(Boolean).join("\n"); + // A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in + // use by a running run or rebuild is not. + const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id); + const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id))); + const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : ""; + // **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot + // be read is a problem here, though lab_run refuses only the first. + const base = readBaseImage(); + const unfit = baseProblem(base); + if (unfit) problems.push(unfit); + const output = [`${check.stdout}${check.stderr}`.trim(), describeBase(base), removed, ...leftovers, ...problems] + .filter(Boolean).join("\n"); return { ok: check.status === 0 && problems.length === 0, output }; }, }, @@ -96,11 +103,43 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { } const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r)); if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` }; - const busy = running(work); - if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy }; - const registry = await registryOf(forge); - return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry), - registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) }; + // **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698, + // review 256): the registry asked first, then the check and the start with nothing awaited between. + const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() }, + (registry) => start(work, forge, tests, refs, credentialOf(forge), registry)); + if (!launched.started) return launched; + return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry, + ...(launched.problem ? { problem: launched.problem } : {}) }; + }, + }, + { + name: "lab_base", + description: + "The lab's base image, which every bed machine with a container runtime is raised from: when it was built, its " + + `age, the bound lab_run refuses past (${BASE_MAX_AGE_DAYS} days), and the last rebuilds with the newest one's log. ` + + "Reads only, unless rebuild is true: then it rebuilds the image with mesh-lab's `base build` from the forge's " + + "main, in the background, and answers at once; ask again for the build's state and the image's new age.", + input: { + rebuild: { type: "boolean", description: "rebuild the image (default false: only read)" }, + }, + run: async (args) => { + const before = readBaseImage(); + const builds = listBaseBuilds(work).slice(0, 3); + const newest = builds[0]; + const reading = { + base: before, said: describeBase(before), bound: BASE_MAX_AGE_DAYS, + ...(baseProblem(before) ? { problem: baseProblem(before) } : {}), + builds, ...(newest ? { log: baseBuildLog(work, newest.id) } : {}), + }; + if (args.rebuild !== true) return reading; + const forge = forgeOf(); + if (!forge) return { ...reading, started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; + // Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with + // nothing awaited between (review 256). + const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before }, + (registry) => startBaseBuild(work, forge, credentialOf(forge), registry)); + if (launched.started === false) return { ...reading, started: false, reason: launched.reason }; + return { ...reading, started: true, build: launched.job }; }, }, { diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts index 3e96ae51..c65bfb41 100644 --- a/modules/lab/tools/runs.ts +++ b/modules/lab/tools/runs.ts @@ -6,7 +6,7 @@ // and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers // for it, and says it was lost rather than pretending it is still going. -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -223,16 +223,18 @@ trap ${shellQuote(`${rm}; exit 129`)} HUP`; } /** - * cleanCredentials removes every run's credential file left in the work directory but the one named — what - * a crash or a reboot left behind, which no trap could remove. Answers what it removed. + * cleanCredentials removes every run's and every base rebuild's credential file left in the work directory + * but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it + * removed. */ -export function cleanCredentials(work: string, keep?: string): string[] { +export function cleanCredentials(work: string, keep?: string | string[]): string[] { if (!existsSync(work)) return []; + const kept = new Set([keep ?? []].flat()); const removed: string[] = []; for (const name of readdirSync(work)) { - if (!/^\.run-.*\.git-credentials$/.test(name)) continue; + if (!/^\.(run|base)-.*\.git-credentials$/.test(name)) continue; const path = join(work, name); - if (path === keep) continue; + if (kept.has(path)) continue; rmSync(path, { force: true }); removed.push(path); } @@ -411,8 +413,10 @@ export function readStatus(work: string, id: string): RunStatus | undefined { try { const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus; // A run whose process is gone while its status still says it is going was lost — the runtime or - // the machine restarted under it. Said, rather than left reading as running for ever. - if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) { + // the machine restarted under it. Said, rather than left reading as running for ever. One that never + // recorded a process (the runtime died between writing its status and starting it) is lost too: it + // would otherwise hold up every run after it (review 256). + if (!["passed", "failed", "stopped", "lost"].includes(s.state) && gone(s.pid)) { s.state = "lost"; } return settled(work, s); @@ -421,6 +425,11 @@ export function readStatus(work: string, id: string): RunStatus | undefined { } } +/** gone is whether a job's process is not running: none recorded, or none by that id. */ +function gone(pid: number | undefined): boolean { + return !pid || !alive(pid); +} + function alive(pid: number): boolean { try { process.kill(pid, 0); @@ -557,3 +566,238 @@ export function tail(work: string, id: string, lines: number): string { return ""; } } + +// ---- the base image (novox/hq issue 698) +// +// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which +// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the +// package database the image was built with. On a rolling distribution the mirrors stop serving what an old +// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror +// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a +// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image +// older than the bound, and rebuilds the image when asked to. +// +// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is +// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds +// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little; +// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test. + +/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */ +export const BASE_IMAGE_ALIAS = "mesh-lab/base"; + +/** How old the base image may be, in days, before a run is refused (see the head of this file). */ +export const BASE_MAX_AGE_DAYS = 14; + +const DAY = 24 * 60 * 60 * 1000; + +export interface BaseImage { + alias: string; + /** Whether the image is there at all. */ + exists: boolean; + /** When this machine made it, ISO. */ + built?: string; + /** Its age in whole days. */ + ageDays?: number; + /** Why its age could not be read, when it could not. */ + problem?: string; +} + +/** + * baseImageFrom reads `incus image list --format json`: the image whose aliases hold the base's + * name, and when it was made here (its upload, which a publish sets). + */ +export function baseImageFrom(listed: string | undefined, now: Date): BaseImage { + const alias = BASE_IMAGE_ALIAS; + if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" }; + let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>; + try { + images = JSON.parse(listed); + } catch { + return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" }; + } + const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias)); + if (!image) return { alias, exists: false }; + const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN)) + .find((t) => Number.isFinite(t) && t > 0); + if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" }; + return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) }; +} + +/** readBaseImage asks incus, on this machine, for the base image. */ +export function readBaseImage(now = new Date()): BaseImage { + const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" }); + return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now); +} + +/** The base image's age in one sentence, as lab_check and lab_run say it. */ +export function describeBase(b: BaseImage): string { + if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`; + if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`; + return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`; +} + +/** + * staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the + * bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says + * so itself, and a bed that does not need it runs. + */ +export function staleBase(b: BaseImage): string | undefined { + if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined; + return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` + + `${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` + + `(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`; +} + +/** + * baseProblem is what lab_check and lab_base say is wrong with the base image, or undefined: past the bound, not + * on this machine, or of an age that cannot be read (review 256). Only the first refuses a run. + */ +export function baseProblem(b: BaseImage): string | undefined { + if (b.problem || !b.exists) return describeBase(b); + return staleBase(b); +} + +/** + * runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which + * replaces the image a bed raises its machines from, or an image older than the bound. + */ +export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined { + if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`; + return staleBase(base); +} + +// ---- its rebuild: mesh-lab's `base build`, in the background, as a run is. + +export interface BaseBuild { + id: string; + state: "building" | "built" | "failed" | "lost"; + started: string; + ended?: string; + exit?: number; + pid?: number; +} + +function buildStatusPath(work: string, id: string): string { + return join(work, id, "status.json"); +} + +/** The base builds, newest first. A build whose process is gone while it says building was lost. */ +export function listBaseBuilds(work: string): BaseBuild[] { + if (!existsSync(work)) return []; + const builds: BaseBuild[] = []; + for (const id of readdirSync(work).filter((d) => d.startsWith("base-"))) { + try { + const b = JSON.parse(readFileSync(buildStatusPath(work, id), "utf8")) as BaseBuild; + if (b.state === "building" && gone(b.pid)) b.state = "lost"; + builds.push(b); + } catch { + // A directory without a status is not a build. + } + } + return builds.sort((a, b) => b.started.localeCompare(a.started)); +} + +/** The base build still going, if any. */ +export function buildingBase(work: string): BaseBuild | undefined { + return listBaseBuilds(work).find((b) => b.state === "building"); +} + +/** The last lines of a base build's log. */ +export function baseBuildLog(work: string, id: string, lines = 40): string { + try { + return readFileSync(join(work, id, "build.log"), "utf8").split("\n").slice(-Math.max(1, lines)).join("\n"); + } catch { + return ""; + } +} + +/** + * baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it, + * then its `base build`, which replaces the image and reads back that it works before publishing it. + */ +export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string { + const lab = join(dir, "mesh-lab"); + return `set -euo pipefail +export GIT_TERMINAL_PROMPT=0 +${credentialGuard(credential)} +cd ${shellQuote(dir)} +git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)} +${credential ? `rm -f ${shellQuote(credential)}` : ""} +echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)" +printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))} +export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))} +${installStep(lab, false)} +cd ${shellQuote(lab)} +node --experimental-strip-types src/cli.ts base build +`; +} + +/** startBaseBuild rebuilds the base image in the background and answers at once with the build. */ +export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild { + const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`; + const dir = join(work, id); + mkdirSync(dir, { recursive: true }); + const build: BaseBuild = { id, state: "building", started: new Date().toISOString() }; + writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2)); + // What a crashed or rebooted run or rebuild left behind goes before this one writes its own. + cleanCredentials(work); + const credentials = writeCredential(credentialPath(work, id), credential); + writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 }); + const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$? +node -e ${shellQuote( + `const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, + )} "$code" +rm -f ${shellQuote(credentialPath(work, id))} +rm -rf ${shellQuote(join(dir, "mesh-lab"))}`; + const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); + child.unref(); + build.pid = child.pid; + writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2)); + return build; +} + +// ---- starting a run or a rebuild: one job on the lab's machine at a time (review 256). +// +// **Everything that waits is done first; the check and the start are one step.** A run and a rebuild each +// refused while the other was going, but each asked the package registry — an await — between its check and +// writing its status, so a run and a rebuild asked together could both pass their checks and both start: a +// bed raising machines from an image being replaced under it. Here the registry is asked first, and nothing +// is awaited between reading what is going and writing the new job's status, so in the one runtime that +// serves the lab's tools no other call runs in between. + +export interface Launch { + /** The address a run or a rebuild names the package registry by, asked of the forge (npmRegistry). */ + registry: () => Promise<{ registry: string; problem?: string }>; + /** The base image as it is now. */ + base: () => BaseImage; +} + +export type Launched = { started: true; registry: string; problem?: string; base: BaseImage; job: T } + | { started: false; reason: string; base: BaseImage }; + +/** launchRun starts a run unless a run or a rebuild is going, or the base image is past its bound. */ +export async function launchRun(work: string, launch: Launch, begin: (registry: string) => RunStatus): + Promise> { + const registry = await launch.registry(); + // From here to begin(): nothing awaited. + const base = launch.base(); + const busy = running(work); + if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, base }; + const refused = runRefusal(base, buildingBase(work)); + if (refused) return { started: false, reason: refused, base }; + return { started: true, ...registry, base, job: begin(registry.registry) }; +} + +/** launchRebuild starts a rebuild of the base image unless a run or another rebuild is going. */ +export async function launchRebuild(work: string, launch: Launch, begin: (registry: string) => BaseBuild): + Promise> { + const registry = await launch.registry(); + // From here to begin(): nothing awaited. + const base = launch.base(); + const rebuilding = buildingBase(work); + if (rebuilding) return { started: false, reason: `${rebuilding.id} is still rebuilding it`, base }; + // A run raises its machines from the image; replacing it under one would change what the run tests. + const busy = running(work); + if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image`, base }; + return { started: true, ...registry, base, job: begin(registry.registry) }; +}