From ff9d3ee6a06c03a85e94b324aec375d39f4ccc26 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 03:21:23 +0200 Subject: [PATCH] lab: name the base image's age, refuse a stale one, and rebuild it (issue 698) A bed machine carries the package database its base image was built with, and the mirrors stop serving it within weeks: genesis-keeps-its-bus failed on a 404 from an image 44 days old, with nothing saying its age and no mesh tool to rebuild it. lab_check and lab_run now say the age, lab_run refuses an image older than 14 days and names the remedy, and lab_base reads the image and rebuilds it with mesh-lab's base build when asked. --- modules/lab/test/base.test.ts | 98 +++++++++++++++++++++++++++++++++++ modules/lab/tools/base.ts | 91 ++++++++++++++++++++++++++++++++ modules/lab/tools/index.ts | 47 +++++++++++++++-- modules/lab/tools/runs.ts | 88 +++++++++++++++++++++++++++++++ modules/lab/tsconfig.json | 2 +- 5 files changed, 322 insertions(+), 4 deletions(-) create mode 100644 modules/lab/test/base.test.ts create mode 100644 modules/lab/tools/base.ts diff --git a/modules/lab/test/base.test.ts b/modules/lab/test/base.test.ts new file mode 100644 index 00000000..f05bc8a3 --- /dev/null +++ b/modules/lab/test/base.test.ts @@ -0,0 +1,98 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, describeBase, runRefusal, staleBase } from "../tools/base.ts"; +import { baseScript, buildingBase, listBaseBuilds, listRuns } from "../tools/runs.ts"; + +// Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a +// package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for +// postgresql-libs. Nothing said how old the image was, nothing refused it, and no mesh tool rebuilt it. + +const now = new Date("2026-10-14T01:00:00Z"); +const listed = (uploaded: string, alias = BASE_IMAGE_ALIAS) => JSON.stringify([ + { aliases: [{ name: "images:other" }], uploaded_at: "2026-10-13T00:00:00Z" }, + { aliases: [{ name: alias }], uploaded_at: uploaded, created_at: "2026-08-01T00:00:00Z" }, +]); + +test("the base image's age is read from when this machine made it", () => { + const b = baseImageFrom(listed("2026-08-31T09:00:00Z"), now); + assert.equal(b.exists, true); + assert.equal(b.built, "2026-08-31T09:00:00.000Z"); + assert.equal(b.ageDays, 43); + assert.match(describeBase(b), /built 2026-08-31T09:00:00.000Z \(43 day\(s\) ago; the bound is 14\)/); +}); + +test("an image older than the bound refuses a run, naming the tool that rebuilds it", () => { + const stale = baseImageFrom(listed("2026-08-31T09:00:00Z"), now); + const reason = runRefusal(stale); + assert.ok(reason, "a 43-day-old base image was not refused"); + assert.match(reason, /older than the lab's bound of 14 days/); + assert.match(reason, /lab_base \{"rebuild": true\}/); + assert.equal(staleBase(stale), reason); +}); + +test("an image within the bound runs", () => { + const fresh = baseImageFrom(listed(new Date(now.getTime() - BASE_MAX_AGE_DAYS * 86_400_000).toISOString()), now); + assert.equal(fresh.ageDays, BASE_MAX_AGE_DAYS); + assert.equal(runRefusal(fresh), undefined); +}); + +test("a missing image, or one whose age cannot be read, is said and not refused", () => { + const none = baseImageFrom(listed("2026-10-01T00:00:00Z", "something-else"), now); + assert.equal(none.exists, false); + assert.equal(runRefusal(none), undefined); + assert.match(describeBase(none), /is not on this machine; rebuild it with lab_base/); + const unread = baseImageFrom(undefined, now); + assert.equal(runRefusal(unread), undefined); + assert.match(describeBase(unread), /its age is not known/); +}); + +test("no run starts while the base image is rebuilt", () => { + const fresh = baseImageFrom(listed("2026-10-13T00:00:00Z"), now); + const reason = runRefusal(fresh, { id: "base-x", started: "2026-10-14T00:59:00Z" }); + assert.match(reason ?? "", /is being rebuilt \(base-x/); +}); + +test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => { + const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials"); + assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/); + assert.match(s, /src\/cli.ts base build/); + assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone"); +}); + +test("a rebuild is known by its status, and is not a run", () => { + const work = mkdtempSync(join(tmpdir(), "lab-base-")); + try { + mkdirSync(join(work, "base-1")); + writeFileSync(join(work, "base-1", "status.json"), JSON.stringify({ id: "base-1", state: "built", started: "2026-10-13T00:00:00Z" })); + mkdirSync(join(work, "base-2")); + // A build whose process is gone while it says building was lost, and holds nothing up. + writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: 2 ** 22 + 7 })); + assert.deepEqual(listBaseBuilds(work).map((b) => `${b.id} ${b.state}`), ["base-2 lost", "base-1 built"]); + assert.equal(buildingBase(work), undefined); + writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: process.pid })); + assert.equal(buildingBase(work)?.id, "base-2"); + assert.deepEqual(listRuns(work), []); + assert.ok(readFileSync(join(work, "base-1", "status.json"), "utf8")); + } finally { + rmSync(work, { recursive: true, force: true }); + } +}); + +// The tools themselves, as written in index.ts (it registers with the SDK when imported, so it is read). +test("lab_run refuses on the base before it starts anything, lab_check names its age, lab_base reads unless asked", () => { + const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8"); + const run = index.slice(index.indexOf('name: "lab_run"'), index.indexOf('name: "lab_base"')); + assert.ok(run.indexOf("runRefusal(base, buildingBase(work))") > 0, "lab_run does not ask about the base"); + assert.ok(run.indexOf("runRefusal(") < run.indexOf("start(work"), "lab_run starts before it asks about the base"); + const check = index.slice(index.indexOf('name: "lab_check"'), index.indexOf('name: "lab_run"')); + assert.match(check, /if \(stale\) problems\.push\(stale\)/); + assert.match(check, /describeBase\(base\)/); + const base = index.slice(index.indexOf('name: "lab_base"'), index.indexOf('name: "lab_status"')); + assert.ok(base.indexOf("if (args.rebuild !== true) return reading;") > 0, "lab_base does not read by default"); + assert.ok(base.indexOf("if (args.rebuild !== true) return reading;") < base.indexOf("startBaseBuild("), "lab_base rebuilds unasked"); + assert.ok(base.indexOf("running(work)") < base.indexOf("startBaseBuild("), "lab_base rebuilds under a run"); +}); diff --git a/modules/lab/tools/base.ts b/modules/lab/tools/base.ts new file mode 100644 index 00000000..ade888e5 --- /dev/null +++ b/modules/lab/tools/base.ts @@ -0,0 +1,91 @@ +// The lab's base image: how old it is, and rebuilding it (novox/hq issue 698). +// +// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which +// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the +// package database the image was built with. On a rolling distribution the mirrors stop serving what an old +// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror +// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a +// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image +// older than the bound, and rebuilds the image when asked to. +// +// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is +// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds +// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little; +// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test. + +import { spawnSync } from "node:child_process"; + +/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */ +export const BASE_IMAGE_ALIAS = "mesh-lab/base"; + +/** How old the base image may be, in days, before a run is refused (see the head of this file). */ +export const BASE_MAX_AGE_DAYS = 14; + +const DAY = 24 * 60 * 60 * 1000; + +export interface BaseImage { + alias: string; + /** Whether the image is there at all. */ + exists: boolean; + /** When this machine made it, ISO. */ + built?: string; + /** Its age in whole days. */ + ageDays?: number; + /** Why its age could not be read, when it could not. */ + problem?: string; +} + +/** + * baseImageFrom reads `incus image list --format json`: the image whose aliases hold the base's + * name, and when it was made here (its upload, which a publish sets). + */ +export function baseImageFrom(listed: string | undefined, now: Date): BaseImage { + const alias = BASE_IMAGE_ALIAS; + if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" }; + let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>; + try { + images = JSON.parse(listed); + } catch { + return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" }; + } + const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias)); + if (!image) return { alias, exists: false }; + const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN)) + .find((t) => Number.isFinite(t) && t > 0); + if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" }; + return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) }; +} + +/** readBaseImage asks incus, on this machine, for the base image. */ +export function readBaseImage(now = new Date()): BaseImage { + const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" }); + return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now); +} + +/** The base image's age in one sentence, as lab_check and lab_run say it. */ +export function describeBase(b: BaseImage): string { + if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`; + if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`; + return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`; +} + +/** + * staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the + * bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says + * so itself, and a bed that does not need it runs. + */ +export function staleBase(b: BaseImage): string | undefined { + if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined; + return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` + + `${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` + + `(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`; +} + +/** + * runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which + * replaces the image a bed raises its machines from, or an image older than the bound. + */ +export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined { + if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`; + return staleBase(base); +} diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts index 4511164a..1b7b071b 100644 --- a/modules/lab/tools/index.ts +++ b/modules/lab/tools/index.ts @@ -9,8 +9,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start, - stop, tail, writeCredential, + stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, } from "./runs.js"; +import { BASE_MAX_AGE_DAYS, describeBase, readBaseImage, runRefusal, staleBase } from "./base.js"; export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs"; @@ -64,7 +65,12 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const busy = running(work); const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined); const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : ""; - const output = [`${check.stdout}${check.stderr}`.trim(), removed, ...leftovers, ...problems].filter(Boolean).join("\n"); + // **And how old the base image is** (novox/hq issue 698): one past the bound is a problem, as lab_run refuses it. + const base = readBaseImage(); + const stale = staleBase(base); + if (stale) problems.push(stale); + const output = [`${check.stdout}${check.stderr}`.trim(), describeBase(base), removed, ...leftovers, ...problems] + .filter(Boolean).join("\n"); return { ok: check.status === 0 && problems.length === 0, output }; }, }, @@ -98,9 +104,44 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` }; const busy = running(work); if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy }; + // **Not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698). + const base = readBaseImage(); + const refused = runRefusal(base, buildingBase(work)); + if (refused) return { started: false, reason: refused, base }; const registry = await registryOf(forge); return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry), - registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) }; + base: describeBase(base), registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) }; + }, + }, + { + name: "lab_base", + description: + "The lab's base image, which every bed machine with a container runtime is raised from: when it was built, its " + + `age, the bound lab_run refuses past (${BASE_MAX_AGE_DAYS} days), and the last rebuilds with the newest one's log. ` + + "Reads only, unless rebuild is true: then it rebuilds the image with mesh-lab's `base build` from the forge's " + + "main, in the background, and answers at once; ask again for the build's state and the image's new age.", + input: { + rebuild: { type: "boolean", description: "rebuild the image (default false: only read)" }, + }, + run: async (args) => { + const before = readBaseImage(); + const builds = listBaseBuilds(work).slice(0, 3); + const newest = builds[0]; + const reading = { + base: before, said: describeBase(before), bound: BASE_MAX_AGE_DAYS, + ...(staleBase(before) ? { stale: staleBase(before) } : {}), + builds, ...(newest ? { log: baseBuildLog(work, newest.id) } : {}), + }; + if (args.rebuild !== true) return reading; + const forge = forgeOf(); + if (!forge) return { ...reading, started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; + const rebuilding = buildingBase(work); + if (rebuilding) return { ...reading, started: false, reason: `${rebuilding.id} is still rebuilding it` }; + // A run raises its machines from the image; replacing it under one would change what the run tests. + const busy = running(work); + if (busy) return { ...reading, started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image` }; + const registry = await registryOf(forge); + return { ...reading, started: true, build: startBaseBuild(work, forge, credentialOf(forge), registry.registry) }; }, }, { diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts index e3eb1b9d..14ce72d3 100644 --- a/modules/lab/tools/runs.ts +++ b/modules/lab/tools/runs.ts @@ -446,3 +446,91 @@ export function tail(work: string, id: string, lines: number): string { return ""; } } + +// ---- the base image's rebuild (novox/hq issue 698): mesh-lab's `base build`, in the background, as a run is. + +export interface BaseBuild { + id: string; + state: "building" | "built" | "failed" | "lost"; + started: string; + ended?: string; + exit?: number; + pid?: number; +} + +function buildStatusPath(work: string, id: string): string { + return join(work, id, "status.json"); +} + +/** The base builds, newest first. A build whose process is gone while it says building was lost. */ +export function listBaseBuilds(work: string): BaseBuild[] { + if (!existsSync(work)) return []; + const builds: BaseBuild[] = []; + for (const id of readdirSync(work).filter((d) => d.startsWith("base-"))) { + try { + const b = JSON.parse(readFileSync(buildStatusPath(work, id), "utf8")) as BaseBuild; + if (b.state === "building" && b.pid && !alive(b.pid)) b.state = "lost"; + builds.push(b); + } catch { + // A directory without a status is not a build. + } + } + return builds.sort((a, b) => b.started.localeCompare(a.started)); +} + +/** The base build still going, if any. */ +export function buildingBase(work: string): BaseBuild | undefined { + return listBaseBuilds(work).find((b) => b.state === "building"); +} + +/** The last lines of a base build's log. */ +export function baseBuildLog(work: string, id: string, lines = 40): string { + try { + return readFileSync(join(work, id, "build.log"), "utf8").split("\n").slice(-Math.max(1, lines)).join("\n"); + } catch { + return ""; + } +} + +/** + * baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it, + * then its `base build`, which replaces the image and reads back that it works before publishing it. + */ +export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string { + const lab = join(dir, "mesh-lab"); + return `set -euo pipefail +export GIT_TERMINAL_PROMPT=0 +${credentialGuard(credential)} +cd ${shellQuote(dir)} +git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)} +${credential ? `rm -f ${shellQuote(credential)}` : ""} +echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)" +printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))} +export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))} +${installStep(lab, false)} +cd ${shellQuote(lab)} +node --experimental-strip-types src/cli.ts base build +`; +} + +/** startBaseBuild rebuilds the base image in the background and answers at once with the build. */ +export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild { + const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`; + const dir = join(work, id); + mkdirSync(dir, { recursive: true }); + const build: BaseBuild = { id, state: "building", started: new Date().toISOString() }; + writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2)); + const credentials = writeCredential(join(work, `.${id}.git-credentials`), credential); + writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 }); + const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$? +node -e ${shellQuote( + `const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, + )} "$code" +rm -f ${shellQuote(join(work, `.${id}.git-credentials`))} +rm -rf ${shellQuote(join(dir, "mesh-lab"))}`; + const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); + child.unref(); + build.pid = child.pid; + writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2)); + return build; +} diff --git a/modules/lab/tsconfig.json b/modules/lab/tsconfig.json index f4f306fe..002e8e27 100644 --- a/modules/lab/tsconfig.json +++ b/modules/lab/tsconfig.json @@ -8,5 +8,5 @@ "skipLibCheck": true, "noEmit": true }, - "include": ["tools/index.ts", "tools/runs.ts"] + "include": ["tools/index.ts", "tools/runs.ts", "tools/base.ts"] }