// lab's tools — the lab, as the mesh asks for it (novox/hq ADR 0172). They run on the machine the // lab is assigned to, and only there: a bed raises virtual machines on that machine's virtualisation. import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start, stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun, launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem, } from "./runs.js"; export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs"; // The forge is an operator's setting, which reaches a file and never a bundle's words (novox/hq // ADR 0192): read from the env-file the mesh fills, at each call, so a changed setting is used // without restarting the runtime. MESH_LAB_FORGE itself still wins, for a hand-run instance. const forgeOf = (): string => (env.MESH_LAB_FORGE ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE")).replace(/\/+$/, ""); // The forge credential the mesh binds and seals to the lab (novox/hq issue 624), read at each call. // The registry's public root the forge serves in the lab's binding (novox/hq ADR 0346), read at each call. const bindingOf = (): string | undefined => env.MESH_LAB_FORGE_BINDING ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING") || undefined); const registryOf = async (forge: string) => { const root = publicRootOf(bindingOf()); const registry = await npmRegistry(forge, fetch, root.root); return registry.problem && root.problem ? { ...registry, problem: `${registry.problem}; ${root.problem}` } : registry; }; const credentialOf = (forge: string): string | undefined => forgeCredential(forge, env.MESH_LAB_FORGE_BINDING ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING"), env.MESH_LAB_FORGE_SECRET ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_SECRET")); return [ { name: "lab_check", description: "Whether this machine can run the lab's beds: the lab's own check, against the forge's main branch.", input: {}, run: async () => { const forge = forgeOf(); if (!forge) return { ok: false, output: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; // What earlier checks left is cleared, and what could not be is said; this check works in a fresh // directory of its own either way, and removes it when it is done. const leftovers = clearChecks(work); const dir = checkDir(work); let check: ReturnType; let problems: string[]; try { const clone = spawnSync("git", ["clone", "--quiet", "--depth", "1", `${forge}/novox/mesh-lab.git`, dir], { encoding: "utf8" }); if (clone.status !== 0) return { ok: false, output: [String(clone.stderr).trim(), ...leftovers].join("\n") }; spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" }); check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" }); problems = setupProblems(forge, `${dir}/setup`, credentialOf(forge)); // And the address a run names the package registry by (npmRegistry): said when the registry does not say it. const registry = await registryOf(forge); if (registry.problem) problems.push(registry.problem); } finally { rmSync(dir, { recursive: true, force: true }); } // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every // repository it clones answers on the forge, and every one it installs has a lock file where the run // installs from. // A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in // use by a running run or rebuild is not. const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id); const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id))); const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : ""; // **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot // be read is a problem here, though lab_run refuses only the first. const base = readBaseImage(); const unfit = baseProblem(base); if (unfit) problems.push(unfit); const output = [`${check.stdout}${check.stderr}`.trim(), describeBase(base), removed, ...leftovers, ...problems] .filter(Boolean).join("\n"); return { ok: check.status === 0 && problems.length === 0, output }; }, }, { name: "lab_run", description: "Run the lab's beds against branches on the forge: fresh checkouts of every repository the lab builds, " + "side by side, then the suite on the named test files. Answers at once with the run's id; lab_status " + "and lab_log follow it. One run at a time.", input: { tests: { type: "string", description: "the bed test files, comma-separated, relative to mesh-lab (e.g. test/integration/mesh.test.ts)" }, refs: { type: "string", description: `a JSON object of repository to branch, for any of ${REPOSITORIES.join(", ")}; the rest run main`, }, }, run: async (args) => { const forge = forgeOf(); if (!forge) return { started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; const tests = String(args.tests ?? "").split(",").map((s) => s.trim()).filter(Boolean); if (tests.length === 0) return { started: false, reason: "name at least one bed test file" }; let refs: Record = {}; if (args.refs) { try { refs = JSON.parse(String(args.refs)) as Record; } catch { return { started: false, reason: "refs is not a JSON object of repository to branch" }; } } const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r)); if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` }; // **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698, // review 256): the registry asked first, then the check and the start with nothing awaited between. const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() }, (registry) => start(work, forge, tests, refs, credentialOf(forge), registry)); if (!launched.started) return launched; return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry, ...(launched.problem ? { problem: launched.problem } : {}) }; }, }, { name: "lab_base", description: "The lab's base image, which every bed machine with a container runtime is raised from: when it was built, its " + `age, the bound lab_run refuses past (${BASE_MAX_AGE_DAYS} days), and the last rebuilds with the newest one's log. ` + "Reads only, unless rebuild is true: then it rebuilds the image with mesh-lab's `base build` from the forge's " + "main, in the background, and answers at once; ask again for the build's state and the image's new age.", input: { rebuild: { type: "boolean", description: "rebuild the image (default false: only read)" }, }, run: async (args) => { const before = readBaseImage(); const builds = listBaseBuilds(work).slice(0, 3); const newest = builds[0]; const reading = { base: before, said: describeBase(before), bound: BASE_MAX_AGE_DAYS, ...(baseProblem(before) ? { problem: baseProblem(before) } : {}), builds, ...(newest ? { log: baseBuildLog(work, newest.id) } : {}), }; if (args.rebuild !== true) return reading; const forge = forgeOf(); if (!forge) return { ...reading, started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; // Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with // nothing awaited between (review 256). const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before }, (registry) => startBaseBuild(work, forge, credentialOf(forge), registry)); if (launched.started === false) return { ...reading, started: false, reason: launched.reason }; return { ...reading, started: true, build: launched.job }; }, }, { name: "lab_status", description: "A run's state, the commits it tested and how it ended — or every run, newest first, when no id is given.", input: { id: { type: "string", description: "the run's id (optional)" } }, run: async (args) => { if (args.id) return readStatus(work, String(args.id)) ?? { found: false, id: String(args.id) }; return { runs: listRuns(work).slice(0, 10) }; }, }, { name: "lab_log", description: "The last lines of a run's log.", input: { id: { type: "string", description: "the run's id" }, lines: { type: "number", description: "how many lines from the end (default 200)" }, }, run: async (args) => ({ id: String(args.id), log: tail(work, String(args.id), Number(args.lines ?? 200)) }), }, { name: "lab_stop", description: "Stop a run and everything it started.", input: { id: { type: "string", description: "the run's id" } }, run: async (args) => stop(work, String(args.id)) ?? { found: false, id: String(args.id) }, }, ]; } /** * What would stop a run at its setup, in sentences: a repository the run clones that the forge does not * answer for, and a checkout it installs with nothing installable (installPlan). Empty when a run's setup * would go through. */ export function setupProblems(forge: string, dir: string, credential?: string): string[] { const problems: string[] = []; spawnSync("rm", ["-rf", dir]); // The credential a run would offer, in a file of its own outside the check's clones, removed after. const keep = mkdtempSync(join(tmpdir(), "lab-check-")); const file = writeCredential(join(keep, "git-credentials"), credential); const git = (args: string[]) => spawnSync("sh", ["-c", `git ${credentialArgs(file)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, }); try { for (const repo of REPOSITORIES) { const url = `${forge}/novox/${repo}.git`; const remote = git(["ls-remote", "--heads", url, "main"]); if (remote.status !== 0 || !remote.stdout.includes("refs/heads/main")) { const said = remote.stderr.trim().split("\n")[0] ?? ""; problems.push(`a run checks out ${repo}, and ${url} could not be read` + (said ? `: ${said}` : ": no main branch") + (credential ? "" : " — the lab holds no forge credential (its npm-package-registry binding and secret)")); continue; } if (!NPM_REPOSITORIES.includes(repo)) continue; const at = `${dir}/${repo}`; const clone = git(["clone", "--quiet", "--depth", "1", url, at]); if (clone.status !== 0) { problems.push(`${repo} could not be cloned from ${url}: ${clone.stderr.trim()}`); continue; } const plan = installPlan(at); for (const d of plan.unlocked) problems.push(`${repo}: ${d.slice(at.length) || "/"} has a package.json and no package-lock.json, so a run's npm ci cannot install it`); if (plan.install.length === 0 && plan.unlocked.length === 0) { problems.push(`${repo}: no package-lock.json at its root or one directory down, so a run installs nothing of it`); } } } finally { rmSync(keep, { recursive: true, force: true }); spawnSync("rm", ["-rf", dir]); } return problems; } /** One word from an env-file (`KEY=value` lines), or "" when the file or the word is absent. */ export function wordIn(file: string | undefined, word: string): string { if (!file) return ""; let text: string; try { text = readFileSync(file, "utf8"); } catch { return ""; } for (const line of text.split("\n")) { const at = line.indexOf("="); if (at > 0 && line.slice(0, at).trim() === word) return line.slice(at + 1).trim(); } return ""; } registerModuleTools("lab", (env) => getLabTools(env));