// lab's tools — the lab, as the mesh asks for it (novox/hq ADR 0172). They run on the machine the // lab is assigned to, and only there: a bed raises virtual machines on that machine's virtualisation. import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start, stop, tail, writeCredential, } from "./runs.js"; export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs"; // The forge is an operator's setting, which reaches a file and never a bundle's words (novox/hq // ADR 0192): read from the env-file the mesh fills, at each call, so a changed setting is used // without restarting the runtime. MESH_LAB_FORGE itself still wins, for a hand-run instance. const forgeOf = (): string => (env.MESH_LAB_FORGE ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE")).replace(/\/+$/, ""); // The forge credential the mesh binds and seals to the lab (novox/hq issue 624), read at each call. const credentialOf = (forge: string): string | undefined => forgeCredential(forge, env.MESH_LAB_FORGE_BINDING ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING"), env.MESH_LAB_FORGE_SECRET ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_SECRET")); return [ { name: "lab_check", description: "Whether this machine can run the lab's beds: the lab's own check, against the forge's main branch.", input: {}, run: async () => { const forge = forgeOf(); if (!forge) return { ok: false, output: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; // What earlier checks left is cleared, and what could not be is said; this check works in a fresh // directory of its own either way, and removes it when it is done. const leftovers = clearChecks(work); const dir = checkDir(work); let check: ReturnType; let problems: string[]; try { const clone = spawnSync("git", ["clone", "--quiet", "--depth", "1", `${forge}/novox/mesh-lab.git`, dir], { encoding: "utf8" }); if (clone.status !== 0) return { ok: false, output: [String(clone.stderr).trim(), ...leftovers].join("\n") }; spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" }); check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" }); problems = setupProblems(forge, `${dir}/setup`, credentialOf(forge)); // And the address a run names the package registry by (npmRegistry): said when the registry does not say it. const registry = await npmRegistry(forge); if (registry.problem) problems.push(registry.problem); } finally { rmSync(dir, { recursive: true, force: true }); } // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every // repository it clones answers on the forge, and every one it installs has a lock file where the run // installs from. // A credential a stopped, crashed or rebooted run left behind is removed, and said; one still in use // by a running run is not. const busy = running(work); const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined); const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : ""; const output = [`${check.stdout}${check.stderr}`.trim(), removed, ...leftovers, ...problems].filter(Boolean).join("\n"); return { ok: check.status === 0 && problems.length === 0, output }; }, }, { name: "lab_run", description: "Run the lab's beds against branches on the forge: fresh checkouts of every repository the lab builds, " + "side by side, then the suite on the named test files. Answers at once with the run's id; lab_status " + "and lab_log follow it. One run at a time.", input: { tests: { type: "string", description: "the bed test files, comma-separated, relative to mesh-lab (e.g. test/integration/mesh.test.ts)" }, refs: { type: "string", description: `a JSON object of repository to branch, for any of ${REPOSITORIES.join(", ")}; the rest run main`, }, }, run: async (args) => { const forge = forgeOf(); if (!forge) return { started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; const tests = String(args.tests ?? "").split(",").map((s) => s.trim()).filter(Boolean); if (tests.length === 0) return { started: false, reason: "name at least one bed test file" }; let refs: Record = {}; if (args.refs) { try { refs = JSON.parse(String(args.refs)) as Record; } catch { return { started: false, reason: "refs is not a JSON object of repository to branch" }; } } const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r)); if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` }; const busy = running(work); if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy }; const registry = await npmRegistry(forge); return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry), registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) }; }, }, { name: "lab_status", description: "A run's state, the commits it tested and how it ended — or every run, newest first, when no id is given.", input: { id: { type: "string", description: "the run's id (optional)" } }, run: async (args) => { if (args.id) return readStatus(work, String(args.id)) ?? { found: false, id: String(args.id) }; return { runs: listRuns(work).slice(0, 10) }; }, }, { name: "lab_log", description: "The last lines of a run's log.", input: { id: { type: "string", description: "the run's id" }, lines: { type: "number", description: "how many lines from the end (default 200)" }, }, run: async (args) => ({ id: String(args.id), log: tail(work, String(args.id), Number(args.lines ?? 200)) }), }, { name: "lab_stop", description: "Stop a run and everything it started.", input: { id: { type: "string", description: "the run's id" } }, run: async (args) => stop(work, String(args.id)) ?? { found: false, id: String(args.id) }, }, ]; } /** * What would stop a run at its setup, in sentences: a repository the run clones that the forge does not * answer for, and a checkout it installs with nothing installable (installPlan). Empty when a run's setup * would go through. */ export function setupProblems(forge: string, dir: string, credential?: string): string[] { const problems: string[] = []; spawnSync("rm", ["-rf", dir]); // The credential a run would offer, in a file of its own outside the check's clones, removed after. const keep = mkdtempSync(join(tmpdir(), "lab-check-")); const file = writeCredential(join(keep, "git-credentials"), credential); const git = (args: string[]) => spawnSync("sh", ["-c", `git ${credentialArgs(file)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, }); try { for (const repo of REPOSITORIES) { const url = `${forge}/novox/${repo}.git`; const remote = git(["ls-remote", "--heads", url, "main"]); if (remote.status !== 0 || !remote.stdout.includes("refs/heads/main")) { const said = remote.stderr.trim().split("\n")[0] ?? ""; problems.push(`a run checks out ${repo}, and ${url} could not be read` + (said ? `: ${said}` : ": no main branch") + (credential ? "" : " — the lab holds no forge credential (its npm-package-registry binding and secret)")); continue; } if (!NPM_REPOSITORIES.includes(repo)) continue; const at = `${dir}/${repo}`; const clone = git(["clone", "--quiet", "--depth", "1", url, at]); if (clone.status !== 0) { problems.push(`${repo} could not be cloned from ${url}: ${clone.stderr.trim()}`); continue; } const plan = installPlan(at); for (const d of plan.unlocked) problems.push(`${repo}: ${d.slice(at.length) || "/"} has a package.json and no package-lock.json, so a run's npm ci cannot install it`); if (plan.install.length === 0 && plan.unlocked.length === 0) { problems.push(`${repo}: no package-lock.json at its root or one directory down, so a run installs nothing of it`); } } } finally { rmSync(keep, { recursive: true, force: true }); spawnSync("rm", ["-rf", dir]); } return problems; } /** One word from an env-file (`KEY=value` lines), or "" when the file or the word is absent. */ export function wordIn(file: string | undefined, word: string): string { if (!file) return ""; let text: string; try { text = readFileSync(file, "utf8"); } catch { return ""; } for (const line of text.split("\n")) { const at = line.indexOf("="); if (at > 0 && line.slice(0, at).trim() === word) return line.slice(at + 1).trim(); } return ""; } registerModuleTools("lab", (env) => getLabTools(env));