// A lab run: fresh checkouts of the named branches, side by side, then the lab's suite on the named // beds (novox/hq ADR 0172). // // **A run is a detached script with its own process group**, so it outlives the tool call that started // it and `stop` ends everything it started. It writes what it is doing to a status file beside its log, // and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers // for it, and says it was lost rather than pretending it is still going. import { spawn } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; /** * The repositories a lab run checks out, side by side, as the lab expects its siblings. mesh-tools-go is * one since genesis builds the Go toolchain at its base step, from the repository beside mesh-tools * (novox/hq issue 616): a bed that raises genesis serves it from its forge. */ export const REPOSITORIES = ["mesh-lab", "mesh-controller", "mesh-host", "mesh-catalog", "mesh-tools", "mesh-tools-go", "mesh-sdk"]; /** The checkouts whose JavaScript dependencies a run installs: the lab, the SDK and the base. */ export const NPM_REPOSITORIES = ["mesh-sdk", "mesh-tools", "mesh-lab"]; /** * Where a checkout's JavaScript dependencies are installed (novox/hq issue 624): its root when the root * holds a lock file, otherwise each directory one level down that holds one. `npm ci` installs only from a * lock file, and a repository that moved its package into a directory of its own (mesh-tools' node-tools) * has none at its root; the run asked for one there and failed before it built anything. A package.json * with no lock file beside it cannot be installed reproducibly, and is named rather than installed. */ export function installPlan(root: string): { install: string[]; unlocked: string[] } { const has = (dir: string, file: string) => existsSync(join(dir, file)); if (has(root, "package-lock.json")) return { install: [root], unlocked: [] }; const install: string[] = []; const unlocked: string[] = has(root, "package.json") ? [root] : []; for (const entry of readdirSync(root, { withFileTypes: true })) { if (!entry.isDirectory() || entry.name.startsWith(".") || entry.name === "node_modules") continue; const dir = join(root, entry.name); if (has(dir, "package-lock.json")) install.push(dir); else if (has(dir, "package.json")) unlocked.push(dir); } return { install: install.sort(), unlocked: unlocked.sort() }; } /** * installStep is the shell a run installs one checkout with: the same rule as installPlan, so what * lab_check says a run would install is what it installs. Building runs where the package says how. */ export function installStep(root: string, build: boolean): string { const q = shellQuote(root); const install = `(cd "$d" && npm ci --no-audit --no-fund --loglevel=error${build ? " && npm run build --if-present" : ""})`; return `if [ -f ${q}/package-lock.json ]; then d=${q}; ${install}; else [ ! -f ${q}/package.json ] || { echo "${root}: a package.json and no package-lock.json, so npm ci cannot install it (novox/hq issue 624)" >&2; exit 1; } found=0 for d in ${q}/*/; do d="\${d%/}"; [ "$(basename "$d")" = node_modules ] && continue if [ -f "$d/package-lock.json" ]; then found=1; ${install} elif [ -f "$d/package.json" ]; then echo "$d: a package.json and no package-lock.json, so npm ci cannot install it (novox/hq issue 624)" >&2; exit 1; fi done [ "$found" = 1 ] || { echo "${root}: no package-lock.json at its root or one directory down, so nothing of it can be installed (novox/hq issue 624)" >&2; exit 1; } fi`; } export interface RunStatus { id: string; state: "checking-out" | "building" | "running" | "passed" | "failed" | "stopped" | "lost"; started: string; ended?: string; tests: string[]; refs: Record; commits?: Record; exit?: number; pid?: number; } export function runDir(work: string, id: string): string { return join(work, id); } function statusPath(work: string, id: string): string { return join(runDir(work, id), "status.json"); } export function readStatus(work: string, id: string): RunStatus | undefined { try { const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus; // A run whose process is gone while its status still says it is going was lost — the runtime or // the machine restarted under it. Said, rather than left reading as running for ever. if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) { s.state = "lost"; } return s; } catch { return undefined; } } function alive(pid: number): boolean { try { process.kill(pid, 0); return true; } catch { return false; } } export function listRuns(work: string): RunStatus[] { if (!existsSync(work)) return []; return readdirSync(work) .filter((d) => d.startsWith("run-")) .map((id) => readStatus(work, id)) .filter((s): s is RunStatus => !!s) .sort((a, b) => b.started.localeCompare(a.started)); } /** The run still going, if any: one at a time, because two would contend for the same machine. */ export function running(work: string): RunStatus | undefined { return listRuns(work).find((s) => !["passed", "failed", "stopped", "lost"].includes(s.state)); } const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`; /** * The script one run executes. Every step writes its state first, so a run that dies says where. * * The environment is the one the lab's README describes for a run against sibling checkouts, pointed * at this run's own tree, so what is built and claimed is exactly what was checked out. */ export function script(work: string, id: string, forge: string, tests: string[], refs: Record): string { const dir = runDir(work, id); const setState = (state: string) => `node -e ${shellQuote( `const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=${JSON.stringify(state)};require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, )}`; const clones = REPOSITORIES.map((repo) => { const ref = refs[repo] ?? "main"; return [ `git clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, `echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`, ].join("\n"); }).join("\n"); const bin = join(dir, "bin"); return `set -euo pipefail cd ${shellQuote(dir)} ${setState("checking-out")} ${clones} node -e ${shellQuote( `const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`, )} ${setState("building")} # The @novox scope resolves from the mesh's own package registry on the forge, as the builder # resolves it; nothing else is asked of it. printf '%s\n' ${shellQuote(`@novox:registry=${forge}/api/packages/novox/npm/`)} > ${shellQuote(join(dir, ".npmrc"))} export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))} ${NPM_REPOSITORIES.map((repo) => installStep(join(dir, repo), repo !== "mesh-lab")).join("\n")} mkdir -p ${shellQuote(bin)} for p in postgres-provisioner objectstore-provisioner route-proxy; do (cd ${shellQuote(dir)}/mesh-controller && CGO_ENABLED=0 go build -o ${shellQuote(bin)}/$p ./examples/$p) done export MESH_LAB_HOST_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-host"))} export MESH_LAB_BUNDLE=${shellQuote(join(dir, "mesh-host", "examples", "foundation-first-node.lock"))} export MESH_LAB_MODULES=${shellQuote(join(dir, "mesh-controller", "examples", "modules"))} export MESH_LAB_BUILDER=${shellQuote(join(dir, "mesh-controller", "build", "mesh-builder"))} export MESH_LAB_BOOTSTRAP_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-bootstrap"))} export MESH_LAB_CATALOG=${shellQuote(join(dir, "mesh-catalog", "modules"))} # Where a bed that serves repositories from its own forge finds them: this run's checkouts, side by side. export MESH_LAB_REPOS=${shellQuote(dir)} export MESH_LAB_PROVISIONER=${shellQuote(join(bin, "postgres-provisioner"))} export MESH_LAB_OBJECTSTORE_PROVISIONER=${shellQuote(join(bin, "objectstore-provisioner"))} export MESH_LAB_ROUTE_PROXY=${shellQuote(join(bin, "route-proxy"))} ${setState("running")} cd ${shellQuote(join(dir, "mesh-lab"))} node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" ")} `; } /** start begins a run and returns at once with its status. */ export function start(work: string, forge: string, tests: string[], refs: Record): RunStatus { const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`; const dir = runDir(work, id); mkdirSync(dir, { recursive: true }); const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs }; writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2)); writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs), { mode: 0o700 }); // The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a // run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172). const wrapper = `bash ${shellQuote(join(dir, "run.sh"))} > ${shellQuote(join(dir, "run.log"))} 2>&1; code=$? node -e ${shellQuote( `const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, )} "$code" cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`; const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); child.unref(); status.pid = child.pid; writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2)); return status; } /** stop ends a run and everything it started, by its process group. */ export function stop(work: string, id: string): RunStatus | undefined { const s = readStatus(work, id); if (!s || !s.pid) return s; if (["passed", "failed", "stopped", "lost"].includes(s.state)) return s; s.state = "stopped"; writeFileSync(statusPath(work, id), JSON.stringify(s, null, 2)); try { process.kill(-s.pid, "SIGTERM"); } catch { // Already gone between the read and the kill. } return s; } /** tail is the last lines of a run's log. */ export function tail(work: string, id: string, lines: number): string { try { const all = readFileSync(join(runDir(work, id), "run.log"), "utf8").split("\n"); return all.slice(-Math.max(1, lines)).join("\n"); } catch { return ""; } }