lab: the lab as a module, running beds when the mesh asks
Five tools on the machine the lab runs on: check, run beds against branches on the forge, a run's status, its log, and stop. A run checks out every repository the lab builds, side by side, and runs the suite; one at a time, answered at once with an id (novox/hq ADR 0172).
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
// lab's tools — the lab, as the mesh asks for it (novox/hq ADR 0172). They run on the machine the
|
||||
// lab is assigned to, and only there: a bed raises virtual machines on that machine's virtualisation.
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { listRuns, readStatus, REPOSITORIES, running, start, stop, tail } from "./runs.js";
|
||||
|
||||
export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
|
||||
const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs";
|
||||
const forge = (env.MESH_LAB_FORGE ?? "").replace(/\/+$/, "");
|
||||
return [
|
||||
{
|
||||
name: "lab_check",
|
||||
description: "Whether this machine can run the lab's beds: the lab's own check, against the forge's main branch.",
|
||||
input: {},
|
||||
run: async () => {
|
||||
if (!forge) return { ok: false, output: "the lab's forge is not set: settings for lab, {\"forge\": \"<url>\"}" };
|
||||
const dir = `${work}/check`;
|
||||
spawnSync("rm", ["-rf", dir]);
|
||||
const clone = spawnSync("git", ["clone", "--quiet", "--depth", "1", `${forge}/novox/mesh-lab.git`, dir], { encoding: "utf8" });
|
||||
if (clone.status !== 0) return { ok: false, output: clone.stderr };
|
||||
spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" });
|
||||
const check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" });
|
||||
return { ok: check.status === 0, output: `${check.stdout}${check.stderr}`.trim() };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "lab_run",
|
||||
description:
|
||||
"Run the lab's beds against branches on the forge: fresh checkouts of every repository the lab builds, " +
|
||||
"side by side, then the suite on the named test files. Answers at once with the run's id; lab_status " +
|
||||
"and lab_log follow it. One run at a time.",
|
||||
input: {
|
||||
tests: { type: "string", description: "the bed test files, comma-separated, relative to mesh-lab (e.g. test/integration/mesh.test.ts)" },
|
||||
refs: {
|
||||
type: "string",
|
||||
description: `a JSON object of repository to branch, for any of ${REPOSITORIES.join(", ")}; the rest run main`,
|
||||
},
|
||||
},
|
||||
run: async (args) => {
|
||||
if (!forge) return { started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"<url>\"}" };
|
||||
const tests = String(args.tests ?? "").split(",").map((s) => s.trim()).filter(Boolean);
|
||||
if (tests.length === 0) return { started: false, reason: "name at least one bed test file" };
|
||||
let refs: Record<string, string> = {};
|
||||
if (args.refs) {
|
||||
try {
|
||||
refs = JSON.parse(String(args.refs)) as Record<string, string>;
|
||||
} catch {
|
||||
return { started: false, reason: "refs is not a JSON object of repository to branch" };
|
||||
}
|
||||
}
|
||||
const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r));
|
||||
if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` };
|
||||
const busy = running(work);
|
||||
if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy };
|
||||
return { started: true, run: start(work, forge, tests, refs) };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "lab_status",
|
||||
description: "A run's state, the commits it tested and how it ended — or every run, newest first, when no id is given.",
|
||||
input: { id: { type: "string", description: "the run's id (optional)" } },
|
||||
run: async (args) => {
|
||||
if (args.id) return readStatus(work, String(args.id)) ?? { found: false, id: String(args.id) };
|
||||
return { runs: listRuns(work).slice(0, 10) };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "lab_log",
|
||||
description: "The last lines of a run's log.",
|
||||
input: {
|
||||
id: { type: "string", description: "the run's id" },
|
||||
lines: { type: "number", description: "how many lines from the end (default 200)" },
|
||||
},
|
||||
run: async (args) => ({ id: String(args.id), log: tail(work, String(args.id), Number(args.lines ?? 200)) }),
|
||||
},
|
||||
{
|
||||
name: "lab_stop",
|
||||
description: "Stop a run and everything it started.",
|
||||
input: { id: { type: "string", description: "the run's id" } },
|
||||
run: async (args) => stop(work, String(args.id)) ?? { found: false, id: String(args.id) },
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
registerModuleTools("lab", (env) => getLabTools(env));
|
||||
@@ -0,0 +1,171 @@
|
||||
// A lab run: fresh checkouts of the named branches, side by side, then the lab's suite on the named
|
||||
// beds (novox/hq ADR 0172).
|
||||
//
|
||||
// **A run is a detached script with its own process group**, so it outlives the tool call that started
|
||||
// it and `stop` ends everything it started. It writes what it is doing to a status file beside its log,
|
||||
// and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers
|
||||
// for it, and says it was lost rather than pretending it is still going.
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
/** The repositories a lab run checks out, side by side, as the lab expects its siblings. */
|
||||
export const REPOSITORIES = ["mesh-lab", "mesh-controller", "mesh-host", "mesh-catalog", "mesh-tools", "mesh-sdk"];
|
||||
|
||||
export interface RunStatus {
|
||||
id: string;
|
||||
state: "checking-out" | "building" | "running" | "passed" | "failed" | "stopped" | "lost";
|
||||
started: string;
|
||||
ended?: string;
|
||||
tests: string[];
|
||||
refs: Record<string, string>;
|
||||
commits?: Record<string, string>;
|
||||
exit?: number;
|
||||
pid?: number;
|
||||
}
|
||||
|
||||
export function runDir(work: string, id: string): string {
|
||||
return join(work, id);
|
||||
}
|
||||
|
||||
function statusPath(work: string, id: string): string {
|
||||
return join(runDir(work, id), "status.json");
|
||||
}
|
||||
|
||||
export function readStatus(work: string, id: string): RunStatus | undefined {
|
||||
try {
|
||||
const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus;
|
||||
// A run whose process is gone while its status still says it is going was lost — the runtime or
|
||||
// the machine restarted under it. Said, rather than left reading as running for ever.
|
||||
if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) {
|
||||
s.state = "lost";
|
||||
}
|
||||
return s;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function alive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function listRuns(work: string): RunStatus[] {
|
||||
if (!existsSync(work)) return [];
|
||||
return readdirSync(work)
|
||||
.filter((d) => d.startsWith("run-"))
|
||||
.map((id) => readStatus(work, id))
|
||||
.filter((s): s is RunStatus => !!s)
|
||||
.sort((a, b) => b.started.localeCompare(a.started));
|
||||
}
|
||||
|
||||
/** The run still going, if any: one at a time, because two would contend for the same machine. */
|
||||
export function running(work: string): RunStatus | undefined {
|
||||
return listRuns(work).find((s) => !["passed", "failed", "stopped", "lost"].includes(s.state));
|
||||
}
|
||||
|
||||
const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`;
|
||||
|
||||
/**
|
||||
* The script one run executes. Every step writes its state first, so a run that dies says where.
|
||||
*
|
||||
* The environment is the one the lab's README describes for a run against sibling checkouts, pointed
|
||||
* at this run's own tree, so what is built and claimed is exactly what was checked out.
|
||||
*/
|
||||
export function script(work: string, id: string, forge: string, tests: string[], refs: Record<string, string>): string {
|
||||
const dir = runDir(work, id);
|
||||
const setState = (state: string) =>
|
||||
`node -e ${shellQuote(
|
||||
`const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=${JSON.stringify(state)};require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
|
||||
)}`;
|
||||
const clones = REPOSITORIES.map((repo) => {
|
||||
const ref = refs[repo] ?? "main";
|
||||
return [
|
||||
`git clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`,
|
||||
`echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`,
|
||||
].join("\n");
|
||||
}).join("\n");
|
||||
const bin = join(dir, "bin");
|
||||
return `set -euo pipefail
|
||||
cd ${shellQuote(dir)}
|
||||
${setState("checking-out")}
|
||||
${clones}
|
||||
node -e ${shellQuote(
|
||||
`const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`,
|
||||
)}
|
||||
${setState("building")}
|
||||
for repo in mesh-sdk mesh-tools mesh-lab; do (cd ${shellQuote(dir)}/$repo && npm ci --no-audit --no-fund --loglevel=error); done
|
||||
(cd ${shellQuote(dir)}/mesh-sdk && npm run build --if-present)
|
||||
(cd ${shellQuote(dir)}/mesh-tools && npm run build --if-present)
|
||||
mkdir -p ${shellQuote(bin)}
|
||||
for p in postgres-provisioner objectstore-provisioner route-proxy; do
|
||||
(cd ${shellQuote(dir)}/mesh-controller && CGO_ENABLED=0 go build -o ${shellQuote(bin)}/$p ./examples/$p)
|
||||
done
|
||||
export MESH_LAB_HOST_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-host"))}
|
||||
export MESH_LAB_BUNDLE=${shellQuote(join(dir, "mesh-host", "examples", "foundation-first-node-nats.lock"))}
|
||||
export MESH_LAB_MODULES=${shellQuote(join(dir, "mesh-controller", "examples", "modules"))}
|
||||
export MESH_LAB_BUILDER=${shellQuote(join(dir, "mesh-controller", "build", "mesh-builder"))}
|
||||
export MESH_LAB_BOOTSTRAP_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-bootstrap"))}
|
||||
export MESH_LAB_CATALOG=${shellQuote(join(dir, "mesh-catalog", "modules"))}
|
||||
export MESH_LAB_PROVISIONER=${shellQuote(join(bin, "postgres-provisioner"))}
|
||||
export MESH_LAB_OBJECTSTORE_PROVISIONER=${shellQuote(join(bin, "objectstore-provisioner"))}
|
||||
export MESH_LAB_ROUTE_PROXY=${shellQuote(join(bin, "route-proxy"))}
|
||||
${setState("running")}
|
||||
cd ${shellQuote(join(dir, "mesh-lab"))}
|
||||
node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" ")}
|
||||
`;
|
||||
}
|
||||
|
||||
/** start begins a run and returns at once with its status. */
|
||||
export function start(work: string, forge: string, tests: string[], refs: Record<string, string>): RunStatus {
|
||||
const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
||||
const dir = runDir(work, id);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs };
|
||||
writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
|
||||
writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs), { mode: 0o700 });
|
||||
|
||||
// The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a
|
||||
// run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172).
|
||||
const wrapper = `bash ${shellQuote(join(dir, "run.sh"))} > ${shellQuote(join(dir, "run.log"))} 2>&1; code=$?
|
||||
node -e ${shellQuote(
|
||||
`const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
|
||||
)} "$code"
|
||||
cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`;
|
||||
const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
|
||||
child.unref();
|
||||
status.pid = child.pid;
|
||||
writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
|
||||
return status;
|
||||
}
|
||||
|
||||
/** stop ends a run and everything it started, by its process group. */
|
||||
export function stop(work: string, id: string): RunStatus | undefined {
|
||||
const s = readStatus(work, id);
|
||||
if (!s || !s.pid) return s;
|
||||
if (["passed", "failed", "stopped", "lost"].includes(s.state)) return s;
|
||||
s.state = "stopped";
|
||||
writeFileSync(statusPath(work, id), JSON.stringify(s, null, 2));
|
||||
try {
|
||||
process.kill(-s.pid, "SIGTERM");
|
||||
} catch {
|
||||
// Already gone between the read and the kill.
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/** tail is the last lines of a run's log. */
|
||||
export function tail(work: string, id: string, lines: number): string {
|
||||
try {
|
||||
const all = readFileSync(join(runDir(work, id), "run.log"), "utf8").split("\n");
|
||||
return all.slice(-Math.max(1, lines)).join("\n");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user