From 67d1a400e879f366cb064b1488e5a4aea7a3314b Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 2 Oct 2026 14:15:42 +0200 Subject: [PATCH] lab: the lab as a module, running beds when the mesh asks Five tools on the machine the lab runs on: check, run beds against branches on the forge, a run's status, its log, and stop. A run checks out every repository the lab builds, side by side, and runs the suite; one at a time, answered at once with an id (novox/hq ADR 0172). --- modules/lab/Dockerfile | 31 +++++++ modules/lab/module.json | 81 ++++++++++++++++++ modules/lab/package.json | 9 ++ modules/lab/tools/index.ts | 86 +++++++++++++++++++ modules/lab/tools/runs.ts | 171 +++++++++++++++++++++++++++++++++++++ modules/lab/tsconfig.json | 12 +++ 6 files changed, 390 insertions(+) create mode 100644 modules/lab/Dockerfile create mode 100644 modules/lab/module.json create mode 100644 modules/lab/package.json create mode 100644 modules/lab/tools/index.ts create mode 100644 modules/lab/tools/runs.ts create mode 100644 modules/lab/tsconfig.json diff --git a/modules/lab/Dockerfile b/modules/lab/Dockerfile new file mode 100644 index 0000000..beb7c8f --- /dev/null +++ b/modules/lab/Dockerfile @@ -0,0 +1,31 @@ +# lab's runtime: the tool runtime, carrying this module's code, and the toolchain the lab's suite +# builds the mesh with (novox/hq ADR 0172). It reaches the machine's virtualisation and container +# runtime through their sockets, so what it raises is what a hand run on this machine raises. +# +# Every download is pinned by its checksum: an image that builds the mesh is the last place to take +# whatever an upstream serves today. +ARG BUILD_BASE +ARG RUNTIME_BASE + +FROM ${BUILD_BASE} AS build +WORKDIR /app/modules/lab +COPY . . +RUN node /app/node_modules/typescript/bin/tsc tools/index.ts tools/runs.ts \ + --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist + +FROM ${RUNTIME_BASE} +RUN apt-get update \ + && apt-get install -y --no-install-recommends git make ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +RUN curl -fsSL -o /tmp/go.tgz https://go.dev/dl/go1.26.8.linux-amd64.tar.gz \ + && echo "d0f743b33e8d8945e6b1f432edd15785c70507121d6e2a723b21285eddf8b57b /tmp/go.tgz" | sha256sum -c - \ + && tar -C /usr/local -xzf /tmp/go.tgz && rm /tmp/go.tgz +RUN curl -fsSL -o /usr/local/bin/incus https://github.com/lxc/incus/releases/download/v7.5.1/bin.linux.incus.x86_64 \ + && echo "7bd6223b369f4d693fcde695bd8549a73b5b3d403735329212483702aa22c179 /usr/local/bin/incus" | sha256sum -c - \ + && chmod 0755 /usr/local/bin/incus +RUN curl -fsSL -o /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-28.5.2.tgz \ + && echo "ea90cfd12e1eeb12aa1c971741adb8bd4ed88e2a574eaac13f5029a1dbc6300d /tmp/docker.tgz" | sha256sum -c - \ + && tar -C /tmp -xzf /tmp/docker.tgz docker/docker && mv /tmp/docker/docker /usr/local/bin/docker && rm -rf /tmp/docker /tmp/docker.tgz +ENV PATH=/usr/local/go/bin:$PATH +COPY --from=build /app/modules/lab/dist /app/modules/lab/dist +ENV MESH_TOOL_MODULES=/app/modules/lab/dist/tools/index.js diff --git a/modules/lab/module.json b/modules/lab/module.json new file mode 100644 index 0000000..16fdac7 --- /dev/null +++ b/modules/lab/module.json @@ -0,0 +1,81 @@ +{ + "module": "lab", + "version": "1", + "capabilities": [ + "container-runtime" + ], + "own-secrets": { + "broker": "${dir:mesh-state}/broker" + }, + "resources": [ + { + "id": "mesh-state", + "type": "directory", + "mode": "0700", + "place": "mesh" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": "." + }, + { + "id": "work", + "type": "directory", + "path": "/var/lib/mesh-lab-runs", + "mode": "0700" + }, + { + "id": "runtime-env", + "type": "file", + "path": "${dir:state}/lab.env", + "mode": "0600", + "content": "MESH_LAB_FORGE=${setting:forge}\n" + }, + { + "id": "runtime", + "type": "container", + "name": "mesh-lab", + "network": "host", + "env-file": [ + "${dir:state}/lab.env" + ], + "volumes": [ + "${dir:mesh-state}/broker:/run/secrets/broker:ro", + "${dir:work}:${dir:work}", + "/var/run/docker.sock:/var/run/docker.sock", + "/var/lib/incus/unix.socket:/var/lib/incus/unix.socket" + ], + "env": { + "MESH_BROKER_FILE": "/run/secrets/broker", + "MESH_LAB_WORK": "${dir:work}" + }, + "restart-on": [ + "runtime-env" + ], + "artifact": "runtime" + } + ], + "build": { + "on": [ + { + "arg": "BUILD_BASE", + "module": "mesh-tools", + "artifact": "build" + }, + { + "arg": "RUNTIME_BASE", + "module": "mesh-tools", + "artifact": "runtime" + } + ], + "artifacts": [ + { + "name": "runtime", + "kind": "image", + "from": "Dockerfile" + } + ] + } +} diff --git a/modules/lab/package.json b/modules/lab/package.json new file mode 100644 index 0000000..a313822 --- /dev/null +++ b/modules/lab/package.json @@ -0,0 +1,9 @@ +{ + "name": "@novox/module-lab", + "version": "0.1.0", + "description": "lab — the lab, as a module: runs beds against the forge's branches when the mesh asks (novox/hq ADR 0172).", + "type": "module", + "private": true, + "dependencies": { "@novox/mesh-sdk": "^0.1.0" }, + "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" } +} diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts new file mode 100644 index 0000000..097b022 --- /dev/null +++ b/modules/lab/tools/index.ts @@ -0,0 +1,86 @@ +// lab's tools — the lab, as the mesh asks for it (novox/hq ADR 0172). They run on the machine the +// lab is assigned to, and only there: a bed raises virtual machines on that machine's virtualisation. + +import { spawnSync } from "node:child_process"; +import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; +import { listRuns, readStatus, REPOSITORIES, running, start, stop, tail } from "./runs.js"; + +export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { + const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs"; + const forge = (env.MESH_LAB_FORGE ?? "").replace(/\/+$/, ""); + return [ + { + name: "lab_check", + description: "Whether this machine can run the lab's beds: the lab's own check, against the forge's main branch.", + input: {}, + run: async () => { + if (!forge) return { ok: false, output: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; + const dir = `${work}/check`; + spawnSync("rm", ["-rf", dir]); + const clone = spawnSync("git", ["clone", "--quiet", "--depth", "1", `${forge}/novox/mesh-lab.git`, dir], { encoding: "utf8" }); + if (clone.status !== 0) return { ok: false, output: clone.stderr }; + spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" }); + const check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" }); + return { ok: check.status === 0, output: `${check.stdout}${check.stderr}`.trim() }; + }, + }, + { + name: "lab_run", + description: + "Run the lab's beds against branches on the forge: fresh checkouts of every repository the lab builds, " + + "side by side, then the suite on the named test files. Answers at once with the run's id; lab_status " + + "and lab_log follow it. One run at a time.", + input: { + tests: { type: "string", description: "the bed test files, comma-separated, relative to mesh-lab (e.g. test/integration/mesh.test.ts)" }, + refs: { + type: "string", + description: `a JSON object of repository to branch, for any of ${REPOSITORIES.join(", ")}; the rest run main`, + }, + }, + run: async (args) => { + if (!forge) return { started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"\"}" }; + const tests = String(args.tests ?? "").split(",").map((s) => s.trim()).filter(Boolean); + if (tests.length === 0) return { started: false, reason: "name at least one bed test file" }; + let refs: Record = {}; + if (args.refs) { + try { + refs = JSON.parse(String(args.refs)) as Record; + } catch { + return { started: false, reason: "refs is not a JSON object of repository to branch" }; + } + } + const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r)); + if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` }; + const busy = running(work); + if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy }; + return { started: true, run: start(work, forge, tests, refs) }; + }, + }, + { + name: "lab_status", + description: "A run's state, the commits it tested and how it ended — or every run, newest first, when no id is given.", + input: { id: { type: "string", description: "the run's id (optional)" } }, + run: async (args) => { + if (args.id) return readStatus(work, String(args.id)) ?? { found: false, id: String(args.id) }; + return { runs: listRuns(work).slice(0, 10) }; + }, + }, + { + name: "lab_log", + description: "The last lines of a run's log.", + input: { + id: { type: "string", description: "the run's id" }, + lines: { type: "number", description: "how many lines from the end (default 200)" }, + }, + run: async (args) => ({ id: String(args.id), log: tail(work, String(args.id), Number(args.lines ?? 200)) }), + }, + { + name: "lab_stop", + description: "Stop a run and everything it started.", + input: { id: { type: "string", description: "the run's id" } }, + run: async (args) => stop(work, String(args.id)) ?? { found: false, id: String(args.id) }, + }, + ]; +} + +registerModuleTools("lab", (env) => getLabTools(env)); diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts new file mode 100644 index 0000000..b2404c8 --- /dev/null +++ b/modules/lab/tools/runs.ts @@ -0,0 +1,171 @@ +// A lab run: fresh checkouts of the named branches, side by side, then the lab's suite on the named +// beds (novox/hq ADR 0172). +// +// **A run is a detached script with its own process group**, so it outlives the tool call that started +// it and `stop` ends everything it started. It writes what it is doing to a status file beside its log, +// and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers +// for it, and says it was lost rather than pretending it is still going. + +import { spawn } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +/** The repositories a lab run checks out, side by side, as the lab expects its siblings. */ +export const REPOSITORIES = ["mesh-lab", "mesh-controller", "mesh-host", "mesh-catalog", "mesh-tools", "mesh-sdk"]; + +export interface RunStatus { + id: string; + state: "checking-out" | "building" | "running" | "passed" | "failed" | "stopped" | "lost"; + started: string; + ended?: string; + tests: string[]; + refs: Record; + commits?: Record; + exit?: number; + pid?: number; +} + +export function runDir(work: string, id: string): string { + return join(work, id); +} + +function statusPath(work: string, id: string): string { + return join(runDir(work, id), "status.json"); +} + +export function readStatus(work: string, id: string): RunStatus | undefined { + try { + const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus; + // A run whose process is gone while its status still says it is going was lost — the runtime or + // the machine restarted under it. Said, rather than left reading as running for ever. + if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) { + s.state = "lost"; + } + return s; + } catch { + return undefined; + } +} + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +export function listRuns(work: string): RunStatus[] { + if (!existsSync(work)) return []; + return readdirSync(work) + .filter((d) => d.startsWith("run-")) + .map((id) => readStatus(work, id)) + .filter((s): s is RunStatus => !!s) + .sort((a, b) => b.started.localeCompare(a.started)); +} + +/** The run still going, if any: one at a time, because two would contend for the same machine. */ +export function running(work: string): RunStatus | undefined { + return listRuns(work).find((s) => !["passed", "failed", "stopped", "lost"].includes(s.state)); +} + +const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`; + +/** + * The script one run executes. Every step writes its state first, so a run that dies says where. + * + * The environment is the one the lab's README describes for a run against sibling checkouts, pointed + * at this run's own tree, so what is built and claimed is exactly what was checked out. + */ +export function script(work: string, id: string, forge: string, tests: string[], refs: Record): string { + const dir = runDir(work, id); + const setState = (state: string) => + `node -e ${shellQuote( + `const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=${JSON.stringify(state)};require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, + )}`; + const clones = REPOSITORIES.map((repo) => { + const ref = refs[repo] ?? "main"; + return [ + `git clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, + `echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`, + ].join("\n"); + }).join("\n"); + const bin = join(dir, "bin"); + return `set -euo pipefail +cd ${shellQuote(dir)} +${setState("checking-out")} +${clones} +node -e ${shellQuote( + `const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`, + )} +${setState("building")} +for repo in mesh-sdk mesh-tools mesh-lab; do (cd ${shellQuote(dir)}/$repo && npm ci --no-audit --no-fund --loglevel=error); done +(cd ${shellQuote(dir)}/mesh-sdk && npm run build --if-present) +(cd ${shellQuote(dir)}/mesh-tools && npm run build --if-present) +mkdir -p ${shellQuote(bin)} +for p in postgres-provisioner objectstore-provisioner route-proxy; do + (cd ${shellQuote(dir)}/mesh-controller && CGO_ENABLED=0 go build -o ${shellQuote(bin)}/$p ./examples/$p) +done +export MESH_LAB_HOST_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-host"))} +export MESH_LAB_BUNDLE=${shellQuote(join(dir, "mesh-host", "examples", "foundation-first-node-nats.lock"))} +export MESH_LAB_MODULES=${shellQuote(join(dir, "mesh-controller", "examples", "modules"))} +export MESH_LAB_BUILDER=${shellQuote(join(dir, "mesh-controller", "build", "mesh-builder"))} +export MESH_LAB_BOOTSTRAP_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-bootstrap"))} +export MESH_LAB_CATALOG=${shellQuote(join(dir, "mesh-catalog", "modules"))} +export MESH_LAB_PROVISIONER=${shellQuote(join(bin, "postgres-provisioner"))} +export MESH_LAB_OBJECTSTORE_PROVISIONER=${shellQuote(join(bin, "objectstore-provisioner"))} +export MESH_LAB_ROUTE_PROXY=${shellQuote(join(bin, "route-proxy"))} +${setState("running")} +cd ${shellQuote(join(dir, "mesh-lab"))} +node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" ")} +`; +} + +/** start begins a run and returns at once with its status. */ +export function start(work: string, forge: string, tests: string[], refs: Record): RunStatus { + const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`; + const dir = runDir(work, id); + mkdirSync(dir, { recursive: true }); + const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs }; + writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2)); + writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs), { mode: 0o700 }); + + // The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a + // run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172). + const wrapper = `bash ${shellQuote(join(dir, "run.sh"))} > ${shellQuote(join(dir, "run.log"))} 2>&1; code=$? +node -e ${shellQuote( + `const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, + )} "$code" +cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`; + const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); + child.unref(); + status.pid = child.pid; + writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2)); + return status; +} + +/** stop ends a run and everything it started, by its process group. */ +export function stop(work: string, id: string): RunStatus | undefined { + const s = readStatus(work, id); + if (!s || !s.pid) return s; + if (["passed", "failed", "stopped", "lost"].includes(s.state)) return s; + s.state = "stopped"; + writeFileSync(statusPath(work, id), JSON.stringify(s, null, 2)); + try { + process.kill(-s.pid, "SIGTERM"); + } catch { + // Already gone between the read and the kill. + } + return s; +} + +/** tail is the last lines of a run's log. */ +export function tail(work: string, id: string, lines: number): string { + try { + const all = readFileSync(join(runDir(work, id), "run.log"), "utf8").split("\n"); + return all.slice(-Math.max(1, lines)).join("\n"); + } catch { + return ""; + } +} diff --git a/modules/lab/tsconfig.json b/modules/lab/tsconfig.json new file mode 100644 index 0000000..f4f306f --- /dev/null +++ b/modules/lab/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["tools/index.ts", "tools/runs.ts"] +}