Files
mesh-catalog/modules/lab/tools/runs.ts
T
jochen 9e2c1531e4
mesh/merge-gate out of date: judged against trunk 4d15a155e634, the trunk is now e209d46faf3d; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk 4d15a155e634, the trunk is now e209d46faf3d; judged again on a recheck or when it is merged
mesh/delivery superseded: a newer head of the same pull request
gitea serves the registry's public root, and the lab trusts a tarball only at it or the forge's own origin (hq ADR 0346, issue 624)
The forge writes every tarball at its public name and the lab reaches it at
a private address, so npm 12 refused every locked @novox package. The public
name now comes from the mesh's own composition of the forge's route, through
the lab's binding, and the lab compares scheme, host and port.
2026-10-13 16:53:26 +02:00

447 lines
22 KiB
TypeScript

// A lab run: fresh checkouts of the named branches, side by side, then the lab's suite on the named
// beds (novox/hq ADR 0172).
//
// **A run is a detached script with its own process group**, so it outlives the tool call that started
// it and `stop` ends everything it started. It writes what it is doing to a status file beside its log,
// and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers
// for it, and says it was lost rather than pretending it is still going.
import { spawn } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
/**
* The repositories a lab run checks out, side by side, as the lab expects its siblings. mesh-tools-go is
* one since genesis builds the Go toolchain at its base step, from the repository beside mesh-tools
* (novox/hq issue 616): a bed that raises genesis serves it from its forge.
*/
export const REPOSITORIES = ["mesh-lab", "mesh-controller", "mesh-host", "mesh-catalog", "mesh-tools", "mesh-tools-go", "mesh-sdk"];
/** The checkouts whose JavaScript dependencies a run installs: the lab, the SDK and the base. */
export const NPM_REPOSITORIES = ["mesh-sdk", "mesh-tools", "mesh-lab"];
/**
* Where a checkout's JavaScript dependencies are installed (novox/hq issue 624): its root when the root
* holds a lock file, otherwise each directory one level down that holds one. `npm ci` installs only from a
* lock file, and a repository that moved its package into a directory of its own (mesh-tools' node-tools)
* has none at its root; the run asked for one there and failed before it built anything. A package.json
* with no lock file beside it cannot be installed reproducibly, and is named rather than installed.
*/
export function installPlan(root: string): { install: string[]; unlocked: string[] } {
const has = (dir: string, file: string) => existsSync(join(dir, file));
if (has(root, "package-lock.json")) return { install: [root], unlocked: [] };
const install: string[] = [];
const unlocked: string[] = has(root, "package.json") ? [root] : [];
for (const entry of readdirSync(root, { withFileTypes: true })) {
if (!entry.isDirectory() || entry.name.startsWith(".") || entry.name === "node_modules") continue;
const dir = join(root, entry.name);
if (has(dir, "package-lock.json")) install.push(dir);
else if (has(dir, "package.json")) unlocked.push(dir);
}
return { install: install.sort(), unlocked: unlocked.sort() };
}
/**
* installStep is the shell a run installs one checkout with: the same rule as installPlan, so what
* lab_check says a run would install is what it installs. Building runs where the package says how.
*/
export function installStep(root: string, build: boolean): string {
const q = shellQuote(root);
const install = `(cd "$d" && npm ci --no-audit --no-fund --loglevel=error${build ? " && npm run build --if-present" : ""})`;
return `if [ -f ${q}/package-lock.json ]; then d=${q}; ${install}; else
[ ! -f ${q}/package.json ] || { echo "${root}: a package.json and no package-lock.json, so npm ci cannot install it (novox/hq issue 624)" >&2; exit 1; }
found=0
for d in ${q}/*/; do
d="\${d%/}"; [ "$(basename "$d")" = node_modules ] && continue
if [ -f "$d/package-lock.json" ]; then found=1; ${install}
elif [ -f "$d/package.json" ]; then echo "$d: a package.json and no package-lock.json, so npm ci cannot install it (novox/hq issue 624)" >&2; exit 1; fi
done
[ "$found" = 1 ] || { echo "${root}: no package-lock.json at its root or one directory down, so nothing of it can be installed (novox/hq issue 624)" >&2; exit 1; }
fi`;
}
/** Where the forge serves the mesh's npm packages, under its own address (gitea's npm registry for the novox org). */
export const NPM_PATH = "/api/packages/novox/npm/";
/**
* The registry's public root, as the forge serves it in the lab's npm-package-registry binding
* (`serves.public-root`, novox/hq ADR 0346): the forge's route's composed public name under the npm path.
* The mesh writes the binding, and it is checked all the same: an https URL whose path is the npm path and
* nothing else. Undefined, with why, when the binding has none or it is not that shape — a forge on a
* machine with no public domain serves none (ADR 0346 §3).
*/
export function publicRootOf(bindingFile: string | undefined): { root?: string; problem?: string } {
if (!bindingFile) return { problem: "the lab has no npm-package-registry binding" };
let said: unknown;
try {
said = (JSON.parse(readFileSync(bindingFile, "utf8")) as { serves?: Record<string, unknown> }).serves?.["public-root"];
} catch (e) {
return { problem: `its npm-package-registry binding could not be read: ${e instanceof Error ? e.message : String(e)}` };
}
if (typeof said !== "string" || !said) return { problem: "its npm-package-registry binding serves no public-root" };
if (/[\s\u0000-\u001f\u007f]/.test(said)) return { problem: "the binding's public-root holds whitespace or a control character" };
let u: URL;
try {
u = new URL(said);
} catch {
return { problem: `the binding's public-root ${JSON.stringify(said)} is not a URL` };
}
if (u.protocol !== "https:" || u.username || u.password || u.search || u.hash || u.pathname !== NPM_PATH) {
return { problem: `the binding's public-root ${JSON.stringify(said)} is not https://<host>${NPM_PATH}` };
}
return { root: `${u.origin}${NPM_PATH}` };
}
/**
* The address a run names the @novox registry by: the one the registry names itself by, when that is the
* forge's public root or the forge's own address (novox/hq issue 624, ADR 0346).
*
* **npm 12 installs a locked tarball only from the registry it is configured with.** Its `allow-remote`
* defaults to `none`, and a lock file's `resolved` URL counts as the registry's only when its origin and
* path are the configured scope registry's; any other is a "remote" tarball and refused (EALLOWREMOTE).
* The forge writes every tarball URL — in what it answers, and so in every lock file made against it — at
* its own public address, whichever address it was asked at. So the lab asks the registry, at the forge,
* for the SDK's newest tarball and names the registry by the origin of that URL.
*
* **What the registry answers is not trusted for where packages come from** (review of mesh-catalog #275):
* it may be fetched over plain HTTP. The tarball URL is parsed, and taken only when it carries no
* whitespace, control character, user, password, query or fragment, its path is under the npm path, and
* its origin — scheme, host and port — is one the lab already trusts: the public root the forge serves in
* the lab's binding (`publicRoot`, ADR 0346), or the forge setting's own. The registry is then built from
* the parsed origin and the npm path alone, never from the string answered, so nothing the registry says
* reaches the run's npm configuration but a scheme, a host and a port. Anything else falls back to the
* forge's own address and says why, so a run still starts and lab_check names it.
*/
export async function npmRegistry(forge: string, get: typeof fetch = fetch, publicRoot?: string):
Promise<{ registry: string; problem?: string }> {
const asked = `${forge.replace(/\/+$/, "")}${NPM_PATH}`;
const url = `${asked}@novox%2fmesh-sdk`;
const fallback = (why: string) => ({
registry: asked,
problem: `the package registry at ${url} did not give an address the lab takes (${why}), so a run names it ${asked}, and npm 12 refuses a locked @novox tarball at any other`,
});
let own: URL;
try {
own = new URL(forge);
} catch {
return fallback(`the forge ${JSON.stringify(forge)} is not a URL`);
}
const trusted = [own.origin];
if (publicRoot) {
try {
trusted.unshift(new URL(publicRoot).origin);
} catch {
// publicRootOf hands only a parsed root; a caller's other string is not trusted.
}
}
try {
const res = await get(url, { signal: AbortSignal.timeout(15_000) });
if (!res.ok) return fallback(`it answered ${res.status}`);
const doc = await res.json() as { "dist-tags"?: { latest?: string }; versions?: Record<string, { dist?: { tarball?: unknown } }> };
const latest = doc["dist-tags"]?.latest ?? "";
const tarball = doc.versions?.[latest]?.dist?.tarball;
if (typeof tarball !== "string" || !tarball) return fallback("no tarball for its latest version");
if (/[\s\u0000-\u001f\u007f]/.test(tarball)) return fallback("its tarball URL holds whitespace or a control character");
let t: URL;
try {
t = new URL(tarball);
} catch {
return fallback("its tarball is not a URL");
}
if (t.username || t.password) return fallback("its tarball URL carries a user or password");
if (t.search || t.hash) return fallback("its tarball URL carries a query or a fragment");
if (!t.pathname.startsWith(NPM_PATH)) return fallback(`its tarball is not under ${NPM_PATH}`);
if (!trusted.includes(t.origin)) {
return fallback(`its tarball is at ${t.origin}, which is neither the forge's public root nor its own address (${trusted.join(", ")})`);
}
return { registry: `${t.origin}${NPM_PATH}` };
} catch (e) {
return fallback(e instanceof Error ? e.message : String(e));
}
}
/**
* The forge credential a run's clones may present, as a git credential-store line —
* scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none.
*
* **The mesh's credential for the forge, the way the build agent holds it** (novox/hq issue 624): the lab
* requires the package registry, the forge's provisioner binds it an account (`as`) and seals that
* account's password, and the forge that answers npm is the forge that hosts the repositories. Private
* repositories (mesh-tools-go) are cloned with it; public ones clone as before. Written to a file git reads
* through its credential store, never into a URL, an argument, the run's script or its log, and offered
* only to the forge's own scheme, host and port.
*/
export function forgeCredential(forge: string, bindingFile: string, secretFile: string): string | undefined {
if (!forge || !bindingFile || !secretFile) return undefined;
let as = "";
let secret = "";
try {
as = String((JSON.parse(readFileSync(bindingFile, "utf8")) as { as?: string }).as ?? "");
secret = readFileSync(secretFile, "utf8").trim();
} catch {
return undefined;
}
if (!as || !secret) return undefined;
let origin: URL;
try {
origin = new URL(forge);
} catch {
return undefined;
}
const line = new URL(`${origin.protocol}//${origin.host}`);
line.username = as;
line.password = secret;
return line.toString().replace(/\/$/, "");
}
/** Where a run keeps its forge credential: beside its directory, never in it, and removed when it ends. */
export function credentialPath(work: string, id: string): string {
return join(work, `.${id}.git-credentials`);
}
/** writeCredential puts a credential-store line where git reads it, readable by this account alone. */
export function writeCredential(path: string, line: string | undefined): string | undefined {
if (!line) return undefined;
writeFileSync(path, `${line}\n`, { mode: 0o600 });
return path;
}
/**
* credentialGuard is the shell that removes the credential however the script ends — its end, an error,
* or a stop's SIGTERM to the run's process group, which kills the wrapper before its own cleanup runs
* (review 196 on mesh-catalog #272). The script also removes it as soon as its clones are done.
*/
export function credentialGuard(path: string | undefined): string {
if (!path) return "";
const rm = `rm -f ${shellQuote(path)}`;
return `trap ${shellQuote(rm)} EXIT
trap ${shellQuote(`${rm}; exit 143`)} TERM
trap ${shellQuote(`${rm}; exit 130`)} INT
trap ${shellQuote(`${rm}; exit 129`)} HUP`;
}
/**
* cleanCredentials removes every run's credential file left in the work directory but the one named — what
* a crash or a reboot left behind, which no trap could remove. Answers what it removed.
*/
export function cleanCredentials(work: string, keep?: string): string[] {
if (!existsSync(work)) return [];
const removed: string[] = [];
for (const name of readdirSync(work)) {
if (!/^\.run-.*\.git-credentials$/.test(name)) continue;
const path = join(work, name);
if (path === keep) continue;
rmSync(path, { force: true });
removed.push(path);
}
return removed;
}
/**
* checkDir is a directory of lab_check's own for one check, fresh each time (novox/hq issue 624). The check
* cloned into the one fixed path `<work>/check`, removing it first and ignoring whether that worked; a tree
* an earlier check left that this account could not remove — one cloned as another account before the
* work directory was the operator's — made every check fail on "destination path already exists".
*/
export function checkDir(work: string): string {
mkdirSync(work, { recursive: true });
return mkdtempSync(join(work, "check-"));
}
/**
* clearChecks removes what earlier checks left in the work directory — the fixed `check` and `check-setup`
* of before, and any `check-*` — and says, for each it could not remove, which and why, so a check reports
* it rather than tripping over it.
*/
export function clearChecks(work: string): string[] {
if (!existsSync(work)) return [];
const said: string[] = [];
for (const name of readdirSync(work)) {
if (name !== "check" && name !== "check-setup" && !name.startsWith("check-")) continue;
const path = join(work, name);
try {
rmSync(path, { recursive: true, force: true });
} catch (err) {
said.push(`${path}, left by an earlier check, could not be removed: ${(err as Error).message}`);
}
}
return said;
}
/** The arguments that have git offer the credential in that file, and only that one. */
export function credentialArgs(path: string | undefined): string {
if (!path) return "";
return `-c credential.helper= -c ${shellQuote(`credential.helper=store --file=${path}`)} `;
}
export interface RunStatus {
id: string;
state: "checking-out" | "building" | "running" | "passed" | "failed" | "stopped" | "lost";
started: string;
ended?: string;
tests: string[];
refs: Record<string, string>;
commits?: Record<string, string>;
exit?: number;
pid?: number;
}
export function runDir(work: string, id: string): string {
return join(work, id);
}
function statusPath(work: string, id: string): string {
return join(runDir(work, id), "status.json");
}
export function readStatus(work: string, id: string): RunStatus | undefined {
try {
const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus;
// A run whose process is gone while its status still says it is going was lost — the runtime or
// the machine restarted under it. Said, rather than left reading as running for ever.
if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) {
s.state = "lost";
}
return s;
} catch {
return undefined;
}
}
function alive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
export function listRuns(work: string): RunStatus[] {
if (!existsSync(work)) return [];
return readdirSync(work)
.filter((d) => d.startsWith("run-"))
.map((id) => readStatus(work, id))
.filter((s): s is RunStatus => !!s)
.sort((a, b) => b.started.localeCompare(a.started));
}
/** The run still going, if any: one at a time, because two would contend for the same machine. */
export function running(work: string): RunStatus | undefined {
return listRuns(work).find((s) => !["passed", "failed", "stopped", "lost"].includes(s.state));
}
const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`;
/**
* The script one run executes. Every step writes its state first, so a run that dies says where.
*
* The environment is the one the lab's README describes for a run against sibling checkouts, pointed
* at this run's own tree, so what is built and claimed is exactly what was checked out.
*/
export function script(work: string, id: string, forge: string, tests: string[], refs: Record<string, string>,
credential?: string, registry = `${forge}${NPM_PATH}`): string {
const dir = runDir(work, id);
const setState = (state: string) =>
`node -e ${shellQuote(
`const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=${JSON.stringify(state)};require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
)}`;
const clones = REPOSITORIES.map((repo) => {
const ref = refs[repo] ?? "main";
return [
`git ${credentialArgs(credential)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`,
`echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`,
].join("\n");
}).join("\n");
const bin = join(dir, "bin");
return `set -euo pipefail
# A repository the forge will not show fails its clone at once, rather than waiting on a prompt.
export GIT_TERMINAL_PROMPT=0
${credentialGuard(credential)}
cd ${shellQuote(dir)}
${setState("checking-out")}
${clones}
${credential ? `# The forge credential is for the clones, and nothing after them runs with it on disk.\nrm -f ${shellQuote(credential)}` : ""}
node -e ${shellQuote(
`const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`,
)}
${setState("building")}
# The @novox scope resolves from the mesh's own package registry on the forge, named by the address it
# names itself by (npmRegistry), which is the address its lock files hold; nothing else is asked of it.
printf '%s\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))}
export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))}
${NPM_REPOSITORIES.map((repo) => installStep(join(dir, repo), repo !== "mesh-lab")).join("\n")}
mkdir -p ${shellQuote(bin)}
for p in postgres-provisioner objectstore-provisioner route-proxy; do
(cd ${shellQuote(dir)}/mesh-controller && CGO_ENABLED=0 go build -o ${shellQuote(bin)}/$p ./examples/$p)
done
export MESH_LAB_HOST_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-host"))}
export MESH_LAB_BUNDLE=${shellQuote(join(dir, "mesh-host", "examples", "foundation-first-node.lock"))}
export MESH_LAB_MODULES=${shellQuote(join(dir, "mesh-controller", "examples", "modules"))}
export MESH_LAB_BUILDER=${shellQuote(join(dir, "mesh-controller", "build", "mesh-builder"))}
export MESH_LAB_BOOTSTRAP_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-bootstrap"))}
export MESH_LAB_CATALOG=${shellQuote(join(dir, "mesh-catalog", "modules"))}
# Where a bed that serves repositories from its own forge finds them: this run's checkouts, side by side.
export MESH_LAB_REPOS=${shellQuote(dir)}
export MESH_LAB_PROVISIONER=${shellQuote(join(bin, "postgres-provisioner"))}
export MESH_LAB_OBJECTSTORE_PROVISIONER=${shellQuote(join(bin, "objectstore-provisioner"))}
export MESH_LAB_ROUTE_PROXY=${shellQuote(join(bin, "route-proxy"))}
${setState("running")}
cd ${shellQuote(join(dir, "mesh-lab"))}
node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" ")}
`;
}
/** start begins a run and returns at once with its status. */
export function start(work: string, forge: string, tests: string[], refs: Record<string, string>,
credential?: string, registry?: string): RunStatus {
const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`;
const dir = runDir(work, id);
mkdirSync(dir, { recursive: true });
const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs };
writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
// What a crashed or rebooted run left behind goes before this run writes its own.
cleanCredentials(work);
const credentials = writeCredential(credentialPath(work, id), credential);
writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials, registry), { mode: 0o700 });
// The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a
// run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172).
const wrapper = `bash ${shellQuote(join(dir, "run.sh"))} > ${shellQuote(join(dir, "run.log"))} 2>&1; code=$?
node -e ${shellQuote(
`const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
)} "$code"
rm -f ${shellQuote(credentialPath(work, id))}
cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`;
const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
child.unref();
status.pid = child.pid;
writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
return status;
}
/** stop ends a run and everything it started, by its process group. */
export function stop(work: string, id: string): RunStatus | undefined {
const s = readStatus(work, id);
if (!s || !s.pid) return s;
if (["passed", "failed", "stopped", "lost"].includes(s.state)) return s;
s.state = "stopped";
writeFileSync(statusPath(work, id), JSON.stringify(s, null, 2));
try {
process.kill(-s.pid, "SIGTERM");
} catch {
// Already gone between the read and the kill.
}
return s;
}
/** tail is the last lines of a run's log. */
export function tail(work: string, id: string, lines: number): string {
try {
const all = readFileSync(join(runDir(work, id), "run.log"), "utf8").split("\n");
return all.slice(-Math.max(1, lines)).join("\n");
} catch {
return "";
}
}