804 lines
40 KiB
TypeScript
804 lines
40 KiB
TypeScript
// A lab run: fresh checkouts of the named branches, side by side, then the lab's suite on the named
|
|
// beds (novox/hq ADR 0172).
|
|
//
|
|
// **A run is a detached script with its own process group**, so it outlives the tool call that started
|
|
// it and `stop` ends everything it started. It writes what it is doing to a status file beside its log,
|
|
// and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers
|
|
// for it, and says it was lost rather than pretending it is still going.
|
|
|
|
import { spawn, spawnSync } from "node:child_process";
|
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
|
|
/**
|
|
* The repositories a lab run checks out, side by side, as the lab expects its siblings. mesh-tools-go is
|
|
* one since genesis builds the Go toolchain at its base step, from the repository beside mesh-tools
|
|
* (novox/hq issue 616): a bed that raises genesis serves it from its forge.
|
|
*/
|
|
export const REPOSITORIES = ["mesh-lab", "mesh-controller", "mesh-host", "mesh-catalog", "mesh-tools", "mesh-tools-go", "mesh-sdk"];
|
|
|
|
/** The checkouts whose JavaScript dependencies a run installs: the lab, the SDK and the base. */
|
|
export const NPM_REPOSITORIES = ["mesh-sdk", "mesh-tools", "mesh-lab"];
|
|
|
|
/**
|
|
* Where a checkout's JavaScript dependencies are installed (novox/hq issue 624): its root when the root
|
|
* holds a lock file, otherwise each directory one level down that holds one. `npm ci` installs only from a
|
|
* lock file, and a repository that moved its package into a directory of its own (mesh-tools' node-tools)
|
|
* has none at its root; the run asked for one there and failed before it built anything. A package.json
|
|
* with no lock file beside it cannot be installed reproducibly, and is named rather than installed.
|
|
*/
|
|
export function installPlan(root: string): { install: string[]; unlocked: string[] } {
|
|
const has = (dir: string, file: string) => existsSync(join(dir, file));
|
|
if (has(root, "package-lock.json")) return { install: [root], unlocked: [] };
|
|
const install: string[] = [];
|
|
const unlocked: string[] = has(root, "package.json") ? [root] : [];
|
|
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
|
if (!entry.isDirectory() || entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
const dir = join(root, entry.name);
|
|
if (has(dir, "package-lock.json")) install.push(dir);
|
|
else if (has(dir, "package.json")) unlocked.push(dir);
|
|
}
|
|
return { install: install.sort(), unlocked: unlocked.sort() };
|
|
}
|
|
|
|
/**
|
|
* installStep is the shell a run installs one checkout with: the same rule as installPlan, so what
|
|
* lab_check says a run would install is what it installs. Building runs where the package says how.
|
|
*/
|
|
export function installStep(root: string, build: boolean): string {
|
|
const q = shellQuote(root);
|
|
const install = `(cd "$d" && npm ci --no-audit --no-fund --loglevel=error${build ? " && npm run build --if-present" : ""})`;
|
|
return `if [ -f ${q}/package-lock.json ]; then d=${q}; ${install}; else
|
|
[ ! -f ${q}/package.json ] || { echo "${root}: a package.json and no package-lock.json, so npm ci cannot install it (novox/hq issue 624)" >&2; exit 1; }
|
|
found=0
|
|
for d in ${q}/*/; do
|
|
d="\${d%/}"; [ "$(basename "$d")" = node_modules ] && continue
|
|
if [ -f "$d/package-lock.json" ]; then found=1; ${install}
|
|
elif [ -f "$d/package.json" ]; then echo "$d: a package.json and no package-lock.json, so npm ci cannot install it (novox/hq issue 624)" >&2; exit 1; fi
|
|
done
|
|
[ "$found" = 1 ] || { echo "${root}: no package-lock.json at its root or one directory down, so nothing of it can be installed (novox/hq issue 624)" >&2; exit 1; }
|
|
fi`;
|
|
}
|
|
|
|
/** Where the forge serves the mesh's npm packages, under its own address (gitea's npm registry for the novox org). */
|
|
export const NPM_PATH = "/api/packages/novox/npm/";
|
|
|
|
/**
|
|
* The registry's public root, as the forge serves it in the lab's npm-package-registry binding
|
|
* (`serves.public-root`, novox/hq ADR 0346): the forge's route's composed public name under the npm path.
|
|
* The mesh writes the binding, and it is checked all the same: an https URL whose path is the npm path and
|
|
* nothing else. Undefined, with why, when the binding has none or it is not that shape — a forge on a
|
|
* machine with no public domain serves none (ADR 0346 §3).
|
|
*/
|
|
export function publicRootOf(bindingFile: string | undefined): { root?: string; problem?: string } {
|
|
if (!bindingFile) return { problem: "the lab has no npm-package-registry binding" };
|
|
let said: unknown;
|
|
try {
|
|
said = (JSON.parse(readFileSync(bindingFile, "utf8")) as { serves?: Record<string, unknown> }).serves?.["public-root"];
|
|
} catch (e) {
|
|
return { problem: `its npm-package-registry binding could not be read: ${e instanceof Error ? e.message : String(e)}` };
|
|
}
|
|
if (typeof said !== "string" || !said) return { problem: "its npm-package-registry binding serves no public-root" };
|
|
if (/[\s\u0000-\u001f\u007f]/.test(said)) return { problem: "the binding's public-root holds whitespace or a control character" };
|
|
let u: URL;
|
|
try {
|
|
u = new URL(said);
|
|
} catch {
|
|
return { problem: `the binding's public-root ${JSON.stringify(said)} is not a URL` };
|
|
}
|
|
if (u.protocol !== "https:" || u.username || u.password || u.search || u.hash || u.pathname !== NPM_PATH) {
|
|
return { problem: `the binding's public-root ${JSON.stringify(said)} is not https://<host>${NPM_PATH}` };
|
|
}
|
|
return { root: `${u.origin}${NPM_PATH}` };
|
|
}
|
|
|
|
/**
|
|
* The address a run names the @novox registry by: the one the registry names itself by, when that is the
|
|
* forge's public root or the forge's own address (novox/hq issue 624, ADR 0346).
|
|
*
|
|
* **npm 12 installs a locked tarball only from the registry it is configured with.** Its `allow-remote`
|
|
* defaults to `none`, and a lock file's `resolved` URL counts as the registry's only when its origin and
|
|
* path are the configured scope registry's; any other is a "remote" tarball and refused (EALLOWREMOTE).
|
|
* The forge writes every tarball URL — in what it answers, and so in every lock file made against it — at
|
|
* its own public address, whichever address it was asked at. So the lab asks the registry, at the forge,
|
|
* for the SDK's newest tarball and names the registry by the origin of that URL.
|
|
*
|
|
* **What the registry answers is not trusted for where packages come from** (review of mesh-catalog #275):
|
|
* it may be fetched over plain HTTP. The tarball URL is parsed, and taken only when it carries no
|
|
* whitespace, control character, user, password, query or fragment, its path is under the npm path, and
|
|
* its origin — scheme, host and port — is one the lab already trusts: the public root the forge serves in
|
|
* the lab's binding (`publicRoot`, ADR 0346), or the forge setting's own. The registry is then built from
|
|
* the parsed origin and the npm path alone, never from the string answered, so nothing the registry says
|
|
* reaches the run's npm configuration but a scheme, a host and a port. Anything else falls back to the
|
|
* forge's own address and says why, so a run still starts and lab_check names it.
|
|
*/
|
|
export async function npmRegistry(forge: string, get: typeof fetch = fetch, publicRoot?: string):
|
|
Promise<{ registry: string; problem?: string }> {
|
|
const asked = `${forge.replace(/\/+$/, "")}${NPM_PATH}`;
|
|
const url = `${asked}@novox%2fmesh-sdk`;
|
|
const fallback = (why: string) => ({
|
|
registry: asked,
|
|
problem: `the package registry at ${url} did not give an address the lab takes (${why}), so a run names it ${asked}, and npm 12 refuses a locked @novox tarball at any other`,
|
|
});
|
|
let own: URL;
|
|
try {
|
|
own = new URL(forge);
|
|
} catch {
|
|
return fallback(`the forge ${JSON.stringify(forge)} is not a URL`);
|
|
}
|
|
// The forge setting's own origin is trusted only as an http(s) one: any other scheme's origin is
|
|
// "null", or names no registry npm would fetch from.
|
|
const trusted = own.protocol === "https:" || own.protocol === "http:" ? [own.origin] : [];
|
|
if (publicRoot) {
|
|
try {
|
|
trusted.unshift(new URL(publicRoot).origin);
|
|
} catch {
|
|
// publicRootOf hands only a parsed root; a caller's other string is not trusted.
|
|
}
|
|
}
|
|
try {
|
|
const res = await get(url, { signal: AbortSignal.timeout(15_000) });
|
|
if (!res.ok) return fallback(`it answered ${res.status}`);
|
|
const doc = await res.json() as { "dist-tags"?: { latest?: string }; versions?: Record<string, { dist?: { tarball?: unknown } }> };
|
|
const latest = doc["dist-tags"]?.latest ?? "";
|
|
const tarball = doc.versions?.[latest]?.dist?.tarball;
|
|
if (typeof tarball !== "string" || !tarball) return fallback("no tarball for its latest version");
|
|
if (/[\s\u0000-\u001f\u007f]/.test(tarball)) return fallback("its tarball URL holds whitespace or a control character");
|
|
let t: URL;
|
|
try {
|
|
t = new URL(tarball);
|
|
} catch {
|
|
return fallback("its tarball is not a URL");
|
|
}
|
|
if (t.username || t.password) return fallback("its tarball URL carries a user or password");
|
|
if (t.search || t.hash) return fallback("its tarball URL carries a query or a fragment");
|
|
if (!t.pathname.startsWith(NPM_PATH)) return fallback(`its tarball is not under ${NPM_PATH}`);
|
|
if (!trusted.includes(t.origin)) {
|
|
return fallback(`its tarball is at ${t.origin}, which is neither the forge's public root nor its own address (${trusted.join(", ")})`);
|
|
}
|
|
return { registry: `${t.origin}${NPM_PATH}` };
|
|
} catch (e) {
|
|
return fallback(e instanceof Error ? e.message : String(e));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The forge credential a run's clones may present, as a git credential-store line —
|
|
* scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none.
|
|
*
|
|
* **The mesh's credential for the forge, the way the build agent holds it** (novox/hq issue 624): the lab
|
|
* requires the package registry, the forge's provisioner binds it an account (`as`) and seals that
|
|
* account's password, and the forge that answers npm is the forge that hosts the repositories. Private
|
|
* repositories (mesh-tools-go) are cloned with it; public ones clone as before. Written to a file git reads
|
|
* through its credential store, never into a URL, an argument, the run's script or its log, and offered
|
|
* only to the forge's own scheme, host and port.
|
|
*/
|
|
export function forgeCredential(forge: string, bindingFile: string, secretFile: string): string | undefined {
|
|
if (!forge || !bindingFile || !secretFile) return undefined;
|
|
let as = "";
|
|
let secret = "";
|
|
try {
|
|
as = String((JSON.parse(readFileSync(bindingFile, "utf8")) as { as?: string }).as ?? "");
|
|
secret = readFileSync(secretFile, "utf8").trim();
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
if (!as || !secret) return undefined;
|
|
let origin: URL;
|
|
try {
|
|
origin = new URL(forge);
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
const line = new URL(`${origin.protocol}//${origin.host}`);
|
|
line.username = as;
|
|
line.password = secret;
|
|
return line.toString().replace(/\/$/, "");
|
|
}
|
|
|
|
/** Where a run keeps its forge credential: beside its directory, never in it, and removed when it ends. */
|
|
export function credentialPath(work: string, id: string): string {
|
|
return join(work, `.${id}.git-credentials`);
|
|
}
|
|
|
|
/** writeCredential puts a credential-store line where git reads it, readable by this account alone. */
|
|
export function writeCredential(path: string, line: string | undefined): string | undefined {
|
|
if (!line) return undefined;
|
|
writeFileSync(path, `${line}\n`, { mode: 0o600 });
|
|
return path;
|
|
}
|
|
|
|
/**
|
|
* credentialGuard is the shell that removes the credential however the script ends — its end, an error,
|
|
* or a stop's SIGTERM to the run's process group, which kills the wrapper before its own cleanup runs
|
|
* (review 196 on mesh-catalog #272). The script also removes it as soon as its clones are done.
|
|
*/
|
|
export function credentialGuard(path: string | undefined): string {
|
|
if (!path) return "";
|
|
const rm = `rm -f ${shellQuote(path)}`;
|
|
return `trap ${shellQuote(rm)} EXIT
|
|
trap ${shellQuote(`${rm}; exit 143`)} TERM
|
|
trap ${shellQuote(`${rm}; exit 130`)} INT
|
|
trap ${shellQuote(`${rm}; exit 129`)} HUP`;
|
|
}
|
|
|
|
/**
|
|
* cleanCredentials removes every run's and every base rebuild's credential file left in the work directory
|
|
* but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it
|
|
* removed.
|
|
*/
|
|
export function cleanCredentials(work: string, keep?: string | string[]): string[] {
|
|
if (!existsSync(work)) return [];
|
|
const kept = new Set([keep ?? []].flat());
|
|
const removed: string[] = [];
|
|
for (const name of readdirSync(work)) {
|
|
if (!/^\.(run|base)-.*\.git-credentials$/.test(name)) continue;
|
|
const path = join(work, name);
|
|
if (kept.has(path)) continue;
|
|
rmSync(path, { force: true });
|
|
removed.push(path);
|
|
}
|
|
return removed;
|
|
}
|
|
|
|
/**
|
|
* checkDir is a directory of lab_check's own for one check, fresh each time (novox/hq issue 624). The check
|
|
* cloned into the one fixed path `<work>/check`, removing it first and ignoring whether that worked; a tree
|
|
* an earlier check left that this account could not remove — one cloned as another account before the
|
|
* work directory was the operator's — made every check fail on "destination path already exists".
|
|
*/
|
|
export function checkDir(work: string): string {
|
|
mkdirSync(work, { recursive: true });
|
|
return mkdtempSync(join(work, "check-"));
|
|
}
|
|
|
|
/**
|
|
* clearChecks removes what earlier checks left in the work directory — the fixed `check` and `check-setup`
|
|
* of before, and any `check-*` — and says, for each it could not remove, which and why, so a check reports
|
|
* it rather than tripping over it.
|
|
*/
|
|
export function clearChecks(work: string): string[] {
|
|
if (!existsSync(work)) return [];
|
|
const said: string[] = [];
|
|
for (const name of readdirSync(work)) {
|
|
if (name !== "check" && name !== "check-setup" && !name.startsWith("check-")) continue;
|
|
const path = join(work, name);
|
|
try {
|
|
rmSync(path, { recursive: true, force: true });
|
|
} catch (err) {
|
|
said.push(`${path}, left by an earlier check, could not be removed: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
return said;
|
|
}
|
|
|
|
/** The arguments that have git offer the credential in that file, and only that one. */
|
|
export function credentialArgs(path: string | undefined): string {
|
|
if (!path) return "";
|
|
return `-c credential.helper= -c ${shellQuote(`credential.helper=store --file=${path}`)} `;
|
|
}
|
|
|
|
export interface RunStatus {
|
|
id: string;
|
|
state: "checking-out" | "building" | "running" | "passed" | "failed" | "stopped" | "lost";
|
|
started: string;
|
|
ended?: string;
|
|
tests: string[];
|
|
refs: Record<string, string>;
|
|
commits?: Record<string, string>;
|
|
exit?: number;
|
|
pid?: number;
|
|
/** What the suite said of its tests, read from the run's log once it ended (novox/hq issue 693). */
|
|
outcome?: Outcome;
|
|
}
|
|
|
|
/** One skipped test, its file and the reason it gave. */
|
|
export interface Skip {
|
|
file: string;
|
|
test: string;
|
|
reason: string;
|
|
}
|
|
|
|
/**
|
|
* What a run's suite said of its tests: its totals, each skip with its reason, and — when the run did not
|
|
* pass although its exit said 0 — why (`notPassed`). Totals the runner did not print are absent.
|
|
*/
|
|
export interface Outcome {
|
|
passed?: number;
|
|
failed?: number;
|
|
skipped?: number;
|
|
skips: Skip[];
|
|
notPassed?: string;
|
|
}
|
|
|
|
/**
|
|
* outcomeOf reads what a run's log says of its tests (novox/hq issue 693).
|
|
*
|
|
* **A run in which no test passed has proven nothing**, and the runner exits 0 when every test skipped:
|
|
* the genesis bed skipped its only test as "NOT REACHED", the suite printed "0 passed, 0 failed", and the
|
|
* lab recorded the run as passed. The lab's suite now says it itself — a "NOT PASSED: <why>" line and a
|
|
* "skipped: <test> (<file>) # <reason>" line per skip after its "recorded:" line, and an exit that is not
|
|
* 0 — and this reads those lines. For a run of a lab from before (a branch, or an old log), the same rule is
|
|
* read from the runner's own output: its summary totals, and its skip lines' first line of reason.
|
|
*
|
|
* The rule, the lab suite's (mesh-lab src/outcomes.ts): **not passed when no test passed**, or when a test
|
|
* skipped as NOT REACHED (its subject was not reached); a skip for another reason beside tests that passed
|
|
* leaves the run passed, its reason listed.
|
|
*/
|
|
export function outcomeOf(log: string): Outcome {
|
|
const plain = log.replace(/\u001b\[[0-9;]*m/g, "");
|
|
const total = (word: string) => {
|
|
// The last summary in the log: the runner's own, once per run.
|
|
const all = [...plain.matchAll(new RegExp(`^\\s*(?:\u2139|#)\\s*${word}\\s+(\\d+)\\s*$`, "gm"))];
|
|
const last = all[all.length - 1];
|
|
return last ? Number(last[1]) : undefined;
|
|
};
|
|
const outcome: Outcome = { skips: [] };
|
|
const passed = total("pass");
|
|
const failed = total("fail");
|
|
const skipped = total("skipped");
|
|
if (passed !== undefined) outcome.passed = passed;
|
|
if (failed !== undefined) outcome.failed = failed;
|
|
if (skipped !== undefined) outcome.skipped = skipped;
|
|
|
|
// **The suite's own lines are read only after its last "recorded:" line** (novox/hq issue 693, review
|
|
// 254): a test prints what it likes before the suite reports, and a line of its that looked like the
|
|
// suite's — "NOT PASSED: …", "skipped: … # NOT REACHED" — would otherwise stand for the suite's verdict.
|
|
const recorded = [...plain.matchAll(/^recorded: .*$/gm)];
|
|
const lastRecorded = recorded[recorded.length - 1];
|
|
const suiteSaid = lastRecorded ? plain.slice((lastRecorded.index ?? 0) + lastRecorded[0].length) : "";
|
|
|
|
// The suite's own line, "skipped: <test> (<file>) # <reason>": the file in brackets, a .ts, parts it.
|
|
for (const m of suiteSaid.matchAll(/^skipped: (.*?) \(([^()]+\.ts)\) # (.*)$/gm)) {
|
|
outcome.skips.push({ file: m[2] ?? "", test: m[1] ?? "", reason: m[3] === "no reason given" ? "" : m[3] ?? "" });
|
|
}
|
|
if (outcome.skips.length === 0) {
|
|
// A lab from before names no skip itself; its runner's spec line does: "﹣ <test> (<ms>ms) # <reason>".
|
|
for (const m of plain.matchAll(/^\s*\ufe63 (.*?) \([\d.]+m?s\) # (.*)$/gm)) {
|
|
const reason = (m[2] ?? "").trim();
|
|
outcome.skips.push({ file: "", test: m[1] ?? "", reason: reason === "SKIP" ? "" : reason });
|
|
}
|
|
}
|
|
|
|
const said = suiteSaid.match(/^NOT PASSED: (.*)$/m);
|
|
if (said) {
|
|
outcome.notPassed = (said[1] ?? "").trim();
|
|
} else if ((failed ?? 0) === 0) {
|
|
// In any case: "not reached" says the same as "NOT REACHED".
|
|
const unreached = outcome.skips.filter((s) => s.reason.toUpperCase().includes("NOT REACHED")).length;
|
|
if (passed === 0) {
|
|
outcome.notPassed = outcome.skips.length > 0 || (skipped ?? 0) > 0
|
|
? `no test passed: ${skipped ?? outcome.skips.length} skipped${unreached > 0 ? `, ${unreached} of them as NOT REACHED` : ""}`
|
|
: "no test passed: none ran";
|
|
} else if (unreached > 0) {
|
|
outcome.notPassed = `${unreached} test(s) skipped as NOT REACHED: their subject was not reached`;
|
|
}
|
|
}
|
|
return outcome;
|
|
}
|
|
|
|
/**
|
|
* settled is a run that ended, with its outcome read from its log: a run whose exit said 0 and whose suite
|
|
* proved nothing reads as failed, with why (novox/hq issue 693). Read once and kept in its status, so the
|
|
* record says it too; a run still going, stopped or lost is left as it is.
|
|
*/
|
|
export function settled(work: string, s: RunStatus): RunStatus {
|
|
if (s.outcome || !s.ended || (s.state !== "passed" && s.state !== "failed")) return s;
|
|
let log: string;
|
|
try {
|
|
log = readFileSync(join(runDir(work, s.id), "run.log"), "utf8");
|
|
} catch {
|
|
return s;
|
|
}
|
|
const outcome = outcomeOf(log);
|
|
const next: RunStatus = { ...s, outcome };
|
|
if (next.state === "passed" && outcome.notPassed) next.state = "failed";
|
|
try {
|
|
writeFileSync(statusPath(work, s.id), JSON.stringify(next, null, 2));
|
|
} catch {
|
|
// Read again next time: what lab_status answers is the same either way.
|
|
}
|
|
return next;
|
|
}
|
|
|
|
export function runDir(work: string, id: string): string {
|
|
return join(work, id);
|
|
}
|
|
|
|
function statusPath(work: string, id: string): string {
|
|
return join(runDir(work, id), "status.json");
|
|
}
|
|
|
|
export function readStatus(work: string, id: string): RunStatus | undefined {
|
|
try {
|
|
const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus;
|
|
// A run whose process is gone while its status still says it is going was lost — the runtime or
|
|
// the machine restarted under it. Said, rather than left reading as running for ever. One that never
|
|
// recorded a process (the runtime died between writing its status and starting it) is lost too: it
|
|
// would otherwise hold up every run after it (review 256).
|
|
if (!["passed", "failed", "stopped", "lost"].includes(s.state) && gone(s.pid)) {
|
|
s.state = "lost";
|
|
}
|
|
return settled(work, s);
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
/** gone is whether a job's process is not running: none recorded, or none by that id. */
|
|
function gone(pid: number | undefined): boolean {
|
|
return !pid || !alive(pid);
|
|
}
|
|
|
|
function alive(pid: number): boolean {
|
|
try {
|
|
process.kill(pid, 0);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
export function listRuns(work: string): RunStatus[] {
|
|
if (!existsSync(work)) return [];
|
|
return readdirSync(work)
|
|
.filter((d) => d.startsWith("run-"))
|
|
.map((id) => readStatus(work, id))
|
|
.filter((s): s is RunStatus => !!s)
|
|
.sort((a, b) => b.started.localeCompare(a.started));
|
|
}
|
|
|
|
/** The run still going, if any: one at a time, because two would contend for the same machine. */
|
|
export function running(work: string): RunStatus | undefined {
|
|
return listRuns(work).find((s) => !["passed", "failed", "stopped", "lost"].includes(s.state));
|
|
}
|
|
|
|
const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`;
|
|
|
|
/**
|
|
* The script one run executes. Every step writes its state first, so a run that dies says where.
|
|
*
|
|
* The environment is the one the lab's README describes for a run against sibling checkouts, pointed
|
|
* at this run's own tree, so what is built and claimed is exactly what was checked out.
|
|
*/
|
|
export function script(work: string, id: string, forge: string, tests: string[], refs: Record<string, string>,
|
|
credential?: string, registry = `${forge}${NPM_PATH}`): string {
|
|
const dir = runDir(work, id);
|
|
const setState = (state: string) =>
|
|
`node -e ${shellQuote(
|
|
`const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=${JSON.stringify(state)};require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
|
|
)}`;
|
|
const clones = REPOSITORIES.map((repo) => {
|
|
const ref = refs[repo] ?? "main";
|
|
return [
|
|
`git ${credentialArgs(credential)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`,
|
|
`echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`,
|
|
].join("\n");
|
|
}).join("\n");
|
|
const bin = join(dir, "bin");
|
|
return `set -euo pipefail
|
|
# A repository the forge will not show fails its clone at once, rather than waiting on a prompt.
|
|
export GIT_TERMINAL_PROMPT=0
|
|
${credentialGuard(credential)}
|
|
cd ${shellQuote(dir)}
|
|
${setState("checking-out")}
|
|
${clones}
|
|
${credential ? `# The forge credential is for the clones, and nothing after them runs with it on disk.\nrm -f ${shellQuote(credential)}` : ""}
|
|
node -e ${shellQuote(
|
|
`const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`,
|
|
)}
|
|
${setState("building")}
|
|
# The @novox scope resolves from the mesh's own package registry on the forge, named by the address it
|
|
# names itself by (npmRegistry), which is the address its lock files hold; nothing else is asked of it.
|
|
printf '%s\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))}
|
|
export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))}
|
|
${NPM_REPOSITORIES.map((repo) => installStep(join(dir, repo), repo !== "mesh-lab")).join("\n")}
|
|
mkdir -p ${shellQuote(bin)}
|
|
for p in postgres-provisioner objectstore-provisioner route-proxy; do
|
|
(cd ${shellQuote(dir)}/mesh-controller && CGO_ENABLED=0 go build -o ${shellQuote(bin)}/$p ./examples/$p)
|
|
done
|
|
export MESH_LAB_HOST_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-host"))}
|
|
export MESH_LAB_BUNDLE=${shellQuote(join(dir, "mesh-host", "examples", "foundation-first-node.lock"))}
|
|
export MESH_LAB_MODULES=${shellQuote(join(dir, "mesh-controller", "examples", "modules"))}
|
|
export MESH_LAB_BUILDER=${shellQuote(join(dir, "mesh-controller", "build", "mesh-builder"))}
|
|
export MESH_LAB_BOOTSTRAP_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-bootstrap"))}
|
|
export MESH_LAB_CATALOG=${shellQuote(join(dir, "mesh-catalog", "modules"))}
|
|
# Where a bed that serves repositories from its own forge finds them: this run's checkouts, side by side.
|
|
export MESH_LAB_REPOS=${shellQuote(dir)}
|
|
export MESH_LAB_PROVISIONER=${shellQuote(join(bin, "postgres-provisioner"))}
|
|
export MESH_LAB_OBJECTSTORE_PROVISIONER=${shellQuote(join(bin, "objectstore-provisioner"))}
|
|
export MESH_LAB_ROUTE_PROXY=${shellQuote(join(bin, "route-proxy"))}
|
|
${setState("running")}
|
|
cd ${shellQuote(join(dir, "mesh-lab"))}
|
|
node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" ")}
|
|
`;
|
|
}
|
|
|
|
/** start begins a run and returns at once with its status. */
|
|
export function start(work: string, forge: string, tests: string[], refs: Record<string, string>,
|
|
credential?: string, registry?: string): RunStatus {
|
|
const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
|
const dir = runDir(work, id);
|
|
mkdirSync(dir, { recursive: true });
|
|
const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs };
|
|
writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
|
|
// What a crashed or rebooted run left behind goes before this run writes its own.
|
|
cleanCredentials(work);
|
|
const credentials = writeCredential(credentialPath(work, id), credential);
|
|
writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials, registry), { mode: 0o700 });
|
|
|
|
// The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a
|
|
// run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172).
|
|
const wrapper = `bash ${shellQuote(join(dir, "run.sh"))} > ${shellQuote(join(dir, "run.log"))} 2>&1; code=$?
|
|
node -e ${shellQuote(
|
|
`const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
|
|
)} "$code"
|
|
rm -f ${shellQuote(credentialPath(work, id))}
|
|
cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`;
|
|
const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
|
|
child.unref();
|
|
status.pid = child.pid;
|
|
writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
|
|
return status;
|
|
}
|
|
|
|
/** stop ends a run and everything it started, by its process group. */
|
|
export function stop(work: string, id: string): RunStatus | undefined {
|
|
const s = readStatus(work, id);
|
|
if (!s || !s.pid) return s;
|
|
if (["passed", "failed", "stopped", "lost"].includes(s.state)) return s;
|
|
s.state = "stopped";
|
|
writeFileSync(statusPath(work, id), JSON.stringify(s, null, 2));
|
|
try {
|
|
process.kill(-s.pid, "SIGTERM");
|
|
} catch {
|
|
// Already gone between the read and the kill.
|
|
}
|
|
return s;
|
|
}
|
|
|
|
/** tail is the last lines of a run's log. */
|
|
export function tail(work: string, id: string, lines: number): string {
|
|
try {
|
|
const all = readFileSync(join(runDir(work, id), "run.log"), "utf8").split("\n");
|
|
return all.slice(-Math.max(1, lines)).join("\n");
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
// ---- the base image (novox/hq issue 698)
|
|
//
|
|
// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which
|
|
// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the
|
|
// package database the image was built with. On a rolling distribution the mirrors stop serving what an old
|
|
// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror
|
|
// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a
|
|
// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image
|
|
// older than the bound, and rebuilds the image when asked to.
|
|
//
|
|
// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is
|
|
// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds
|
|
// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little;
|
|
// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test.
|
|
|
|
/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */
|
|
export const BASE_IMAGE_ALIAS = "mesh-lab/base";
|
|
|
|
/** How old the base image may be, in days, before a run is refused (see the head of this file). */
|
|
export const BASE_MAX_AGE_DAYS = 14;
|
|
|
|
const DAY = 24 * 60 * 60 * 1000;
|
|
|
|
export interface BaseImage {
|
|
alias: string;
|
|
/** Whether the image is there at all. */
|
|
exists: boolean;
|
|
/** When this machine made it, ISO. */
|
|
built?: string;
|
|
/** Its age in whole days. */
|
|
ageDays?: number;
|
|
/** Why its age could not be read, when it could not. */
|
|
problem?: string;
|
|
}
|
|
|
|
/**
|
|
* baseImageFrom reads `incus image list <alias> --format json`: the image whose aliases hold the base's
|
|
* name, and when it was made here (its upload, which a publish sets).
|
|
*/
|
|
export function baseImageFrom(listed: string | undefined, now: Date): BaseImage {
|
|
const alias = BASE_IMAGE_ALIAS;
|
|
if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" };
|
|
let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>;
|
|
try {
|
|
images = JSON.parse(listed);
|
|
} catch {
|
|
return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" };
|
|
}
|
|
const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias));
|
|
if (!image) return { alias, exists: false };
|
|
const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN))
|
|
.find((t) => Number.isFinite(t) && t > 0);
|
|
if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" };
|
|
return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) };
|
|
}
|
|
|
|
/** readBaseImage asks incus, on this machine, for the base image. */
|
|
export function readBaseImage(now = new Date()): BaseImage {
|
|
const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" });
|
|
return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now);
|
|
}
|
|
|
|
/** The base image's age in one sentence, as lab_check and lab_run say it. */
|
|
export function describeBase(b: BaseImage): string {
|
|
if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`;
|
|
if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`;
|
|
return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`;
|
|
}
|
|
|
|
/**
|
|
* staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the
|
|
* bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says
|
|
* so itself, and a bed that does not need it runs.
|
|
*/
|
|
export function staleBase(b: BaseImage): string | undefined {
|
|
if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined;
|
|
return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` +
|
|
`${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` +
|
|
`(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`;
|
|
}
|
|
|
|
/**
|
|
* baseProblem is what lab_check and lab_base say is wrong with the base image, or undefined: past the bound, not
|
|
* on this machine, or of an age that cannot be read (review 256). Only the first refuses a run.
|
|
*/
|
|
export function baseProblem(b: BaseImage): string | undefined {
|
|
if (b.problem || !b.exists) return describeBase(b);
|
|
return staleBase(b);
|
|
}
|
|
|
|
/**
|
|
* runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which
|
|
* replaces the image a bed raises its machines from, or an image older than the bound.
|
|
*/
|
|
export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined {
|
|
if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`;
|
|
return staleBase(base);
|
|
}
|
|
|
|
// ---- its rebuild: mesh-lab's `base build`, in the background, as a run is.
|
|
|
|
export interface BaseBuild {
|
|
id: string;
|
|
state: "building" | "built" | "failed" | "lost";
|
|
started: string;
|
|
ended?: string;
|
|
exit?: number;
|
|
pid?: number;
|
|
}
|
|
|
|
function buildStatusPath(work: string, id: string): string {
|
|
return join(work, id, "status.json");
|
|
}
|
|
|
|
/** The base builds, newest first. A build whose process is gone while it says building was lost. */
|
|
export function listBaseBuilds(work: string): BaseBuild[] {
|
|
if (!existsSync(work)) return [];
|
|
const builds: BaseBuild[] = [];
|
|
for (const id of readdirSync(work).filter((d) => d.startsWith("base-"))) {
|
|
try {
|
|
const b = JSON.parse(readFileSync(buildStatusPath(work, id), "utf8")) as BaseBuild;
|
|
if (b.state === "building" && gone(b.pid)) b.state = "lost";
|
|
builds.push(b);
|
|
} catch {
|
|
// A directory without a status is not a build.
|
|
}
|
|
}
|
|
return builds.sort((a, b) => b.started.localeCompare(a.started));
|
|
}
|
|
|
|
/** The base build still going, if any. */
|
|
export function buildingBase(work: string): BaseBuild | undefined {
|
|
return listBaseBuilds(work).find((b) => b.state === "building");
|
|
}
|
|
|
|
/** The last lines of a base build's log. */
|
|
export function baseBuildLog(work: string, id: string, lines = 40): string {
|
|
try {
|
|
return readFileSync(join(work, id, "build.log"), "utf8").split("\n").slice(-Math.max(1, lines)).join("\n");
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it,
|
|
* then its `base build`, which replaces the image and reads back that it works before publishing it.
|
|
*/
|
|
export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string {
|
|
const lab = join(dir, "mesh-lab");
|
|
return `set -euo pipefail
|
|
export GIT_TERMINAL_PROMPT=0
|
|
${credentialGuard(credential)}
|
|
cd ${shellQuote(dir)}
|
|
git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)}
|
|
${credential ? `rm -f ${shellQuote(credential)}` : ""}
|
|
echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)"
|
|
printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))}
|
|
export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))}
|
|
${installStep(lab, false)}
|
|
cd ${shellQuote(lab)}
|
|
node --experimental-strip-types src/cli.ts base build
|
|
`;
|
|
}
|
|
|
|
/** startBaseBuild rebuilds the base image in the background and answers at once with the build. */
|
|
export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild {
|
|
const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
|
const dir = join(work, id);
|
|
mkdirSync(dir, { recursive: true });
|
|
const build: BaseBuild = { id, state: "building", started: new Date().toISOString() };
|
|
writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
|
|
// What a crashed or rebooted run or rebuild left behind goes before this one writes its own.
|
|
cleanCredentials(work);
|
|
const credentials = writeCredential(credentialPath(work, id), credential);
|
|
writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 });
|
|
const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$?
|
|
node -e ${shellQuote(
|
|
`const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
|
|
)} "$code"
|
|
rm -f ${shellQuote(credentialPath(work, id))}
|
|
rm -rf ${shellQuote(join(dir, "mesh-lab"))}`;
|
|
const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
|
|
child.unref();
|
|
build.pid = child.pid;
|
|
writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
|
|
return build;
|
|
}
|
|
|
|
// ---- starting a run or a rebuild: one job on the lab's machine at a time (review 256).
|
|
//
|
|
// **Everything that waits is done first; the check and the start are one step.** A run and a rebuild each
|
|
// refused while the other was going, but each asked the package registry — an await — between its check and
|
|
// writing its status, so a run and a rebuild asked together could both pass their checks and both start: a
|
|
// bed raising machines from an image being replaced under it. Here the registry is asked first, and nothing
|
|
// is awaited between reading what is going and writing the new job's status, so in the one runtime that
|
|
// serves the lab's tools no other call runs in between.
|
|
|
|
export interface Launch {
|
|
/** The address a run or a rebuild names the package registry by, asked of the forge (npmRegistry). */
|
|
registry: () => Promise<{ registry: string; problem?: string }>;
|
|
/** The base image as it is now. */
|
|
base: () => BaseImage;
|
|
}
|
|
|
|
export type Launched<T> = { started: true; registry: string; problem?: string; base: BaseImage; job: T }
|
|
| { started: false; reason: string; base: BaseImage };
|
|
|
|
/** launchRun starts a run unless a run or a rebuild is going, or the base image is past its bound. */
|
|
export async function launchRun(work: string, launch: Launch, begin: (registry: string) => RunStatus):
|
|
Promise<Launched<RunStatus>> {
|
|
const registry = await launch.registry();
|
|
// From here to begin(): nothing awaited.
|
|
const base = launch.base();
|
|
const busy = running(work);
|
|
if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, base };
|
|
const refused = runRefusal(base, buildingBase(work));
|
|
if (refused) return { started: false, reason: refused, base };
|
|
return { started: true, ...registry, base, job: begin(registry.registry) };
|
|
}
|
|
|
|
/** launchRebuild starts a rebuild of the base image unless a run or another rebuild is going. */
|
|
export async function launchRebuild(work: string, launch: Launch, begin: (registry: string) => BaseBuild):
|
|
Promise<Launched<BaseBuild>> {
|
|
const registry = await launch.registry();
|
|
// From here to begin(): nothing awaited.
|
|
const base = launch.base();
|
|
const rebuilding = buildingBase(work);
|
|
if (rebuilding) return { started: false, reason: `${rebuilding.id} is still rebuilding it`, base };
|
|
// A run raises its machines from the image; replacing it under one would change what the run tests.
|
|
const busy = running(work);
|
|
if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image`, base };
|
|
return { started: true, ...registry, base, job: begin(registry.registry) };
|
|
}
|