jochen
0fcc560fec
mesh/merge-gate head 0fcc560fecd6 merged into trunk e846c289c878 — pass: builds lab → shanks; no bus step; every machine composes with the change as it did…
mesh/repo-check head 0fcc560fecd6 merged into trunk e846c289c878 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
A run and a rebuild each asked the package registry between checking for the other and writing their status, so both could start and a bed could raise machines from an image being replaced. The registry is asked first now. A hard-killed rebuild's credential is cleaned like a run's; a job that never recorded its process is lost rather than blocking every run; and lab_check reports a missing or unreadable base image.
235 lines
13 KiB
TypeScript
235 lines
13 KiB
TypeScript
// lab's tools — the lab, as the mesh asks for it (novox/hq ADR 0172). They run on the machine the
|
|
// lab is assigned to, and only there: a bed raises virtual machines on that machine's virtualisation.
|
|
|
|
import { spawnSync } from "node:child_process";
|
|
import { readFileSync } from "node:fs";
|
|
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import {
|
|
checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start,
|
|
stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun,
|
|
launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem,
|
|
} from "./runs.js";
|
|
|
|
export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
|
|
const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs";
|
|
// The forge is an operator's setting, which reaches a file and never a bundle's words (novox/hq
|
|
// ADR 0192): read from the env-file the mesh fills, at each call, so a changed setting is used
|
|
// without restarting the runtime. MESH_LAB_FORGE itself still wins, for a hand-run instance.
|
|
const forgeOf = (): string => (env.MESH_LAB_FORGE ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE")).replace(/\/+$/, "");
|
|
// The forge credential the mesh binds and seals to the lab (novox/hq issue 624), read at each call.
|
|
// The registry's public root the forge serves in the lab's binding (novox/hq ADR 0346), read at each call.
|
|
const bindingOf = (): string | undefined =>
|
|
env.MESH_LAB_FORGE_BINDING ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING") || undefined);
|
|
const registryOf = async (forge: string) => {
|
|
const root = publicRootOf(bindingOf());
|
|
const registry = await npmRegistry(forge, fetch, root.root);
|
|
return registry.problem && root.problem ? { ...registry, problem: `${registry.problem}; ${root.problem}` } : registry;
|
|
};
|
|
const credentialOf = (forge: string): string | undefined => forgeCredential(forge,
|
|
env.MESH_LAB_FORGE_BINDING ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING"),
|
|
env.MESH_LAB_FORGE_SECRET ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_SECRET"));
|
|
return [
|
|
{
|
|
name: "lab_check",
|
|
description: "Whether this machine can run the lab's beds: the lab's own check, against the forge's main branch.",
|
|
input: {},
|
|
run: async () => {
|
|
const forge = forgeOf();
|
|
if (!forge) return { ok: false, output: "the lab's forge is not set: settings for lab, {\"forge\": \"<url>\"}" };
|
|
// What earlier checks left is cleared, and what could not be is said; this check works in a fresh
|
|
// directory of its own either way, and removes it when it is done.
|
|
const leftovers = clearChecks(work);
|
|
const dir = checkDir(work);
|
|
let check: ReturnType<typeof spawnSync>;
|
|
let problems: string[];
|
|
try {
|
|
const clone = spawnSync("git", ["clone", "--quiet", "--depth", "1", `${forge}/novox/mesh-lab.git`, dir], { encoding: "utf8" });
|
|
if (clone.status !== 0) return { ok: false, output: [String(clone.stderr).trim(), ...leftovers].join("\n") };
|
|
spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" });
|
|
check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" });
|
|
problems = setupProblems(forge, `${dir}/setup`, credentialOf(forge));
|
|
// And the address a run names the package registry by (npmRegistry): said when the registry does not say it.
|
|
const registry = await registryOf(forge);
|
|
if (registry.problem) problems.push(registry.problem);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
// **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every
|
|
// repository it clones answers on the forge, and every one it installs has a lock file where the run
|
|
// installs from.
|
|
// A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in
|
|
// use by a running run or rebuild is not.
|
|
const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id);
|
|
const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id)));
|
|
const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : "";
|
|
// **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot
|
|
// be read is a problem here, though lab_run refuses only the first.
|
|
const base = readBaseImage();
|
|
const unfit = baseProblem(base);
|
|
if (unfit) problems.push(unfit);
|
|
const output = [`${check.stdout}${check.stderr}`.trim(), describeBase(base), removed, ...leftovers, ...problems]
|
|
.filter(Boolean).join("\n");
|
|
return { ok: check.status === 0 && problems.length === 0, output };
|
|
},
|
|
},
|
|
{
|
|
name: "lab_run",
|
|
description:
|
|
"Run the lab's beds against branches on the forge: fresh checkouts of every repository the lab builds, " +
|
|
"side by side, then the suite on the named test files. Answers at once with the run's id; lab_status " +
|
|
"and lab_log follow it. One run at a time.",
|
|
input: {
|
|
tests: { type: "string", description: "the bed test files, comma-separated, relative to mesh-lab (e.g. test/integration/mesh.test.ts)" },
|
|
refs: {
|
|
type: "string",
|
|
description: `a JSON object of repository to branch, for any of ${REPOSITORIES.join(", ")}; the rest run main`,
|
|
},
|
|
},
|
|
run: async (args) => {
|
|
const forge = forgeOf();
|
|
if (!forge) return { started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"<url>\"}" };
|
|
const tests = String(args.tests ?? "").split(",").map((s) => s.trim()).filter(Boolean);
|
|
if (tests.length === 0) return { started: false, reason: "name at least one bed test file" };
|
|
let refs: Record<string, string> = {};
|
|
if (args.refs) {
|
|
try {
|
|
refs = JSON.parse(String(args.refs)) as Record<string, string>;
|
|
} catch {
|
|
return { started: false, reason: "refs is not a JSON object of repository to branch" };
|
|
}
|
|
}
|
|
const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r));
|
|
if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` };
|
|
// **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698,
|
|
// review 256): the registry asked first, then the check and the start with nothing awaited between.
|
|
const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() },
|
|
(registry) => start(work, forge, tests, refs, credentialOf(forge), registry));
|
|
if (!launched.started) return launched;
|
|
return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry,
|
|
...(launched.problem ? { problem: launched.problem } : {}) };
|
|
},
|
|
},
|
|
{
|
|
name: "lab_base",
|
|
description:
|
|
"The lab's base image, which every bed machine with a container runtime is raised from: when it was built, its " +
|
|
`age, the bound lab_run refuses past (${BASE_MAX_AGE_DAYS} days), and the last rebuilds with the newest one's log. ` +
|
|
"Reads only, unless rebuild is true: then it rebuilds the image with mesh-lab's `base build` from the forge's " +
|
|
"main, in the background, and answers at once; ask again for the build's state and the image's new age.",
|
|
input: {
|
|
rebuild: { type: "boolean", description: "rebuild the image (default false: only read)" },
|
|
},
|
|
run: async (args) => {
|
|
const before = readBaseImage();
|
|
const builds = listBaseBuilds(work).slice(0, 3);
|
|
const newest = builds[0];
|
|
const reading = {
|
|
base: before, said: describeBase(before), bound: BASE_MAX_AGE_DAYS,
|
|
...(baseProblem(before) ? { problem: baseProblem(before) } : {}),
|
|
builds, ...(newest ? { log: baseBuildLog(work, newest.id) } : {}),
|
|
};
|
|
if (args.rebuild !== true) return reading;
|
|
const forge = forgeOf();
|
|
if (!forge) return { ...reading, started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"<url>\"}" };
|
|
// Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with
|
|
// nothing awaited between (review 256).
|
|
const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before },
|
|
(registry) => startBaseBuild(work, forge, credentialOf(forge), registry));
|
|
if (launched.started === false) return { ...reading, started: false, reason: launched.reason };
|
|
return { ...reading, started: true, build: launched.job };
|
|
},
|
|
},
|
|
{
|
|
name: "lab_status",
|
|
description: "A run's state, the commits it tested and how it ended — or every run, newest first, when no id is given.",
|
|
input: { id: { type: "string", description: "the run's id (optional)" } },
|
|
run: async (args) => {
|
|
if (args.id) return readStatus(work, String(args.id)) ?? { found: false, id: String(args.id) };
|
|
return { runs: listRuns(work).slice(0, 10) };
|
|
},
|
|
},
|
|
{
|
|
name: "lab_log",
|
|
description: "The last lines of a run's log.",
|
|
input: {
|
|
id: { type: "string", description: "the run's id" },
|
|
lines: { type: "number", description: "how many lines from the end (default 200)" },
|
|
},
|
|
run: async (args) => ({ id: String(args.id), log: tail(work, String(args.id), Number(args.lines ?? 200)) }),
|
|
},
|
|
{
|
|
name: "lab_stop",
|
|
description: "Stop a run and everything it started.",
|
|
input: { id: { type: "string", description: "the run's id" } },
|
|
run: async (args) => stop(work, String(args.id)) ?? { found: false, id: String(args.id) },
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* What would stop a run at its setup, in sentences: a repository the run clones that the forge does not
|
|
* answer for, and a checkout it installs with nothing installable (installPlan). Empty when a run's setup
|
|
* would go through.
|
|
*/
|
|
export function setupProblems(forge: string, dir: string, credential?: string): string[] {
|
|
const problems: string[] = [];
|
|
spawnSync("rm", ["-rf", dir]);
|
|
// The credential a run would offer, in a file of its own outside the check's clones, removed after.
|
|
const keep = mkdtempSync(join(tmpdir(), "lab-check-"));
|
|
const file = writeCredential(join(keep, "git-credentials"), credential);
|
|
const git = (args: string[]) =>
|
|
spawnSync("sh", ["-c", `git ${credentialArgs(file)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], {
|
|
encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
|
|
});
|
|
try {
|
|
for (const repo of REPOSITORIES) {
|
|
const url = `${forge}/novox/${repo}.git`;
|
|
const remote = git(["ls-remote", "--heads", url, "main"]);
|
|
if (remote.status !== 0 || !remote.stdout.includes("refs/heads/main")) {
|
|
const said = remote.stderr.trim().split("\n")[0] ?? "";
|
|
problems.push(`a run checks out ${repo}, and ${url} could not be read` +
|
|
(said ? `: ${said}` : ": no main branch") +
|
|
(credential ? "" : " — the lab holds no forge credential (its npm-package-registry binding and secret)"));
|
|
continue;
|
|
}
|
|
if (!NPM_REPOSITORIES.includes(repo)) continue;
|
|
const at = `${dir}/${repo}`;
|
|
const clone = git(["clone", "--quiet", "--depth", "1", url, at]);
|
|
if (clone.status !== 0) {
|
|
problems.push(`${repo} could not be cloned from ${url}: ${clone.stderr.trim()}`);
|
|
continue;
|
|
}
|
|
const plan = installPlan(at);
|
|
for (const d of plan.unlocked) problems.push(`${repo}: ${d.slice(at.length) || "/"} has a package.json and no package-lock.json, so a run's npm ci cannot install it`);
|
|
if (plan.install.length === 0 && plan.unlocked.length === 0) {
|
|
problems.push(`${repo}: no package-lock.json at its root or one directory down, so a run installs nothing of it`);
|
|
}
|
|
}
|
|
} finally {
|
|
rmSync(keep, { recursive: true, force: true });
|
|
spawnSync("rm", ["-rf", dir]);
|
|
}
|
|
return problems;
|
|
}
|
|
|
|
/** One word from an env-file (`KEY=value` lines), or "" when the file or the word is absent. */
|
|
export function wordIn(file: string | undefined, word: string): string {
|
|
if (!file) return "";
|
|
let text: string;
|
|
try {
|
|
text = readFileSync(file, "utf8");
|
|
} catch {
|
|
return "";
|
|
}
|
|
for (const line of text.split("\n")) {
|
|
const at = line.indexOf("=");
|
|
if (at > 0 && line.slice(0, at).trim() === word) return line.slice(at + 1).trim();
|
|
}
|
|
return "";
|
|
}
|
|
|
|
registerModuleTools("lab", (env) => getLabTools(env));
|