/** * Run the end-to-end suite, and leave a receipt saying it ran. * * **Here rather than inside the tests, because the tests cannot know their own totals.** Node's * runner reports them to whatever invoked it, and a test file inventing its own count would be a * receipt that says whatever the last edit made it say. * * Here rather than in a shell script for the same reason the rebuild is: a step that lives in * somebody's terminal history is a step that runs when they remember (novox/hq 04-ISSUES/005). */ import { spawn } from "node:child_process"; import { endToEnd, record, whatWasTested } from "./lastrun.ts"; import { rebuild } from "./rebuild.ts"; /** counted is what the runner said, or nulls when it said nothing recognisable. */ export function counted(output: string): { passed: number | null; failed: number | null } { // The runner's own summary lines, each on a line of its own. Anchored, so a test *named* // "pass 3" cannot be mistaken for the total — which is not a hypothetical worry in a suite whose // tests are named in sentences. // Stripped first: the runner colours its summary even when its stdout is a pipe, so the line is // "\x1b[34m\u2139 pass 8\x1b[39m" and an anchored pattern never sees the start of it. Found by // running this against the real runner — the fixture it was first written against was output I // had imagined, which is a test that agrees with the mistake it was written beside. const plain = output.replace(/\u001b\[[0-9;]*m/g, ""); const total = (what: RegExp) => { const found = plain.match(what); return found ? Number(found[1]) : null; }; return { passed: total(/^\s*(?:\u2139|#)\s*pass\s+(\d+)\s*$/m), failed: total(/^\s*(?:\u2139|#)\s*fail\s+(\d+)\s*$/m), }; } export async function runSuite(args: string[]): Promise { const ran = args.filter((a) => a !== "--no-build"); const files = ran.length > 0 ? ran : [endToEnd]; if (!args.includes("--no-build")) { // Before the run, always. The artifacts are built from two other repositories, and a suite // that tests yesterday's binary reports on code nobody is looking at (novox/hq 04-ISSUES/005). const built = rebuild(process.env, files); if (built.length > 0) console.log(`built: ${built.join(", ")}\n`); } // Read now, while it is true. The receipt names these, and reading them when the run ends // names whatever was committed during the twenty minutes in between instead. const against = whatWasTested(process.env); // **No canary.** There was one: a second scenario, one machine, raised first so a broken mesh // failed in two minutes rather than in forty. It walked exactly the path the first three tests // of the long run walk — a mesh comes up, a module lands, a consumer gets a credential — and // the long run reaches the end of that path in about 160 seconds. // // So it cost a whole scenario, every passing run, to save about 45 seconds on a failing one. // A scenario is machines that each boot a kernel, which is where the two minutes went. // `test/integration/canary.test.ts` is still there and still runs when it is named; it is no // longer raised on the way to everything else. const { code, seen } = await runFiles(files); console.log("\n" + reportOn(counted(seen), (p, f) => record(p, f, files, process.env, against))); return code; } /** Run some test files, passing their output through as it arrives. */ async function runFiles(files: string[]): Promise<{ code: number; seen: string }> { const running = spawn( process.execPath, ["--test", "--test-concurrency=1", "--experimental-strip-types", ...files], { stdio: ["inherit", "pipe", "inherit"] }, ); let seen = ""; running.stdout.on("data", (chunk: Buffer) => { // Passed through as it arrives: a suite that takes a quarter of an hour must not look hung. process.stdout.write(chunk); seen += chunk.toString(); }); const code: number = await new Promise((resolve) => { running.on("close", (c) => resolve(c ?? 1)); }); return { code, seen }; } /** * reportOn decides whether this run says anything worth recording, and records it if so. * * Separated from the spawning so the decision can be tested: **no receipt rather than a guessed * one** is the rule that keeps the record meaning something, and it was written where nothing * could check it — which is 04-ISSUES/005 in miniature, inside the fix for it. */ export function reportOn( counts: { passed: number | null; failed: number | null }, write: (passed: number, failed: number) => { against: Record; ran: string[] }, ): string { if (counts.passed === null || counts.failed === null) { // A run whose result could not be read is a run nobody can say anything about. Writing // "0 failed" because nothing said otherwise is how a green record comes to mean nothing. return "could not read what the runner reported; no receipt written"; } const receipt = write(counts.passed, counts.failed); const against = Object.entries(receipt.against).map(([n, c]) => `${n} ${c}`).join(", "); return `recorded: ${receipt.ran.join(", ")} — ${counts.passed} passed, ` + `${counts.failed} failed, against ${against || "nothing in git"}`; }