/** * When the end-to-end suite last ran, and against what. * * **The fault this exists for is not that the suite breaks — it is that nobody notices it stopped * running** (novox/hq 04-ISSUES/005). The harness it replaces had not built for two and a half * months, and nothing said so; the coverage was assumed rather than checked, and several of the * pipeline's most expensive defects landed inside that window. * * This suite is in a better position and the same danger: it needs a machine with a hypervisor, so * it cannot run on every push, which means it runs when somebody remembers. Remembering is not a * mechanism. * * So a run leaves a receipt, and something can be asked whether the receipt still means anything. * A receipt that is old, or taken against code the repositories have since moved past, is the * thing 005 says nobody was ever told. * * **Kept outside the repository**, because the question is *has this machine run it* rather than * *what is committed* — and a receipt in git would be a claim about everyone's machine made by * whoever committed last. */ import { execFileSync } from "node:child_process"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, join } from "node:path"; import { repositories } from "./repos.ts"; /** What a run was taken against, per repository. */ export type Against = Record; export interface Receipt { /** When it finished, ISO 8601. */ at: string; passed: number; failed: number; /** The commit each repository was at. Absent for anything that was not a git checkout. */ against: Against; /** The test files this run was pointed at. See {@link endToEnd}. */ ran: string[]; } /** * endToEnd is the file that raises real machines. A run that did not include it proved nothing * about the pipeline, however green it was. * * The suite takes paths, so it can be pointed at one quick file — and the receipt from that would * otherwise be indistinguishable from a receipt for the real thing. That is 04-ISSUES/005 again: * not a suite that fails, a record that says more than the run behind it. */ export const endToEnd = "test/integration/mesh.test.ts"; /** Where the receipt lives: XDG state, which is for exactly this — data a tool keeps between runs. */ export function receiptPath(): string { const state = process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state"); return join(state, "mesh-lab", "last-run.json"); } /** * headOf is the commit a directory's repository is at, or "" if it is not one. * * **A tree with uncommitted changes is marked, and never equal to the clean commit it sits on.** * The run tested what was on disk, and that is not what the commit contains — so a receipt naming * the bare hash would claim coverage of code nobody can check out. Nothing else could tell: the * hash is identical either way. That is 04-ISSUES/005's overclaim in its quietest form. */ export function headOf(directory: string): string { const git = (args: string[]) => execFileSync("git", ["-C", directory, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], }); try { const head = git(["rev-parse", "--short", "HEAD"]).trim(); const dirty = git(["status", "--porcelain"]).trim() !== ""; return dirty ? `${head}+uncommitted` : head; } catch { // Not a checkout, or no git. Absent rather than guessed: a receipt claiming a commit it did // not read is worse than one that says it could not tell. return ""; } } /** * whatWasTested is the repositories this run exercised, by the paths it was given. * * From the environment rather than a fixed list, because the paths are how the suite is told what * to run — so anything it was pointed at is something the receipt should account for, and anything * it was not pointed at was not tested. */ export function whatWasTested(env: NodeJS.ProcessEnv = process.env): Against { const against: Against = {}; for (const [name, directory] of Object.entries(repositories(env))) { const head = headOf(directory); if (head) against[name] = head; } return against; } /** record writes the receipt. Failures are recorded too: a run that failed still ran. */ export function record( passed: number, failed: number, ran: string[], env = process.env, ): Receipt { const receipt: Receipt = { at: new Date().toISOString(), passed, failed, against: whatWasTested(env), ran, }; const path = receiptPath(); mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, JSON.stringify(receipt, null, 2) + "\n"); return receipt; } /** read returns the receipt, or null when this machine has never run the suite. */ export function read(): Receipt | null { try { return JSON.parse(readFileSync(receiptPath(), "utf8")) as Receipt; } catch { return null; } } export interface Verdict { /** True when the receipt still says something about the code as it stands. */ current: boolean; lines: string[]; } /** * judge says whether the last run still means anything. * * Two ways it can stop meaning something, and they read differently: it was long ago, or the code * has moved since. The second is the one that matters — a suite that passed against code nobody * runs any more is coverage in name. */ export function judge( receipt: Receipt | null, now: Date, against: Against, staleAfterDays = 7, ): Verdict { if (!receipt) { return { current: false, lines: [ "this machine has never run the end-to-end suite.", " Nothing here has been checked end to end, which is not the same as nothing being wrong.", ], }; } // A receipt written before this field existed says nothing about what it ran, and the honest // reading of "nothing said" is not "everything". const endToEndRan = (receipt.ran ?? []).some((path) => path.endsWith(endToEnd)); const days = (now.getTime() - Date.parse(receipt.at)) / 86_400_000; const lines: string[] = []; const outcome = receipt.failed > 0 ? `last ran ${ago(days)} and ${receipt.failed} test(s) failed` : `last passed ${ago(days)}, ${receipt.passed} test(s)`; lines.push(`the end-to-end suite ${outcome}`); let moved = false; for (const name of Object.keys(against).sort()) { const then = receipt.against[name]; const now = against[name]; if (!then) { lines.push(` ${name.padEnd(14)} was not accounted for in that run`); moved = true; continue; } if (then === now) { lines.push(` ${name.padEnd(14)} at ${then} (unchanged)`); continue; } lines.push(` ${name.padEnd(14)} at ${then}, now at ${now}`); moved = true; } if (!endToEndRan) { lines.push(` That run did not include ${endToEnd}, so it raised no machines.`); } if (receipt.failed > 0) { lines.push(" Nothing has been proven end to end since."); } else if (moved) { lines.push(" What it proved was proven about code that has since changed."); } else if (days > staleAfterDays) { lines.push(` Nothing has changed since, but that was more than ${staleAfterDays} days ago.`); } return { current: endToEndRan && receipt.failed === 0 && !moved && days <= staleAfterDays, lines, }; } function ago(days: number): string { if (days < 1 / 24) return "less than an hour ago"; if (days < 1) return `${Math.round(days * 24)} hour(s) ago`; return `${Math.round(days)} day(s) ago`; }