// A run in which no test passed has proven nothing, and says so (novox/hq issue 693). // // Seen on the lab: the genesis bed stopped before its subject, skipped its only test as "NOT REACHED", // and the suite printed "0 passed, 0 failed" and exited 0, so the lab recorded the run as passed. import { test } from "node:test"; import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { judge, type Receipt, endToEnd } from "../src/lastrun.ts"; import { judgeRun, oneLine, type Skip } from "../src/outcomes.ts"; import { counted, exitOf, reportOn } from "../src/suite.ts"; // The suite as a person runs it: not as a child of this test runner, which would have it report to us. const suiteEnv = (state: string): NodeJS.ProcessEnv => { const env: NodeJS.ProcessEnv = { ...process.env, XDG_STATE_HOME: state, NO_COLOR: "1" }; delete env["NODE_TEST_CONTEXT"]; return env; }; const lab = fileURLToPath(new URL("..", import.meta.url)); const unreached: Skip = { file: "test/integration/genesis-keeps-its-bus.test.ts", test: "genesis adopts its bus", reason: "ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:\n docker exited 1", }; const noBed: Skip = { file: "test/integration/mesh.test.ts", test: "a builder builds", reason: "set MESH_LAB_BUILDER to a built mesh-builder" }; test("a run whose only test skipped did not pass", () => { const said = judgeRun({ passed: 0, failed: 0 }, [unreached]); assert.equal(said.passed, false); assert.match(said.why, /no test passed: 1 skipped, 1 of them as NOT REACHED/); }); test("a run that ran no test at all did not pass", () => { assert.deepEqual(judgeRun({ passed: 0, failed: 0 }, []), { passed: false, why: "no test passed: none ran" }); }); test("a NOT REACHED skip beside tests that passed is still not passed", () => { const said = judgeRun({ passed: 3, failed: 0 }, [unreached]); assert.equal(said.passed, false); assert.match(said.why, /1 test\(s\) skipped as NOT REACHED/); }); // The design limit: a skip its file declares expected, beside tests that passed, leaves the run passed. test("an expected skip beside tests that passed leaves the run passed", () => { assert.deepEqual(judgeRun({ passed: 21, failed: 0 }, [noBed]), { passed: true, why: "" }); }); test("the suite's exit is not 0 for a run that did not pass, and is the runner's otherwise", () => { assert.equal(exitOf(0, { passed: 0, failed: 0 }, [unreached]), 1); assert.equal(exitOf(0, { passed: 21, failed: 0 }, [noBed]), 0); assert.equal(exitOf(1, { passed: 20, failed: 1 }, []), 1); assert.equal(exitOf(0, { passed: null, failed: null }, []), 0, "unread totals are reportOn's case, not this one"); }); test("the recorded line says what skipped and why, and NOT PASSED", () => { let kept: unknown; const said = reportOn({ passed: 0, failed: 0, skipped: 1 }, [unreached], (_p, _f, outcome) => { kept = outcome; return { against: { "mesh-lab": "7ed7da1" }, ran: [unreached.file] }; }); assert.match(said, /— 0 passed, 0 failed, 1 skipped, against mesh-lab 7ed7da1/); assert.match(said, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m); assert.match(said, /^skipped: genesis adopts its bus \(test\/integration\/genesis-keeps-its-bus\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step: docker exited 1$/m); assert.deepEqual(kept, { skipped: 1, skips: [{ ...unreached, reason: oneLine(unreached.reason) }], notPassed: "no test passed: 1 skipped, 1 of them as NOT REACHED", }); }); // A test that prints a summary-shaped line before it skips must not stand for the runner's summary. test("the runner's last summary is read, not a line a test printed before it", () => { const output = "\u2139 pass 5\n\u2139 fail 0\n\u2139 skipped 0\n\ufe63 genesis adopts its bus (1ms) # NOT REACHED\n" + "\u2139 tests 1\n\u2139 pass 0\n\u2139 fail 0\n\u2139 skipped 1\n"; const counts = counted(output); assert.deepEqual(counts, { passed: 0, failed: 0, skipped: 1 }); assert.equal(exitOf(0, counts, [unreached]), 1); }); test("NOT REACHED is matched in any case", () => { const said = judgeRun({ passed: 2, failed: 0 }, [{ ...noBed, reason: "the broker step was not reached" }]); assert.equal(said.passed, false); }); test("the runner's skipped total is read", () => { assert.deepEqual(counted("ℹ tests 1\nℹ pass 0\nℹ fail 0\nℹ skipped 1\n"), { passed: 0, failed: 0, skipped: 1 }); }); test("a receipt of a run that proved nothing is not current, and says why", () => { const receipt: Receipt = { at: "2026-10-14T00:33:11Z", passed: 0, failed: 0, against: { "mesh-lab": "aaa" }, ran: [endToEnd], skipped: 1, skips: [unreached], notPassed: "no test passed: 1 skipped", }; const said = judge(receipt, new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" }); assert.equal(said.current, false, said.lines.join("\n")); const text = said.lines.join("\n"); assert.match(text, /did not pass: no test passed: 1 skipped/); assert.match(text, /skipped: genesis adopts its bus — ADR 0333 §5 NOT REACHED/); // An older receipt, from before the fields, with nothing passed, is judged by its count. const older = judge({ at: receipt.at, passed: 0, failed: 0, against: receipt.against, ran: [endToEnd] }, new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" }); assert.equal(older.current, false); }); // The real runner, end to end: the suite command against a file whose only test skips as NOT REACHED, // the way the genesis bed does. test("the suite exits non-zero for a file whose only test skipped as NOT REACHED, and records why", () => { const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-")); try { const file = join(dir, "unreached.test.ts"); writeFileSync(file, [ `import { test } from "node:test";`, `test("genesis adopts its bus", (t) => {`, ` console.log("a line the bed printed");`, ` t.skip("ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base:\\nNotFound: content digest");`, `});`, ].join("\n")); const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], { cwd: lab, encoding: "utf8", env: suiteEnv(dir), }); assert.equal(ran.status, 1, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`); assert.match(ran.stdout, /0 passed, 0 failed, 1 skipped/); assert.match(ran.stdout, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m); assert.match(ran.stdout, /^skipped: genesis adopts its bus \(.*unreached\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest$/m); const receipt = JSON.parse(readFileSync(join(dir, "mesh-lab", "last-run.json"), "utf8")) as Receipt; assert.equal(receipt.skipped, 1); assert.equal(receipt.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED"); assert.match(receipt.skips?.[0]?.reason ?? "", /NOT REACHED/); } finally { rmSync(dir, { recursive: true, force: true }); } }); test("the suite does not take a summary a test printed for the runner's", () => { const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-")); try { const file = join(dir, "forged.test.ts"); writeFileSync(file, [ `import { test } from "node:test";`, `test("genesis adopts its bus", (t) => {`, ` console.log("\\u2139 pass 5\\n\\u2139 fail 0");`, ` t.skip("ADR 0333 §5 NOT REACHED: stopped at step 16");`, `});`, ].join("\n")); const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], { cwd: lab, encoding: "utf8", env: suiteEnv(dir), }); assert.equal(ran.status, 1, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`); assert.match(ran.stdout, /0 passed, 0 failed, 1 skipped/); } finally { rmSync(dir, { recursive: true, force: true }); } }); test("the suite still exits 0 for a file that passes a test and skips another for a declared reason", () => { const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-")); try { const file = join(dir, "declared.test.ts"); writeFileSync(file, [ `import { test } from "node:test";`, `test("a module lands", () => {});`, `test("a builder builds", { skip: "set MESH_LAB_BUILDER to a built mesh-builder" }, () => {});`, ].join("\n")); const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], { cwd: lab, encoding: "utf8", env: suiteEnv(dir), }); assert.equal(ran.status, 0, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`); assert.match(ran.stdout, /1 passed, 0 failed, 1 skipped/); assert.doesNotMatch(ran.stdout, /NOT PASSED/); assert.match(ran.stdout, /^skipped: a builder builds \(.*declared\.test\.ts\) # set MESH_LAB_BUILDER to a built mesh-builder$/m); } finally { rmSync(dir, { recursive: true, force: true }); } });