A suite run in which no test passed exits non-zero and names its skips (issue 693)

The runner exits 0 when every test skipped, so a bed that stopped before its
subject and skipped its only test as NOT REACHED was recorded as passed, "0
passed, 0 failed". The suite now collects each skip and its reason through a
reporter of its own, prints NOT PASSED and a skipped line per skip after its
recorded line, keeps them in the receipt, and exits 1 when no test passed or a
test was not reached. A skip for a declared reason beside passes stays passed.

test/outcomes.test.ts

@@ -0,0 +1,144 @@
+// A run in which no test passed has proven nothing, and says so (novox/hq issue 693).
+//
+// Seen on the lab: the genesis bed stopped before its subject, skipped its only test as "NOT REACHED",
+// and the suite printed "0 passed, 0 failed" and exited 0, so the lab recorded the run as passed.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { judge, type Receipt, endToEnd } from "../src/lastrun.ts";
+import { judgeRun, oneLine, type Skip } from "../src/outcomes.ts";
+import { counted, exitOf, reportOn } from "../src/suite.ts";
+
+// The suite as a person runs it: not as a child of this test runner, which would have it report to us.
+const suiteEnv = (state: string): NodeJS.ProcessEnv => {
+  const env: NodeJS.ProcessEnv = { ...process.env, XDG_STATE_HOME: state, NO_COLOR: "1" };
+  delete env["NODE_TEST_CONTEXT"];
+  return env;
+};
+
+const lab = fileURLToPath(new URL("..", import.meta.url));
+const unreached: Skip = {
+  file: "test/integration/genesis-keeps-its-bus.test.ts",
+  test: "genesis adopts its bus",
+  reason: "ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:\n  docker exited 1",
+};
+const noBed: Skip = { file: "test/integration/mesh.test.ts", test: "a builder builds", reason: "set MESH_LAB_BUILDER to a built mesh-builder" };
+
+test("a run whose only test skipped did not pass", () => {
+  const said = judgeRun({ passed: 0, failed: 0 }, [unreached]);
+  assert.equal(said.passed, false);
+  assert.match(said.why, /no test passed: 1 skipped, 1 of them as NOT REACHED/);
+});
+
+test("a run that ran no test at all did not pass", () => {
+  assert.deepEqual(judgeRun({ passed: 0, failed: 0 }, []), { passed: false, why: "no test passed: none ran" });
+});
+
+test("a NOT REACHED skip beside tests that passed is still not passed", () => {
+  const said = judgeRun({ passed: 3, failed: 0 }, [unreached]);
+  assert.equal(said.passed, false);
+  assert.match(said.why, /1 test\(s\) skipped as NOT REACHED/);
+});
+
+// The design limit: a skip its file declares expected, beside tests that passed, leaves the run passed.
+test("an expected skip beside tests that passed leaves the run passed", () => {
+  assert.deepEqual(judgeRun({ passed: 21, failed: 0 }, [noBed]), { passed: true, why: "" });
+});
+
+test("the suite's exit is not 0 for a run that did not pass, and is the runner's otherwise", () => {
+  assert.equal(exitOf(0, { passed: 0, failed: 0 }, [unreached]), 1);
+  assert.equal(exitOf(0, { passed: 21, failed: 0 }, [noBed]), 0);
+  assert.equal(exitOf(1, { passed: 20, failed: 1 }, []), 1);
+  assert.equal(exitOf(0, { passed: null, failed: null }, []), 0, "unread totals are reportOn's case, not this one");
+});
+
+test("the recorded line says what skipped and why, and NOT PASSED", () => {
+  let kept: unknown;
+  const said = reportOn({ passed: 0, failed: 0, skipped: 1 }, [unreached], (_p, _f, outcome) => {
+    kept = outcome;
+    return { against: { "mesh-lab": "7ed7da1" }, ran: [unreached.file] };
+  });
+  assert.match(said, /— 0 passed, 0 failed, 1 skipped, against mesh-lab 7ed7da1/);
+  assert.match(said, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m);
+  assert.match(said, /^skipped: genesis adopts its bus \(test\/integration\/genesis-keeps-its-bus\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step: docker exited 1$/m);
+  assert.deepEqual(kept, {
+    skipped: 1,
+    skips: [{ ...unreached, reason: oneLine(unreached.reason) }],
+    notPassed: "no test passed: 1 skipped, 1 of them as NOT REACHED",
+  });
+});
+
+test("the runner's skipped total is read", () => {
+  assert.deepEqual(counted("ℹ tests 1\nℹ pass 0\nℹ fail 0\nℹ skipped 1\n"), { passed: 0, failed: 0, skipped: 1 });
+});
+
+test("a receipt of a run that proved nothing is not current, and says why", () => {
+  const receipt: Receipt = {
+    at: "2026-10-14T00:33:11Z", passed: 0, failed: 0, against: { "mesh-lab": "aaa" }, ran: [endToEnd],
+    skipped: 1, skips: [unreached], notPassed: "no test passed: 1 skipped",
+  };
+  const said = judge(receipt, new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" });
+  assert.equal(said.current, false, said.lines.join("\n"));
+  const text = said.lines.join("\n");
+  assert.match(text, /did not pass: no test passed: 1 skipped/);
+  assert.match(text, /skipped: genesis adopts its bus — ADR 0333 §5 NOT REACHED/);
+  // An older receipt, from before the fields, with nothing passed, is judged by its count.
+  const older = judge({ at: receipt.at, passed: 0, failed: 0, against: receipt.against, ran: [endToEnd] },
+    new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" });
+  assert.equal(older.current, false);
+});
+
+// The real runner, end to end: the suite command against a file whose only test skips as NOT REACHED,
+// the way the genesis bed does.
+test("the suite exits non-zero for a file whose only test skipped as NOT REACHED, and records why", () => {
+  const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
+  try {
+    const file = join(dir, "unreached.test.ts");
+    writeFileSync(file, [
+      `import { test } from "node:test";`,
+      `test("genesis adopts its bus", (t) => {`,
+      `  console.log("a line the bed printed");`,
+      `  t.skip("ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base:\\nNotFound: content digest");`,
+      `});`,
+    ].join("\n"));
+    const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
+      cwd: lab, encoding: "utf8", env: suiteEnv(dir),
+    });
+    assert.equal(ran.status, 1, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
+    assert.match(ran.stdout, /0 passed, 0 failed, 1 skipped/);
+    assert.match(ran.stdout, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m);
+    assert.match(ran.stdout, /^skipped: genesis adopts its bus \(.*unreached\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest$/m);
+    const receipt = JSON.parse(readFileSync(join(dir, "mesh-lab", "last-run.json"), "utf8")) as Receipt;
+    assert.equal(receipt.skipped, 1);
+    assert.equal(receipt.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
+    assert.match(receipt.skips?.[0]?.reason ?? "", /NOT REACHED/);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test("the suite still exits 0 for a file that passes a test and skips another for a declared reason", () => {
+  const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
+  try {
+    const file = join(dir, "declared.test.ts");
+    writeFileSync(file, [
+      `import { test } from "node:test";`,
+      `test("a module lands", () => {});`,
+      `test("a builder builds", { skip: "set MESH_LAB_BUILDER to a built mesh-builder" }, () => {});`,
+    ].join("\n"));
+    const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
+      cwd: lab, encoding: "utf8", env: suiteEnv(dir),
+    });
+    assert.equal(ran.status, 0, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
+    assert.match(ran.stdout, /1 passed, 0 failed, 1 skipped/);
+    assert.doesNotMatch(ran.stdout, /NOT PASSED/);
+    assert.match(ran.stdout, /^skipped: a builder builds \(.*declared\.test\.ts\) # set MESH_LAB_BUILDER to a built mesh-builder$/m);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});

test/suite.test.ts

@@ -5,7 +5,7 @@ import { counted, reportOn } from "../src/suite.ts";
 // The totals come from the runner's own summary, and from nothing else.
 test("the runner's totals are read from its summary", () => {
   const said = counted("✔ something (1ms)\nℹ tests 22\nℹ pass 22\nℹ fail 0\n");
-  assert.deepEqual(said, { passed: 22, failed: 0 });
+  assert.deepEqual(said, { passed: 22, failed: 0, skipped: null });
 });
 
 // A test *named* like a total must not be mistaken for one. The summary is a line of its own, and
@@ -18,7 +18,7 @@ test("a test named like a total is not a total", () => {
 // A failing run is read as a failing run.
 test("failures are read", () => {
   const said = counted("ℹ pass 21\nℹ fail 1\n");
-  assert.deepEqual(said, { passed: 21, failed: 1 });
+  assert.deepEqual(said, { passed: 21, failed: 1, skipped: null });
 });
 
 // **No totals is not zero failures.** A run whose result could not be read is a run nobody can say
@@ -36,7 +36,7 @@ test("output with no summary yields no totals rather than a clean bill", () => {
 // check it — 04-ISSUES/005 in miniature, inside the fix for it.
 test("a run whose result could not be read writes nothing", () => {
   let wrote = false;
-  const said = reportOn({ passed: null, failed: null }, () => {
+  const said = reportOn({ passed: null, failed: null }, [], () => {
     wrote = true;
     return { against: {}, ran: [] };
   });
@@ -46,12 +46,13 @@ test("a run whose result could not be read writes nothing", () => {
 
 test("a run that was read is recorded, with what it was read against", () => {
   let got: [number, number] | null = null;
-  const said = reportOn({ passed: 22, failed: 0 }, (p, f) => {
+  const said = reportOn({ passed: 22, failed: 0 }, [], (p, f) => {
     got = [p, f];
     return { against: { "mesh-lab": "abc1234" }, ran: ["test/integration/mesh.test.ts"] };
   });
   assert.deepEqual(got, [22, 0]);
-  assert.match(said, /mesh\.test\.ts — 22 passed, 0 failed, against mesh-lab abc1234/);
+  assert.match(said, /mesh\.test\.ts — 22 passed, 0 failed, 0 skipped, against mesh-lab abc1234/);
+  assert.doesNotMatch(said, /NOT PASSED/);
 });
 
 // The runner's real output, colours and all.
@@ -64,5 +65,5 @@ test("the runner's totals are read from output as it actually arrives", () => {
     "\u001b[34m\u2139 pass 22\u001b[39m\n" +
     "\u001b[34m\u2139 fail 0\u001b[39m\n" +
     "\u001b[34m\u2139 duration_ms 98.9\u001b[39m\n";
-  assert.deepEqual(counted(real), { passed: 22, failed: 0 });
+  assert.deepEqual(counted(real), { passed: 22, failed: 0, skipped: null });
 });