A suite run in which no test passed exits non-zero and names its skips (issue 693)

The runner exits 0 when every test skipped, so a bed that stopped before its
subject and skipped its only test as NOT REACHED was recorded as passed, "0
passed, 0 failed". The suite now collects each skip and its reason through a
reporter of its own, prints NOT PASSED and a skipped line per skip after its
recorded line, keeps them in the receipt, and exits 1 when no test passed or a
test was not reached. A skip for a declared reason beside passes stays passed.

src/suite.ts

@@ -10,11 +10,16 @@
  */
 
 import { spawn } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, relative } from "node:path";
+import { fileURLToPath } from "node:url";
 import { endToEnd, record, whatWasTested } from "./lastrun.ts";
+import { judgeRun, oneLine, readSkips, type Outcome, type Skip } from "./outcomes.ts";
 import { rebuild } from "./rebuild.ts";
 
 /** counted is what the runner said, or nulls when it said nothing recognisable. */
-export function counted(output: string): { passed: number | null; failed: number | null } {
+export function counted(output: string): { passed: number | null; failed: number | null; skipped: number | null } {
   // The runner's own summary lines, each on a line of its own. Anchored, so a test *named*
   // "pass 3" cannot be mistaken for the total — which is not a hypothetical worry in a suite whose
   // tests are named in sentences.
@@ -30,6 +35,7 @@ export function counted(output: string): { passed: number | null; failed: number
   return {
     passed: total(/^\s*(?:\u2139|#)\s*pass\s+(\d+)\s*$/m),
     failed: total(/^\s*(?:\u2139|#)\s*fail\s+(\d+)\s*$/m),
+    skipped: total(/^\s*(?:\u2139|#)\s*skipped\s+(\d+)\s*$/m),
   };
 }
 
@@ -57,16 +63,38 @@ export async function runSuite(args: string[]): Promise<number> {
   // `test/integration/canary.test.ts` is still there and still runs when it is named; it is no
   // longer raised on the way to everything else.
 
-  const { code, seen } = await runFiles(files);
-  console.log("\n" + reportOn(counted(seen), (p, f) => record(p, f, files, process.env, against)));
-  return code;
+  const { code, seen, skips } = await runFiles(files);
+  const counts = counted(seen);
+  console.log("\n" + reportOn(counts, skips, (p, f, outcome) => record(p, f, files, process.env, against, outcome)));
+  return exitOf(code, counts, skips);
 }
 
-/** Run some test files, passing their output through as it arrives. */
-async function runFiles(files: string[]): Promise<{ code: number; seen: string }> {
+/**
+ * The suite's exit: the runner's, unless it says 0 for a run that did not pass (novox/hq issue 693). The
+ * runner exits 0 when every test skipped; the lab takes this exit as the run's verdict, so a run that
+ * proved nothing must not end in 0.
+ */
+export function exitOf(code: number, counts: { passed: number | null; failed: number | null }, skips: Skip[]): number {
+  if (code !== 0) return code;
+  if (counts.passed === null || counts.failed === null) return code;
+  return judgeRun(counts, skips).passed ? 0 : 1;
+}
+
+/** The reporter that writes each skipped test and its reason (outcomes.ts), by its path. */
+const skipsReporter = fileURLToPath(new URL("./outcomes.ts", import.meta.url));
+
+/**
+ * Run some test files, passing their output through as it arrives, and collect what was skipped and why:
+ * the spec reporter to stdout as before, and the skips reporter beside it into a file of the run's own.
+ */
+async function runFiles(files: string[]): Promise<{ code: number; seen: string; skips: Skip[] }> {
+  const scratch = mkdtempSync(join(tmpdir(), "mesh-lab-suite-"));
+  const skipsFile = join(scratch, "skips.jsonl");
   const running = spawn(
     process.execPath,
     ["--test", "--test-concurrency=1", "--experimental-strip-types",
+      "--test-reporter=spec", "--test-reporter-destination=stdout",
+      `--test-reporter=${skipsReporter}`, `--test-reporter-destination=${skipsFile}`,
       ...files],
     { stdio: ["inherit", "pipe", "inherit"] },
   );
@@ -81,7 +109,14 @@ async function runFiles(files: string[]): Promise<{ code: number; seen: string }
   const code: number = await new Promise((resolve) => {
     running.on("close", (c) => resolve(c ?? 1));
   });
-  return { code, seen };
+  // Named as the files were given, relative to where the suite runs, rather than as the runner's absolute paths.
+  const named = (file: string) => {
+    const rel = isAbsolute(file) ? relative(process.cwd(), file) : file;
+    return rel.startsWith("..") ? file : rel;
+  };
+  const skips = readSkips(skipsFile).map((s) => ({ ...s, file: named(s.file) }));
+  rmSync(scratch, { recursive: true, force: true });
+  return { code, seen, skips };
 }
 
 /**
@@ -92,16 +127,31 @@ async function runFiles(files: string[]): Promise<{ code: number; seen: string }
  * could check it — which is 04-ISSUES/005 in miniature, inside the fix for it.
  */
 export function reportOn(
-  counts: { passed: number | null; failed: number | null },
-  write: (passed: number, failed: number) => { against: Record<string, string>; ran: string[] },
+  counts: { passed: number | null; failed: number | null; skipped?: number | null },
+  skips: Skip[],
+  write: (passed: number, failed: number, outcome: Outcome) => { against: Record<string, string>; ran: string[] },
 ): string {
   if (counts.passed === null || counts.failed === null) {
     // A run whose result could not be read is a run nobody can say anything about. Writing
     // "0 failed" because nothing said otherwise is how a green record comes to mean nothing.
     return "could not read what the runner reported; no receipt written";
   }
-  const receipt = write(counts.passed, counts.failed);
+  // **A skip is said, with its reason, and a run that proved nothing says NOT PASSED** (novox/hq issue
+  // 693): "0 passed, 0 failed" read as a clean run. The lab module reads these lines into its record.
+  const skipped = counts.skipped ?? skips.length;
+  const judged = judgeRun({ passed: counts.passed, failed: counts.failed }, skips);
+  const outcome: Outcome = {
+    skipped,
+    skips: skips.map((s) => ({ ...s, reason: oneLine(s.reason) })),
+    ...(judged.passed ? {} : { notPassed: judged.why }),
+  };
+  const receipt = write(counts.passed, counts.failed, outcome);
   const against = Object.entries(receipt.against).map(([n, c]) => `${n} ${c}`).join(", ");
-  return `recorded: ${receipt.ran.join(", ")} — ${counts.passed} passed, ` +
-    `${counts.failed} failed, against ${against || "nothing in git"}`;
+  const lines = [`recorded: ${receipt.ran.join(", ")} — ${counts.passed} passed, ` +
+    `${counts.failed} failed, ${skipped} skipped, against ${against || "nothing in git"}`];
+  if (!judged.passed) lines.push(`NOT PASSED: ${judged.why}`);
+  // As the runner's own spec line has it, "<test> (<file>) # <reason>": a test's name may hold any
+  // punctuation, so the file in brackets is what the lab module reads the line apart by.
+  for (const s of outcome.skips) lines.push(`skipped: ${s.test} (${s.file}) # ${s.reason || "no reason given"}`);
+  return lines.join("\n");
 }