lab: a run whose suite proved nothing reads as failed, with its skips (issue 693)

lab_status took the suite's exit as the verdict, and the runner exits 0 when
every test skipped: a genesis bed that skipped its only test as NOT REACHED was
recorded as passed. A run that ended now has its outcome read from its log and
kept in its status - totals, each skip with its reason, and why it did not pass
- and one whose exit said 0 while no test passed reads as failed. Read from the
lab's own lines when it prints them, from the runner's summary for a lab of
before.

modules/lab/tools/runs.ts

@@ -286,6 +286,109 @@ export interface RunStatus {
   commits?: Record<string, string>;
   exit?: number;
   pid?: number;
+  /** What the suite said of its tests, read from the run's log once it ended (novox/hq issue 693). */
+  outcome?: Outcome;
+}
+
+/** One skipped test, its file and the reason it gave. */
+export interface Skip {
+  file: string;
+  test: string;
+  reason: string;
+}
+
+/**
+ * What a run's suite said of its tests: its totals, each skip with its reason, and — when the run did not
+ * pass although its exit said 0 — why (`notPassed`). Totals the runner did not print are absent.
+ */
+export interface Outcome {
+  passed?: number;
+  failed?: number;
+  skipped?: number;
+  skips: Skip[];
+  notPassed?: string;
+}
+
+/**
+ * outcomeOf reads what a run's log says of its tests (novox/hq issue 693).
+ *
+ * **A run in which no test passed has proven nothing**, and the runner exits 0 when every test skipped:
+ * the genesis bed skipped its only test as "NOT REACHED", the suite printed "0 passed, 0 failed", and the
+ * lab recorded the run as passed. The lab's suite now says it itself — a "NOT PASSED: <why>" line and a
+ * "skipped: <test> (<file>) # <reason>" line per skip after its "recorded:" line, and an exit that is not
+ * 0 — and this reads those lines. For a run of a lab from before (a branch, or an old log), the same rule is
+ * read from the runner's own output: its summary totals, and its skip lines' first line of reason.
+ *
+ * The rule, the lab suite's (mesh-lab src/outcomes.ts): **not passed when no test passed**, or when a test
+ * skipped as NOT REACHED (its subject was not reached); a skip for another reason beside tests that passed
+ * leaves the run passed, its reason listed.
+ */
+export function outcomeOf(log: string): Outcome {
+  const plain = log.replace(/\u001b\[[0-9;]*m/g, "");
+  const total = (word: string) => {
+    // The last summary in the log: the runner's own, once per run.
+    const all = [...plain.matchAll(new RegExp(`^\\s*(?:\u2139|#)\\s*${word}\\s+(\\d+)\\s*$`, "gm"))];
+    const last = all[all.length - 1];
+    return last ? Number(last[1]) : undefined;
+  };
+  const outcome: Outcome = { skips: [] };
+  const passed = total("pass");
+  const failed = total("fail");
+  const skipped = total("skipped");
+  if (passed !== undefined) outcome.passed = passed;
+  if (failed !== undefined) outcome.failed = failed;
+  if (skipped !== undefined) outcome.skipped = skipped;
+
+  // The suite's own line, "skipped: <test> (<file>) # <reason>": the file in brackets, a .ts, parts it.
+  for (const m of plain.matchAll(/^skipped: (.*?) \(([^()]+\.ts)\) # (.*)$/gm)) {
+    outcome.skips.push({ file: m[2] ?? "", test: m[1] ?? "", reason: m[3] === "no reason given" ? "" : m[3] ?? "" });
+  }
+  if (outcome.skips.length === 0) {
+    // A lab from before names no skip itself; its runner's spec line does: "﹣ <test> (<ms>ms) # <reason>".
+    for (const m of plain.matchAll(/^\s*\ufe63 (.*?) \([\d.]+m?s\) # (.*)$/gm)) {
+      const reason = (m[2] ?? "").trim();
+      outcome.skips.push({ file: "", test: m[1] ?? "", reason: reason === "SKIP" ? "" : reason });
+    }
+  }
+
+  const said = plain.match(/^NOT PASSED: (.*)$/m);
+  if (said) {
+    outcome.notPassed = (said[1] ?? "").trim();
+  } else if ((failed ?? 0) === 0) {
+    const unreached = outcome.skips.filter((s) => s.reason.includes("NOT REACHED")).length;
+    if (passed === 0) {
+      outcome.notPassed = outcome.skips.length > 0 || (skipped ?? 0) > 0
+        ? `no test passed: ${skipped ?? outcome.skips.length} skipped${unreached > 0 ? `, ${unreached} of them as NOT REACHED` : ""}`
+        : "no test passed: none ran";
+    } else if (unreached > 0) {
+      outcome.notPassed = `${unreached} test(s) skipped as NOT REACHED: their subject was not reached`;
+    }
+  }
+  return outcome;
+}
+
+/**
+ * settled is a run that ended, with its outcome read from its log: a run whose exit said 0 and whose suite
+ * proved nothing reads as failed, with why (novox/hq issue 693). Read once and kept in its status, so the
+ * record says it too; a run still going, stopped or lost is left as it is.
+ */
+export function settled(work: string, s: RunStatus): RunStatus {
+  if (s.outcome || !s.ended || (s.state !== "passed" && s.state !== "failed")) return s;
+  let log: string;
+  try {
+    log = readFileSync(join(runDir(work, s.id), "run.log"), "utf8");
+  } catch {
+    return s;
+  }
+  const outcome = outcomeOf(log);
+  const next: RunStatus = { ...s, outcome };
+  if (next.state === "passed" && outcome.notPassed) next.state = "failed";
+  try {
+    writeFileSync(statusPath(work, s.id), JSON.stringify(next, null, 2));
+  } catch {
+    // Read again next time: what lab_status answers is the same either way.
+  }
+  return next;
 }
 
 export function runDir(work: string, id: string): string {
@@ -304,7 +407,7 @@ export function readStatus(work: string, id: string): RunStatus | undefined {
     if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) {
       s.state = "lost";
     }
-    return s;
+    return settled(work, s);
   } catch {
     return undefined;
   }