mesh-lab/src/suite.ts

/**
 * Run the end-to-end suite, and leave a receipt saying it ran.
 *
 * **Here rather than inside the tests, because the tests cannot know their own totals.** Node's
 * runner reports them to whatever invoked it, and a test file inventing its own count would be a
 * receipt that says whatever the last edit made it say.
 *
 * Here rather than in a shell script for the same reason the rebuild is: a step that lives in
 * somebody's terminal history is a step that runs when they remember (novox/hq 04-ISSUES/005).
 */

import { spawn } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { isAbsolute, join, relative } from "node:path";
import { fileURLToPath } from "node:url";
import { endToEnd, record, whatWasTested } from "./lastrun.ts";
import { judgeRun, oneLine, readSkips, type Outcome, type Skip } from "./outcomes.ts";
import { rebuild } from "./rebuild.ts";

/** counted is what the runner said, or nulls when it said nothing recognisable. */
export function counted(output: string): { passed: number | null; failed: number | null; skipped: number | null } {
  // The runner's own summary lines, each on a line of its own. Anchored, so a test *named*
  // "pass 3" cannot be mistaken for the total — which is not a hypothetical worry in a suite whose
  // tests are named in sentences.
  // Stripped first: the runner colours its summary even when its stdout is a pipe, so the line is
  // "\x1b[34m\u2139 pass 8\x1b[39m" and an anchored pattern never sees the start of it. Found by
  // running this against the real runner — the fixture it was first written against was output I
  // had imagined, which is a test that agrees with the mistake it was written beside.
  const plain = output.replace(/\u001b\[[0-9;]*m/g, "");
  // **The last such line, never the first** (novox/hq issue 693): the runner prints its summary after
  // everything a test printed, so a test that prints "ℹ pass 5" before it skips would otherwise be read
  // as five passes, and a run that proved nothing as passed.
  const total = (what: RegExp) => {
    const found = [...plain.matchAll(what)];
    const last = found[found.length - 1];
    return last ? Number(last[1]) : null;
  };
  return {
    passed: total(/^\s*(?:\u2139|#)\s*pass\s+(\d+)\s*$/gm),
    failed: total(/^\s*(?:\u2139|#)\s*fail\s+(\d+)\s*$/gm),
    skipped: total(/^\s*(?:\u2139|#)\s*skipped\s+(\d+)\s*$/gm),
  };
}

export async function runSuite(args: string[]): Promise<number> {
  const ran = args.filter((a) => a !== "--no-build");
  const files = ran.length > 0 ? ran : [endToEnd];

  if (!args.includes("--no-build")) {
    // Before the run, always. The artifacts are built from two other repositories, and a suite
    // that tests yesterday's binary reports on code nobody is looking at (novox/hq 04-ISSUES/005).
    const built = rebuild(process.env, files);
    if (built.length > 0) console.log(`built: ${built.join(", ")}\n`);
  }
  // Read now, while it is true. The receipt names these, and reading them when the run ends
  // names whatever was committed during the twenty minutes in between instead.
  const against = whatWasTested(process.env);

  // **No canary.** There was one: a second scenario, one machine, raised first so a broken mesh
  // failed in two minutes rather than in forty. It walked exactly the path the first three tests
  // of the long run walk — a mesh comes up, a module lands, a consumer gets a credential — and
  // the long run reaches the end of that path in about 160 seconds.
  //
  // So it cost a whole scenario, every passing run, to save about 45 seconds on a failing one.
  // A scenario is machines that each boot a kernel, which is where the two minutes went.
  // `test/integration/canary.test.ts` is still there and still runs when it is named; it is no
  // longer raised on the way to everything else.

  const { code, seen, skips } = await runFiles(files);
  const counts = counted(seen);
  console.log("\n" + reportOn(counts, skips, (p, f, outcome) => record(p, f, files, process.env, against, outcome)));
  return exitOf(code, counts, skips);
}

/**
 * The suite's exit: the runner's, unless it says 0 for a run that did not pass (novox/hq issue 693). The
 * runner exits 0 when every test skipped; the lab takes this exit as the run's verdict, so a run that
 * proved nothing must not end in 0.
 */
export function exitOf(code: number, counts: { passed: number | null; failed: number | null }, skips: Skip[]): number {
  if (code !== 0) return code;
  if (counts.passed === null || counts.failed === null) return code;
  return judgeRun(counts, skips).passed ? 0 : 1;
}

/** The reporter that writes each skipped test and its reason (outcomes.ts), by its path. */
const skipsReporter = fileURLToPath(new URL("./outcomes.ts", import.meta.url));

/**
 * Run some test files, passing their output through as it arrives, and collect what was skipped and why:
 * the spec reporter to stdout as before, and the skips reporter beside it into a file of the run's own.
 */
async function runFiles(files: string[]): Promise<{ code: number; seen: string; skips: Skip[] }> {
  const scratch = mkdtempSync(join(tmpdir(), "mesh-lab-suite-"));
  const skipsFile = join(scratch, "skips.jsonl");
  const running = spawn(
    process.execPath,
    ["--test", "--test-concurrency=1", "--experimental-strip-types",
      "--test-reporter=spec", "--test-reporter-destination=stdout",
      `--test-reporter=${skipsReporter}`, `--test-reporter-destination=${skipsFile}`,
      ...files],
    { stdio: ["inherit", "pipe", "inherit"] },
  );

  let seen = "";
  running.stdout.on("data", (chunk: Buffer) => {
    // Passed through as it arrives: a suite that takes a quarter of an hour must not look hung.
    process.stdout.write(chunk);
    seen += chunk.toString();
  });

  const code: number = await new Promise((resolve) => {
    running.on("close", (c) => resolve(c ?? 1));
  });
  // Named as the files were given, relative to where the suite runs, rather than as the runner's absolute paths.
  const named = (file: string) => {
    const rel = isAbsolute(file) ? relative(process.cwd(), file) : file;
    return rel.startsWith("..") ? file : rel;
  };
  const skips = readSkips(skipsFile).map((s) => ({ ...s, file: named(s.file) }));
  rmSync(scratch, { recursive: true, force: true });
  return { code, seen, skips };
}

/**
 * reportOn decides whether this run says anything worth recording, and records it if so.
 *
 * Separated from the spawning so the decision can be tested: **no receipt rather than a guessed
 * one** is the rule that keeps the record meaning something, and it was written where nothing
 * could check it — which is 04-ISSUES/005 in miniature, inside the fix for it.
 */
export function reportOn(
  counts: { passed: number | null; failed: number | null; skipped?: number | null },
  skips: Skip[],
  write: (passed: number, failed: number, outcome: Outcome) => { against: Record<string, string>; ran: string[] },
): string {
  if (counts.passed === null || counts.failed === null) {
    // A run whose result could not be read is a run nobody can say anything about. Writing
    // "0 failed" because nothing said otherwise is how a green record comes to mean nothing.
    return "could not read what the runner reported; no receipt written";
  }
  // **A skip is said, with its reason, and a run that proved nothing says NOT PASSED** (novox/hq issue
  // 693): "0 passed, 0 failed" read as a clean run. The lab module reads these lines into its record.
  const skipped = counts.skipped ?? skips.length;
  const judged = judgeRun({ passed: counts.passed, failed: counts.failed }, skips);
  const outcome: Outcome = {
    skipped,
    skips: skips.map((s) => ({ ...s, reason: oneLine(s.reason) })),
    ...(judged.passed ? {} : { notPassed: judged.why }),
  };
  const receipt = write(counts.passed, counts.failed, outcome);
  const against = Object.entries(receipt.against).map(([n, c]) => `${n} ${c}`).join(", ");
  const lines = [`recorded: ${receipt.ran.join(", ")} — ${counts.passed} passed, ` +
    `${counts.failed} failed, ${skipped} skipped, against ${against || "nothing in git"}`];
  if (!judged.passed) lines.push(`NOT PASSED: ${judged.why}`);
  // As the runner's own spec line has it, "<test> (<file>) # <reason>": a test's name may hold any
  // punctuation, so the file in brackets is what the lab module reads the line apart by.
  for (const s of outcome.skips) lines.push(`skipped: ${s.test} (${s.file}) # ${s.reason || "no reason given"}`);
  return lines.join("\n");
}