mesh-lab/test/outcomes.test.ts

// A run in which no test passed has proven nothing, and says so (novox/hq issue 693).
//
// Seen on the lab: the genesis bed stopped before its subject, skipped its only test as "NOT REACHED",
// and the suite printed "0 passed, 0 failed" and exited 0, so the lab recorded the run as passed.

import { test } from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { judge, type Receipt, endToEnd } from "../src/lastrun.ts";
import { judgeRun, oneLine, type Skip } from "../src/outcomes.ts";
import { counted, exitOf, reportOn } from "../src/suite.ts";

// The suite as a person runs it: not as a child of this test runner, which would have it report to us.
const suiteEnv = (state: string): NodeJS.ProcessEnv => {
  const env: NodeJS.ProcessEnv = { ...process.env, XDG_STATE_HOME: state, NO_COLOR: "1" };
  delete env["NODE_TEST_CONTEXT"];
  return env;
};

const lab = fileURLToPath(new URL("..", import.meta.url));
const unreached: Skip = {
  file: "test/integration/genesis-keeps-its-bus.test.ts",
  test: "genesis adopts its bus",
  reason: "ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:\n  docker exited 1",
};
const noBed: Skip = { file: "test/integration/mesh.test.ts", test: "a builder builds", reason: "set MESH_LAB_BUILDER to a built mesh-builder" };

test("a run whose only test skipped did not pass", () => {
  const said = judgeRun({ passed: 0, failed: 0 }, [unreached]);
  assert.equal(said.passed, false);
  assert.match(said.why, /no test passed: 1 skipped, 1 of them as NOT REACHED/);
});

test("a run that ran no test at all did not pass", () => {
  assert.deepEqual(judgeRun({ passed: 0, failed: 0 }, []), { passed: false, why: "no test passed: none ran" });
});

test("a NOT REACHED skip beside tests that passed is still not passed", () => {
  const said = judgeRun({ passed: 3, failed: 0 }, [unreached]);
  assert.equal(said.passed, false);
  assert.match(said.why, /1 test\(s\) skipped as NOT REACHED/);
});

// The design limit: a skip its file declares expected, beside tests that passed, leaves the run passed.
test("an expected skip beside tests that passed leaves the run passed", () => {
  assert.deepEqual(judgeRun({ passed: 21, failed: 0 }, [noBed]), { passed: true, why: "" });
});

test("the suite's exit is not 0 for a run that did not pass, and is the runner's otherwise", () => {
  assert.equal(exitOf(0, { passed: 0, failed: 0 }, [unreached]), 1);
  assert.equal(exitOf(0, { passed: 21, failed: 0 }, [noBed]), 0);
  assert.equal(exitOf(1, { passed: 20, failed: 1 }, []), 1);
  assert.equal(exitOf(0, { passed: null, failed: null }, []), 0, "unread totals are reportOn's case, not this one");
});

test("the recorded line says what skipped and why, and NOT PASSED", () => {
  let kept: unknown;
  const said = reportOn({ passed: 0, failed: 0, skipped: 1 }, [unreached], (_p, _f, outcome) => {
    kept = outcome;
    return { against: { "mesh-lab": "7ed7da1" }, ran: [unreached.file] };
  });
  assert.match(said, /— 0 passed, 0 failed, 1 skipped, against mesh-lab 7ed7da1/);
  assert.match(said, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m);
  assert.match(said, /^skipped: genesis adopts its bus \(test\/integration\/genesis-keeps-its-bus\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step: docker exited 1$/m);
  assert.deepEqual(kept, {
    skipped: 1,
    skips: [{ ...unreached, reason: oneLine(unreached.reason) }],
    notPassed: "no test passed: 1 skipped, 1 of them as NOT REACHED",
  });
});

// A test that prints a summary-shaped line before it skips must not stand for the runner's summary.
test("the runner's last summary is read, not a line a test printed before it", () => {
  const output = "\u2139 pass 5\n\u2139 fail 0\n\u2139 skipped 0\n\ufe63 genesis adopts its bus (1ms) # NOT REACHED\n" +
    "\u2139 tests 1\n\u2139 pass 0\n\u2139 fail 0\n\u2139 skipped 1\n";
  const counts = counted(output);
  assert.deepEqual(counts, { passed: 0, failed: 0, skipped: 1 });
  assert.equal(exitOf(0, counts, [unreached]), 1);
});

test("NOT REACHED is matched in any case", () => {
  const said = judgeRun({ passed: 2, failed: 0 }, [{ ...noBed, reason: "the broker step was not reached" }]);
  assert.equal(said.passed, false);
});

test("the runner's skipped total is read", () => {
  assert.deepEqual(counted("ℹ tests 1\nℹ pass 0\nℹ fail 0\nℹ skipped 1\n"), { passed: 0, failed: 0, skipped: 1 });
});

test("a receipt of a run that proved nothing is not current, and says why", () => {
  const receipt: Receipt = {
    at: "2026-10-14T00:33:11Z", passed: 0, failed: 0, against: { "mesh-lab": "aaa" }, ran: [endToEnd],
    skipped: 1, skips: [unreached], notPassed: "no test passed: 1 skipped",
  };
  const said = judge(receipt, new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" });
  assert.equal(said.current, false, said.lines.join("\n"));
  const text = said.lines.join("\n");
  assert.match(text, /did not pass: no test passed: 1 skipped/);
  assert.match(text, /skipped: genesis adopts its bus — ADR 0333 §5 NOT REACHED/);
  // An older receipt, from before the fields, with nothing passed, is judged by its count.
  const older = judge({ at: receipt.at, passed: 0, failed: 0, against: receipt.against, ran: [endToEnd] },
    new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" });
  assert.equal(older.current, false);
});

// The real runner, end to end: the suite command against a file whose only test skips as NOT REACHED,
// the way the genesis bed does.
test("the suite exits non-zero for a file whose only test skipped as NOT REACHED, and records why", () => {
  const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
  try {
    const file = join(dir, "unreached.test.ts");
    writeFileSync(file, [
      `import { test } from "node:test";`,
      `test("genesis adopts its bus", (t) => {`,
      `  console.log("a line the bed printed");`,
      `  t.skip("ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base:\\nNotFound: content digest");`,
      `});`,
    ].join("\n"));
    const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
      cwd: lab, encoding: "utf8", env: suiteEnv(dir),
    });
    assert.equal(ran.status, 1, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
    assert.match(ran.stdout, /0 passed, 0 failed, 1 skipped/);
    assert.match(ran.stdout, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m);
    assert.match(ran.stdout, /^skipped: genesis adopts its bus \(.*unreached\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest$/m);
    const receipt = JSON.parse(readFileSync(join(dir, "mesh-lab", "last-run.json"), "utf8")) as Receipt;
    assert.equal(receipt.skipped, 1);
    assert.equal(receipt.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
    assert.match(receipt.skips?.[0]?.reason ?? "", /NOT REACHED/);
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
});

test("the suite does not take a summary a test printed for the runner's", () => {
  const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
  try {
    const file = join(dir, "forged.test.ts");
    writeFileSync(file, [
      `import { test } from "node:test";`,
      `test("genesis adopts its bus", (t) => {`,
      `  console.log("\\u2139 pass 5\\n\\u2139 fail 0");`,
      `  t.skip("ADR 0333 §5 NOT REACHED: stopped at step 16");`,
      `});`,
    ].join("\n"));
    const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
      cwd: lab, encoding: "utf8", env: suiteEnv(dir),
    });
    assert.equal(ran.status, 1, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
    assert.match(ran.stdout, /0 passed, 0 failed, 1 skipped/);
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
});

test("the suite still exits 0 for a file that passes a test and skips another for a declared reason", () => {
  const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
  try {
    const file = join(dir, "declared.test.ts");
    writeFileSync(file, [
      `import { test } from "node:test";`,
      `test("a module lands", () => {});`,
      `test("a builder builds", { skip: "set MESH_LAB_BUILDER to a built mesh-builder" }, () => {});`,
    ].join("\n"));
    const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
      cwd: lab, encoding: "utf8", env: suiteEnv(dir),
    });
    assert.equal(ran.status, 0, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
    assert.match(ran.stdout, /1 passed, 0 failed, 1 skipped/);
    assert.doesNotMatch(ran.stdout, /NOT PASSED/);
    assert.match(ran.stdout, /^skipped: a builder builds \(.*declared\.test\.ts\) # set MESH_LAB_BUILDER to a built mesh-builder$/m);
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
});