mesh-lab/src/lastrun.ts

/**
 * When the end-to-end suite last ran, and against what.
 *
 * **The fault this exists for is not that the suite breaks — it is that nobody notices it stopped
 * running** (novox/hq 04-ISSUES/005). The harness it replaces had not built for two and a half
 * months, and nothing said so; the coverage was assumed rather than checked, and several of the
 * pipeline's most expensive defects landed inside that window.
 *
 * This suite is in a better position and the same danger: it needs a machine with a hypervisor, so
 * it cannot run on every push, which means it runs when somebody remembers. Remembering is not a
 * mechanism.
 *
 * So a run leaves a receipt, and something can be asked whether the receipt still means anything.
 * A receipt that is old, or taken against code the repositories have since moved past, is the
 * thing 005 says nobody was ever told.
 *
 * **Kept outside the repository**, because the question is *has this machine run it* rather than
 * *what is committed* — and a receipt in git would be a claim about everyone's machine made by
 * whoever committed last.
 */

import { execFileSync } from "node:child_process";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, join } from "node:path";

import { repositories } from "./repos.ts";

/** What a run was taken against, per repository. */
export type Against = Record<string, string>;

export interface Receipt {
  /** When it finished, ISO 8601. */
  at: string;
  passed: number;
  failed: number;
  /** The commit each repository was at. Absent for anything that was not a git checkout. */
  against: Against;
  /** The test files this run was pointed at. See {@link endToEnd}. */
  ran: string[];
}

/**
 * endToEnd is the file that raises real machines. A run that did not include it proved nothing
 * about the pipeline, however green it was.
 *
 * The suite takes paths, so it can be pointed at one quick file — and the receipt from that would
 * otherwise be indistinguishable from a receipt for the real thing. That is 04-ISSUES/005 again:
 * not a suite that fails, a record that says more than the run behind it.
 */
export const endToEnd = "test/integration/mesh.test.ts";

/** Where the receipt lives: XDG state, which is for exactly this — data a tool keeps between runs. */
export function receiptPath(): string {
  const state = process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state");
  return join(state, "mesh-lab", "last-run.json");
}

/**
 * headOf is the commit a directory's repository is at, or "" if it is not one.
 *
 * **A tree with uncommitted changes is marked, and never equal to the clean commit it sits on.**
 * The run tested what was on disk, and that is not what the commit contains — so a receipt naming
 * the bare hash would claim coverage of code nobody can check out. Nothing else could tell: the
 * hash is identical either way. That is 04-ISSUES/005's overclaim in its quietest form.
 */
export function headOf(directory: string): string {
  const git = (args: string[]) =>
    execFileSync("git", ["-C", directory, ...args], {
      encoding: "utf8",
      stdio: ["ignore", "pipe", "ignore"],
    });
  try {
    const head = git(["rev-parse", "--short", "HEAD"]).trim();
    const dirty = git(["status", "--porcelain"]).trim() !== "";
    return dirty ? `${head}+uncommitted` : head;
  } catch {
    // Not a checkout, or no git. Absent rather than guessed: a receipt claiming a commit it did
    // not read is worse than one that says it could not tell.
    return "";
  }
}

/**
 * whatWasTested is the repositories this run exercised, by the paths it was given.
 *
 * From the environment rather than a fixed list, because the paths are how the suite is told what
 * to run — so anything it was pointed at is something the receipt should account for, and anything
 * it was not pointed at was not tested.
 */
export function whatWasTested(env: NodeJS.ProcessEnv = process.env): Against {
  const against: Against = {};
  for (const [name, directory] of Object.entries(repositories(env))) {
    const head = headOf(directory);
    if (head) against[name] = head;
  }
  return against;
}

/** record writes the receipt. Failures are recorded too: a run that failed still ran. */
export function record(
  passed: number,
  failed: number,
  ran: string[],
  env = process.env,
): Receipt {
  const receipt: Receipt = {
    at: new Date().toISOString(),
    passed,
    failed,
    against: whatWasTested(env),
    ran,
  };
  const path = receiptPath();
  mkdirSync(dirname(path), { recursive: true });
  writeFileSync(path, JSON.stringify(receipt, null, 2) + "\n");
  return receipt;
}

/** read returns the receipt, or null when this machine has never run the suite. */
export function read(): Receipt | null {
  try {
    return JSON.parse(readFileSync(receiptPath(), "utf8")) as Receipt;
  } catch {
    return null;
  }
}

export interface Verdict {
  /** True when the receipt still says something about the code as it stands. */
  current: boolean;
  lines: string[];
}

/**
 * judge says whether the last run still means anything.
 *
 * Two ways it can stop meaning something, and they read differently: it was long ago, or the code
 * has moved since. The second is the one that matters — a suite that passed against code nobody
 * runs any more is coverage in name.
 */
export function judge(
  receipt: Receipt | null,
  now: Date,
  against: Against,
  staleAfterDays = 7,
): Verdict {
  if (!receipt) {
    return {
      current: false,
      lines: [
        "this machine has never run the end-to-end suite.",
        "  Nothing here has been checked end to end, which is not the same as nothing being wrong.",
      ],
    };
  }

  // A receipt written before this field existed says nothing about what it ran, and the honest
  // reading of "nothing said" is not "everything".
  const endToEndRan = (receipt.ran ?? []).some((path) => path.endsWith(endToEnd));

  const days = (now.getTime() - Date.parse(receipt.at)) / 86_400_000;
  const lines: string[] = [];
  const outcome = receipt.failed > 0
    ? `last ran ${ago(days)} and ${receipt.failed} test(s) failed`
    : `last passed ${ago(days)}, ${receipt.passed} test(s)`;
  lines.push(`the end-to-end suite ${outcome}`);

  let moved = false;
  for (const name of Object.keys(against).sort()) {
    const then = receipt.against[name];
    const now = against[name];
    if (!then) {
      lines.push(`  ${name.padEnd(14)} was not accounted for in that run`);
      moved = true;
      continue;
    }
    if (then === now) {
      lines.push(`  ${name.padEnd(14)} at ${then} (unchanged)`);
      continue;
    }
    lines.push(`  ${name.padEnd(14)} at ${then}, now at ${now}`);
    moved = true;
  }

  if (!endToEndRan) {
    lines.push(`  That run did not include ${endToEnd}, so it raised no machines.`);
  }

  if (receipt.failed > 0) {
    lines.push("  Nothing has been proven end to end since.");
  } else if (moved) {
    lines.push("  What it proved was proven about code that has since changed.");
  } else if (days > staleAfterDays) {
    lines.push(`  Nothing has changed since, but that was more than ${staleAfterDays} days ago.`);
  }

  return {
    current: endToEndRan && receipt.failed === 0 && !moved && days <= staleAfterDays,
    lines,
  };
}

function ago(days: number): string {
  if (days < 1 / 24) return "less than an hour ago";
  if (days < 1) return `${Math.round(days * 24)} hour(s) ago`;
  return `${Math.round(days)} day(s) ago`;
}