mesh-catalog/modules/lab/tools/runs.ts

// A lab run: fresh checkouts of the named branches, side by side, then the lab's suite on the named
// beds (novox/hq ADR 0172).
//
// **A run is a detached script with its own process group**, so it outlives the tool call that started
// it and `stop` ends everything it started. It writes what it is doing to a status file beside its log,
// and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers
// for it, and says it was lost rather than pretending it is still going.

import { spawn } from "node:child_process";
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";

/** The repositories a lab run checks out, side by side, as the lab expects its siblings. */
export const REPOSITORIES = ["mesh-lab", "mesh-controller", "mesh-host", "mesh-catalog", "mesh-tools", "mesh-sdk"];

export interface RunStatus {
  id: string;
  state: "checking-out" | "building" | "running" | "passed" | "failed" | "stopped" | "lost";
  started: string;
  ended?: string;
  tests: string[];
  refs: Record<string, string>;
  commits?: Record<string, string>;
  exit?: number;
  pid?: number;
}

export function runDir(work: string, id: string): string {
  return join(work, id);
}

function statusPath(work: string, id: string): string {
  return join(runDir(work, id), "status.json");
}

export function readStatus(work: string, id: string): RunStatus | undefined {
  try {
    const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus;
    // A run whose process is gone while its status still says it is going was lost — the runtime or
    // the machine restarted under it. Said, rather than left reading as running for ever.
    if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) {
      s.state = "lost";
    }
    return s;
  } catch {
    return undefined;
  }
}

function alive(pid: number): boolean {
  try {
    process.kill(pid, 0);
    return true;
  } catch {
    return false;
  }
}

export function listRuns(work: string): RunStatus[] {
  if (!existsSync(work)) return [];
  return readdirSync(work)
    .filter((d) => d.startsWith("run-"))
    .map((id) => readStatus(work, id))
    .filter((s): s is RunStatus => !!s)
    .sort((a, b) => b.started.localeCompare(a.started));
}

/** The run still going, if any: one at a time, because two would contend for the same machine. */
export function running(work: string): RunStatus | undefined {
  return listRuns(work).find((s) => !["passed", "failed", "stopped", "lost"].includes(s.state));
}

const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`;

/**
 * The script one run executes. Every step writes its state first, so a run that dies says where.
 *
 * The environment is the one the lab's README describes for a run against sibling checkouts, pointed
 * at this run's own tree, so what is built and claimed is exactly what was checked out.
 */
export function script(work: string, id: string, forge: string, tests: string[], refs: Record<string, string>): string {
  const dir = runDir(work, id);
  const setState = (state: string) =>
    `node -e ${shellQuote(
      `const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=${JSON.stringify(state)};require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
    )}`;
  const clones = REPOSITORIES.map((repo) => {
    const ref = refs[repo] ?? "main";
    return [
      `git clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`,
      `echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`,
    ].join("\n");
  }).join("\n");
  const bin = join(dir, "bin");
  return `set -euo pipefail
cd ${shellQuote(dir)}
${setState("checking-out")}
${clones}
node -e ${shellQuote(
    `const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`,
  )}
${setState("building")}
for repo in mesh-sdk mesh-tools mesh-lab; do (cd ${shellQuote(dir)}/$repo && npm ci --no-audit --no-fund --loglevel=error); done
(cd ${shellQuote(dir)}/mesh-sdk && npm run build --if-present)
(cd ${shellQuote(dir)}/mesh-tools && npm run build --if-present)
mkdir -p ${shellQuote(bin)}
for p in postgres-provisioner objectstore-provisioner route-proxy; do
  (cd ${shellQuote(dir)}/mesh-controller && CGO_ENABLED=0 go build -o ${shellQuote(bin)}/$p ./examples/$p)
done
export MESH_LAB_HOST_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-host"))}
export MESH_LAB_BUNDLE=${shellQuote(join(dir, "mesh-host", "examples", "foundation-first-node-nats.lock"))}
export MESH_LAB_MODULES=${shellQuote(join(dir, "mesh-controller", "examples", "modules"))}
export MESH_LAB_BUILDER=${shellQuote(join(dir, "mesh-controller", "build", "mesh-builder"))}
export MESH_LAB_BOOTSTRAP_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-bootstrap"))}
export MESH_LAB_CATALOG=${shellQuote(join(dir, "mesh-catalog", "modules"))}
export MESH_LAB_PROVISIONER=${shellQuote(join(bin, "postgres-provisioner"))}
export MESH_LAB_OBJECTSTORE_PROVISIONER=${shellQuote(join(bin, "objectstore-provisioner"))}
export MESH_LAB_ROUTE_PROXY=${shellQuote(join(bin, "route-proxy"))}
${setState("running")}
cd ${shellQuote(join(dir, "mesh-lab"))}
node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" ")}
`;
}

/** start begins a run and returns at once with its status. */
export function start(work: string, forge: string, tests: string[], refs: Record<string, string>): RunStatus {
  const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`;
  const dir = runDir(work, id);
  mkdirSync(dir, { recursive: true });
  const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs };
  writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
  writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs), { mode: 0o700 });

  // The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a
  // run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172).
  const wrapper = `bash ${shellQuote(join(dir, "run.sh"))} > ${shellQuote(join(dir, "run.log"))} 2>&1; code=$?
node -e ${shellQuote(
    `const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
  )} "$code"
cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`;
  const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
  child.unref();
  status.pid = child.pid;
  writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
  return status;
}

/** stop ends a run and everything it started, by its process group. */
export function stop(work: string, id: string): RunStatus | undefined {
  const s = readStatus(work, id);
  if (!s || !s.pid) return s;
  if (["passed", "failed", "stopped", "lost"].includes(s.state)) return s;
  s.state = "stopped";
  writeFileSync(statusPath(work, id), JSON.stringify(s, null, 2));
  try {
    process.kill(-s.pid, "SIGTERM");
  } catch {
    // Already gone between the read and the kill.
  }
  return s;
}

/** tail is the last lines of a run's log. */
export function tail(work: string, id: string, lines: number): string {
  try {
    const all = readFileSync(join(runDir(work, id), "run.log"), "utf8").split("\n");
    return all.slice(-Math.max(1, lines)).join("\n");
  } catch {
    return "";
  }
}