lab: a read-only forge account of its own, and no credential left on disk after a stop (hq issue 624, review 196)

The lab's credential came from the package team, which also writes the @novox packages, and the
branches a run clones are not reviewed yet. The lab now contributes {"access": "read"} to its
npm-package-registry requirement, and gitea's provisioner puts such a consumer on a team of its own,
readers: code read on the org's repositories, package read, no write, and off the package team. The
build agent, which publishes, stays on packages.

A stop's SIGTERM to the run's process group killed the wrapper before it removed the credential file.
The run's script now removes it as soon as its clones are done and traps every way it ends, and a run's
start and lab_check remove what a crash or a reboot left.

modules/lab/tools/index.ts

@@ -8,7 +8,7 @@ import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
-  credentialArgs, forgeCredential, installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start,
+  cleanCredentials, credentialArgs, forgeCredential, installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start,
   stop, tail, writeCredential,
 } from "./runs.js";
 
@@ -39,8 +39,13 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
         // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every
         // repository it clones answers on the forge, and every one it installs has a lock file where the run
         // installs from.
+        // A credential a stopped, crashed or rebooted run left behind is removed, and said; one still in use
+        // by a running run is not.
+        const busy = running(work);
+        const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined);
         const problems = setupProblems(forge, `${work}/check-setup`, credentialOf(forge));
-        const output = [`${check.stdout}${check.stderr}`.trim(), ...problems].filter(Boolean).join("\n");
+        const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : "";
+        const output = [`${check.stdout}${check.stderr}`.trim(), removed, ...problems].filter(Boolean).join("\n");
         return { ok: check.status === 0 && problems.length === 0, output };
       },
     },

modules/lab/tools/runs.ts

@@ -7,7 +7,7 @@
 // for it, and says it was lost rather than pretending it is still going.
 
 import { spawn } from "node:child_process";
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
 /**
@@ -106,6 +106,37 @@ export function writeCredential(path: string, line: string | undefined): string
   return path;
 }
 
+/**
+ * credentialGuard is the shell that removes the credential however the script ends — its end, an error,
+ * or a stop's SIGTERM to the run's process group, which kills the wrapper before its own cleanup runs
+ * (review 196 on mesh-catalog #272). The script also removes it as soon as its clones are done.
+ */
+export function credentialGuard(path: string | undefined): string {
+  if (!path) return "";
+  const rm = `rm -f ${shellQuote(path)}`;
+  return `trap ${shellQuote(rm)} EXIT
+trap ${shellQuote(`${rm}; exit 143`)} TERM
+trap ${shellQuote(`${rm}; exit 130`)} INT
+trap ${shellQuote(`${rm}; exit 129`)} HUP`;
+}
+
+/**
+ * cleanCredentials removes every run's credential file left in the work directory but the one named — what
+ * a crash or a reboot left behind, which no trap could remove. Answers what it removed.
+ */
+export function cleanCredentials(work: string, keep?: string): string[] {
+  if (!existsSync(work)) return [];
+  const removed: string[] = [];
+  for (const name of readdirSync(work)) {
+    if (!/^\.run-.*\.git-credentials$/.test(name)) continue;
+    const path = join(work, name);
+    if (path === keep) continue;
+    rmSync(path, { force: true });
+    removed.push(path);
+  }
+  return removed;
+}
+
 /** The arguments that have git offer the credential in that file, and only that one. */
 export function credentialArgs(path: string | undefined): string {
   if (!path) return "";
@@ -195,9 +226,11 @@ export function script(work: string, id: string, forge: string, tests: string[],
   return `set -euo pipefail
 # A repository the forge will not show fails its clone at once, rather than waiting on a prompt.
 export GIT_TERMINAL_PROMPT=0
+${credentialGuard(credential)}
 cd ${shellQuote(dir)}
 ${setState("checking-out")}
 ${clones}
+${credential ? `# The forge credential is for the clones, and nothing after them runs with it on disk.\nrm -f ${shellQuote(credential)}` : ""}
 node -e ${shellQuote(
     `const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`,
   )}
@@ -236,6 +269,8 @@ export function start(work: string, forge: string, tests: string[], refs: Record
   mkdirSync(dir, { recursive: true });
   const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs };
   writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
+  // What a crashed or rebooted run left behind goes before this run writes its own.
+  cleanCredentials(work);
   const credentials = writeCredential(credentialPath(work, id), credential);
   writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials), { mode: 0o700 });