CAT-LAB1: the lab clones through the forge account, and holds no registry secret
npm-package-registry is form none, so the registry pair the lab's clones of mesh-tools-go used is no longer minted. The lab requires forge-account as the build modules do; its clones go over ssh through a git and ssh configuration the mesh writes from that binding. The secret, contributes and MESH_LAB_FORGE_SECRET go. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
+15
-23
@@ -4,12 +4,10 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { rmSync } from "node:fs";
|
||||
import {
|
||||
checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start,
|
||||
stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun,
|
||||
checkDir, cleanCredentials, clearChecks, forgeArgs, forgeConfig, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, start,
|
||||
stop, tail, baseBuildLog, listBaseBuilds, startBaseBuild, launchRun,
|
||||
launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem,
|
||||
} from "./runs.js";
|
||||
|
||||
@@ -19,7 +17,6 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
|
||||
// ADR 0192): read from the env-file the mesh fills, at each call, so a changed setting is used
|
||||
// without restarting the runtime. MESH_LAB_FORGE itself still wins, for a hand-run instance.
|
||||
const forgeOf = (): string => (env.MESH_LAB_FORGE ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE")).replace(/\/+$/, "");
|
||||
// The forge credential the mesh binds and seals to the lab (novox/hq issue 624), read at each call.
|
||||
// The registry's public root the forge serves in the lab's binding (novox/hq ADR 0346), read at each call.
|
||||
const bindingOf = (): string | undefined =>
|
||||
env.MESH_LAB_FORGE_BINDING ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING") || undefined);
|
||||
@@ -28,9 +25,9 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
|
||||
const registry = await npmRegistry(forge, fetch, root.root);
|
||||
return registry.problem && root.problem ? { ...registry, problem: `${registry.problem}; ${root.problem}` } : registry;
|
||||
};
|
||||
const credentialOf = (forge: string): string | undefined => forgeCredential(forge,
|
||||
env.MESH_LAB_FORGE_BINDING ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING"),
|
||||
env.MESH_LAB_FORGE_SECRET ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_SECRET"));
|
||||
// The forge account's git configuration the mesh writes for the lab's clones (forgeConfig), read at each call.
|
||||
const forgeGitOf = (): string | undefined =>
|
||||
forgeConfig(env.MESH_LAB_FORGE_GIT ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_GIT") || undefined));
|
||||
return [
|
||||
{
|
||||
name: "lab_check",
|
||||
@@ -50,7 +47,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
|
||||
if (clone.status !== 0) return { ok: false, output: [String(clone.stderr).trim(), ...leftovers].join("\n") };
|
||||
spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" });
|
||||
check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" });
|
||||
problems = setupProblems(forge, `${dir}/setup`, credentialOf(forge));
|
||||
problems = setupProblems(forge, `${dir}/setup`, forgeGitOf());
|
||||
// And the address a run names the package registry by (npmRegistry): said when the registry does not say it.
|
||||
const registry = await registryOf(forge);
|
||||
if (registry.problem) problems.push(registry.problem);
|
||||
@@ -60,10 +57,8 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
|
||||
// **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every
|
||||
// repository it clones answers on the forge, and every one it installs has a lock file where the run
|
||||
// installs from.
|
||||
// A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in
|
||||
// use by a running run or rebuild is not.
|
||||
const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id);
|
||||
const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id)));
|
||||
// A credential line a run or rebuild of before the forge account left behind is removed, and said.
|
||||
const left = cleanCredentials(work);
|
||||
const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : "";
|
||||
// **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot
|
||||
// be read is a problem here, though lab_run refuses only the first.
|
||||
@@ -106,7 +101,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
|
||||
// **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698,
|
||||
// review 256): the registry asked first, then the check and the start with nothing awaited between.
|
||||
const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() },
|
||||
(registry) => start(work, forge, tests, refs, credentialOf(forge), registry));
|
||||
(registry) => start(work, forge, tests, refs, forgeGitOf(), registry));
|
||||
if (!launched.started) return launched;
|
||||
return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry,
|
||||
...(launched.problem ? { problem: launched.problem } : {}) };
|
||||
@@ -137,7 +132,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
|
||||
// Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with
|
||||
// nothing awaited between (review 256).
|
||||
const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before },
|
||||
(registry) => startBaseBuild(work, forge, credentialOf(forge), registry));
|
||||
(registry) => startBaseBuild(work, forge, forgeGitOf(), registry));
|
||||
if (launched.started === false) return { ...reading, started: false, reason: launched.reason };
|
||||
return { ...reading, started: true, build: launched.job };
|
||||
},
|
||||
@@ -174,14 +169,12 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
|
||||
* answer for, and a checkout it installs with nothing installable (installPlan). Empty when a run's setup
|
||||
* would go through.
|
||||
*/
|
||||
export function setupProblems(forge: string, dir: string, credential?: string): string[] {
|
||||
export function setupProblems(forge: string, dir: string, forgeGit?: string): string[] {
|
||||
const problems: string[] = [];
|
||||
spawnSync("rm", ["-rf", dir]);
|
||||
// The credential a run would offer, in a file of its own outside the check's clones, removed after.
|
||||
const keep = mkdtempSync(join(tmpdir(), "lab-check-"));
|
||||
const file = writeCredential(join(keep, "git-credentials"), credential);
|
||||
// The forge account's configuration a run's clones go through (forgeConfig).
|
||||
const git = (args: string[]) =>
|
||||
spawnSync("sh", ["-c", `git ${credentialArgs(file)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], {
|
||||
spawnSync("sh", ["-c", `git ${forgeArgs(forgeGit)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], {
|
||||
encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
|
||||
});
|
||||
try {
|
||||
@@ -192,7 +185,7 @@ export function setupProblems(forge: string, dir: string, credential?: string):
|
||||
const said = remote.stderr.trim().split("\n")[0] ?? "";
|
||||
problems.push(`a run checks out ${repo}, and ${url} could not be read` +
|
||||
(said ? `: ${said}` : ": no main branch") +
|
||||
(credential ? "" : " — the lab holds no forge credential (its npm-package-registry binding and secret)"));
|
||||
(forgeGit ? "" : " — the lab holds no forge account yet (its forge-account binding, written as forge.gitconfig in its state)"));
|
||||
continue;
|
||||
}
|
||||
if (!NPM_REPOSITORIES.includes(repo)) continue;
|
||||
@@ -209,7 +202,6 @@ export function setupProblems(forge: string, dir: string, credential?: string):
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
rmSync(keep, { recursive: true, force: true });
|
||||
spawnSync("rm", ["-rf", dir]);
|
||||
}
|
||||
return problems;
|
||||
|
||||
+24
-69
@@ -163,68 +163,31 @@ export async function npmRegistry(forge: string, get: typeof fetch = fetch, publ
|
||||
}
|
||||
|
||||
/**
|
||||
* The forge credential a run's clones may present, as a git credential-store line —
|
||||
* scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none.
|
||||
* The forge account's git configuration a run's clones go through, or undefined when the mesh has not written it.
|
||||
*
|
||||
* **The mesh's credential for the forge, the way the build modules hold it** (novox/hq issue 624): the lab
|
||||
* requires the package registry, the forge's provisioner binds it an account (`as`) and seals that
|
||||
* account's password, and the forge that answers npm is the forge that hosts the repositories. Private
|
||||
* repositories (mesh-tools-go) are cloned with it; public ones clone as before. Written to a file git reads
|
||||
* through its credential store, never into a URL, an argument, the run's script or its log, and offered
|
||||
* only to the forge's own scheme, host and port.
|
||||
* **The mesh's account on the forge, the way the build modules hold it** (novox/hq issue 624, issue 1966): the lab
|
||||
* requires `forge-account`, the forge's provider registers the account's key (`~/.ssh/mesh_ed25519`) for it, and
|
||||
* the mesh writes into the lab's state a git configuration that sends every clone of the forge's address over ssh
|
||||
* to the forge, with an ssh configuration that offers that key and takes only the forge's host key. Private
|
||||
* repositories (mesh-tools-go) clone with it; no password is bound, sealed or written for a run.
|
||||
*/
|
||||
export function forgeCredential(forge: string, bindingFile: string, secretFile: string): string | undefined {
|
||||
if (!forge || !bindingFile || !secretFile) return undefined;
|
||||
let as = "";
|
||||
let secret = "";
|
||||
export function forgeConfig(file: string | undefined): string | undefined {
|
||||
if (!file) return undefined;
|
||||
try {
|
||||
as = String((JSON.parse(readFileSync(bindingFile, "utf8")) as { as?: string }).as ?? "");
|
||||
secret = readFileSync(secretFile, "utf8").trim();
|
||||
return /^\s*insteadOf\s*=\s*\S/m.test(readFileSync(file, "utf8")) ? file : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
if (!as || !secret) return undefined;
|
||||
let origin: URL;
|
||||
try {
|
||||
origin = new URL(forge);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
const line = new URL(`${origin.protocol}//${origin.host}`);
|
||||
line.username = as;
|
||||
line.password = secret;
|
||||
return line.toString().replace(/\/$/, "");
|
||||
}
|
||||
|
||||
/** Where a run keeps its forge credential: beside its directory, never in it, and removed when it ends. */
|
||||
/** Where a run of before kept its forge credential line, beside its directory: removed wherever one is left. */
|
||||
export function credentialPath(work: string, id: string): string {
|
||||
return join(work, `.${id}.git-credentials`);
|
||||
}
|
||||
|
||||
/** writeCredential puts a credential-store line where git reads it, readable by this account alone. */
|
||||
export function writeCredential(path: string, line: string | undefined): string | undefined {
|
||||
if (!line) return undefined;
|
||||
writeFileSync(path, `${line}\n`, { mode: 0o600 });
|
||||
return path;
|
||||
}
|
||||
|
||||
/**
|
||||
* credentialGuard is the shell that removes the credential however the script ends — its end, an error,
|
||||
* or a stop's SIGTERM to the run's process group, which kills the wrapper before its own cleanup runs
|
||||
* (review 196 on mesh-catalog #272). The script also removes it as soon as its clones are done.
|
||||
*/
|
||||
export function credentialGuard(path: string | undefined): string {
|
||||
if (!path) return "";
|
||||
const rm = `rm -f ${shellQuote(path)}`;
|
||||
return `trap ${shellQuote(rm)} EXIT
|
||||
trap ${shellQuote(`${rm}; exit 143`)} TERM
|
||||
trap ${shellQuote(`${rm}; exit 130`)} INT
|
||||
trap ${shellQuote(`${rm}; exit 129`)} HUP`;
|
||||
}
|
||||
|
||||
/**
|
||||
* cleanCredentials removes every run's and every base rebuild's credential file left in the work directory
|
||||
* but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it
|
||||
* but the ones named — what runs of before the forge account, a crash or a reboot left behind. Answers what it
|
||||
* removed.
|
||||
*/
|
||||
export function cleanCredentials(work: string, keep?: string | string[]): string[] {
|
||||
@@ -272,10 +235,10 @@ export function clearChecks(work: string): string[] {
|
||||
return said;
|
||||
}
|
||||
|
||||
/** The arguments that have git offer the credential in that file, and only that one. */
|
||||
export function credentialArgs(path: string | undefined): string {
|
||||
/** The arguments that have git read the forge account's configuration (forgeConfig) for one command. */
|
||||
export function forgeArgs(path: string | undefined): string {
|
||||
if (!path) return "";
|
||||
return `-c credential.helper= -c ${shellQuote(`credential.helper=store --file=${path}`)} `;
|
||||
return `-c include.path=${shellQuote(path)} `;
|
||||
}
|
||||
|
||||
export interface RunStatus {
|
||||
@@ -462,7 +425,7 @@ const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`;
|
||||
* at this run's own tree, so what is built and claimed is exactly what was checked out.
|
||||
*/
|
||||
export function script(work: string, id: string, forge: string, tests: string[], refs: Record<string, string>,
|
||||
credential?: string, registry = `${forge}${NPM_PATH}`): string {
|
||||
forgeGit?: string, registry = `${forge}${NPM_PATH}`): string {
|
||||
const dir = runDir(work, id);
|
||||
const setState = (state: string) =>
|
||||
`node -e ${shellQuote(
|
||||
@@ -471,7 +434,7 @@ export function script(work: string, id: string, forge: string, tests: string[],
|
||||
const clones = REPOSITORIES.map((repo) => {
|
||||
const ref = refs[repo] ?? "main";
|
||||
return [
|
||||
`git ${credentialArgs(credential)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`,
|
||||
`git ${forgeArgs(forgeGit)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`,
|
||||
`echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`,
|
||||
].join("\n");
|
||||
}).join("\n");
|
||||
@@ -479,11 +442,9 @@ export function script(work: string, id: string, forge: string, tests: string[],
|
||||
return `set -euo pipefail
|
||||
# A repository the forge will not show fails its clone at once, rather than waiting on a prompt.
|
||||
export GIT_TERMINAL_PROMPT=0
|
||||
${credentialGuard(credential)}
|
||||
cd ${shellQuote(dir)}
|
||||
${setState("checking-out")}
|
||||
${clones}
|
||||
${credential ? `# The forge credential is for the clones, and nothing after them runs with it on disk.\nrm -f ${shellQuote(credential)}` : ""}
|
||||
node -e ${shellQuote(
|
||||
`const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`,
|
||||
)}
|
||||
@@ -516,16 +477,15 @@ node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join("
|
||||
|
||||
/** start begins a run and returns at once with its status. */
|
||||
export function start(work: string, forge: string, tests: string[], refs: Record<string, string>,
|
||||
credential?: string, registry?: string): RunStatus {
|
||||
forgeGit?: string, registry?: string): RunStatus {
|
||||
const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
||||
const dir = runDir(work, id);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs };
|
||||
writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
|
||||
// What a crashed or rebooted run left behind goes before this run writes its own.
|
||||
// A credential line a run of before left behind goes before this run starts.
|
||||
cleanCredentials(work);
|
||||
const credentials = writeCredential(credentialPath(work, id), credential);
|
||||
writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials, registry), { mode: 0o700 });
|
||||
writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, forgeGit, registry), { mode: 0o700 });
|
||||
|
||||
// The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a
|
||||
// run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172).
|
||||
@@ -533,7 +493,6 @@ export function start(work: string, forge: string, tests: string[], refs: Record
|
||||
node -e ${shellQuote(
|
||||
`const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
|
||||
)} "$code"
|
||||
rm -f ${shellQuote(credentialPath(work, id))}
|
||||
cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`;
|
||||
const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
|
||||
child.unref();
|
||||
@@ -715,14 +674,12 @@ export function baseBuildLog(work: string, id: string, lines = 40): string {
|
||||
* baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it,
|
||||
* then its `base build`, which replaces the image and reads back that it works before publishing it.
|
||||
*/
|
||||
export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string {
|
||||
export function baseScript(dir: string, forge: string, forgeGit?: string, registry = `${forge}${NPM_PATH}`): string {
|
||||
const lab = join(dir, "mesh-lab");
|
||||
return `set -euo pipefail
|
||||
export GIT_TERMINAL_PROMPT=0
|
||||
${credentialGuard(credential)}
|
||||
cd ${shellQuote(dir)}
|
||||
git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)}
|
||||
${credential ? `rm -f ${shellQuote(credential)}` : ""}
|
||||
git ${forgeArgs(forgeGit)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)}
|
||||
echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)"
|
||||
printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))}
|
||||
export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))}
|
||||
@@ -733,21 +690,19 @@ node --experimental-strip-types src/cli.ts base build
|
||||
}
|
||||
|
||||
/** startBaseBuild rebuilds the base image in the background and answers at once with the build. */
|
||||
export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild {
|
||||
export function startBaseBuild(work: string, forge: string, forgeGit?: string, registry?: string): BaseBuild {
|
||||
const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
||||
const dir = join(work, id);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const build: BaseBuild = { id, state: "building", started: new Date().toISOString() };
|
||||
writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
|
||||
// What a crashed or rebooted run or rebuild left behind goes before this one writes its own.
|
||||
// A credential line a run or rebuild of before left behind goes before this one starts.
|
||||
cleanCredentials(work);
|
||||
const credentials = writeCredential(credentialPath(work, id), credential);
|
||||
writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 });
|
||||
writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, forgeGit, registry), { mode: 0o700 });
|
||||
const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$?
|
||||
node -e ${shellQuote(
|
||||
`const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
|
||||
)} "$code"
|
||||
rm -f ${shellQuote(credentialPath(work, id))}
|
||||
rm -rf ${shellQuote(join(dir, "mesh-lab"))}`;
|
||||
const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
|
||||
child.unref();
|
||||
|
||||
Reference in New Issue
Block a user