diff --git a/modules/lab/module.json b/modules/lab/module.json index fe523f30..dd9e8921 100644 --- a/modules/lab/module.json +++ b/modules/lab/module.json @@ -5,6 +5,15 @@ "container-runtime", "virtualisation" ], + "requires": [ + "npm-package-registry" + ], + "binds": { + "npm-package-registry": "${dir:state}/package-registry.json" + }, + "secrets": { + "npm-package-registry": "${dir:state}/package-registry.secret" + }, "data": { "own": [ { @@ -33,7 +42,7 @@ "type": "file", "path": "${dir:state}/lab.env", "mode": "0600", - "content": "MESH_LAB_FORGE=${setting:forge}\n" + "content": "MESH_LAB_FORGE=${setting:forge}\nMESH_LAB_FORGE_BINDING=${dir:state}/package-registry.json\nMESH_LAB_FORGE_SECRET=${dir:state}/package-registry.secret\n" }, { "id": "git", diff --git a/modules/lab/test/runs.test.ts b/modules/lab/test/runs.test.ts index 2389b122..aa230ec5 100644 --- a/modules/lab/test/runs.test.ts +++ b/modules/lab/test/runs.test.ts @@ -98,3 +98,54 @@ test("a run checks out mesh-tools-go and says where its checkouts are", () => { assert.doesNotMatch(s, /cd '\/work\/run-x'\/\$repo && npm ci/); for (const repo of NPM_REPOSITORIES) assert.ok(s.includes(`'/work/run-x/${repo}'/package-lock.json`), repo); }); + +// Defends novox/hq issue 624, its second half: mesh-tools-go is private, and the lab cloned over HTTP with +// no credential ("could not read Username"). A run clones with the forge credential the mesh binds and +// seals to the lab, through git's credential store, and the secret is in no script, URL or argument. +import { credentialArgs, credentialPath, forgeCredential } from "../tools/runs.ts"; + +function aBinding(as: string, secret: string): { binding: string; secret: string } { + const dir = mkdtempSync(join(tmpdir(), "lab-binding-")); + writeFileSync(join(dir, "package-registry.json"), JSON.stringify({ provision: "npm-package-registry", at: "10.42.0.1", as, + serves: { scheme: "http", port: 3000 } })); + writeFileSync(join(dir, "package-registry.secret"), `${secret}\n`); + return { binding: join(dir, "package-registry.json"), secret: join(dir, "package-registry.secret") }; +} + +test("the forge credential is the bound account and its sealed secret, for the forge's own origin", () => { + const b = aBinding("lab-shanks", "s3cret-of-forty-characters"); + assert.equal(forgeCredential("http://novox.internal:20000", b.binding, b.secret), + "http://lab-shanks:s3cret-of-forty-characters@novox.internal:20000"); + assert.equal(forgeCredential("http://novox.internal:20000/", b.binding, ""), undefined); + assert.equal(forgeCredential("", b.binding, b.secret), undefined); + assert.equal(forgeCredential("http://novox.internal:20000", "/nowhere", b.secret), undefined); +}); + +test("a run clones through git's credential store, and its script carries no secret", () => { + const b = aBinding("lab-shanks", "s3cret-of-forty-characters"); + const credential = forgeCredential("http://novox.internal:20000", b.binding, b.secret); + const path = credentialPath("/work", "run-x"); + const s = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, path); + assert.ok(credential); + assert.doesNotMatch(s, /s3cret/); + assert.match(s, /git -c credential\.helper= -c 'credential\.helper=store --file=\/work\/\.run-x\.git-credentials' clone --quiet --depth 50 --branch 'main' 'http:\/\/novox\.internal:20000\/novox\/mesh-tools-go\.git'/); + assert.match(s, /export GIT_TERMINAL_PROMPT=0/); + assert.equal(path.startsWith("/work/run-x/"), false, "the credential is kept out of the run's own directory"); + assert.equal(credentialArgs(undefined), ""); +}); + +// Held against git itself: the credential in the store is offered to the forge's origin when it asks, and +// a clone without it fails at once rather than prompting. +test("git presents the stored credential to the forge, and only when one is given", () => { + const dir = mkdtempSync(join(tmpdir(), "lab-git-")); + const store = join(dir, "git-credentials"); + writeFileSync(store, "http://lab-shanks:pw@forge.invalid:20000\n", { mode: 0o600 }); + const fill = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=forge.invalid:20000\\n\\n' | git ${credentialArgs(store)}credential fill`], + { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); + assert.match(fill.stdout, /username=lab-shanks/); + assert.match(fill.stdout, /password=pw/); + const other = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=elsewhere.invalid\\n\\n' | git ${credentialArgs(store)}credential fill`], + { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); + assert.doesNotMatch(other.stdout, /password=pw/); + rmSync(dir, { recursive: true, force: true }); +}); diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts index 718a271f..a9636b58 100644 --- a/modules/lab/tools/index.ts +++ b/modules/lab/tools/index.ts @@ -4,7 +4,13 @@ import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; -import { installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start, stop, tail } from "./runs.js"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + credentialArgs, forgeCredential, installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start, + stop, tail, writeCredential, +} from "./runs.js"; export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs"; @@ -12,6 +18,10 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // ADR 0192): read from the env-file the mesh fills, at each call, so a changed setting is used // without restarting the runtime. MESH_LAB_FORGE itself still wins, for a hand-run instance. const forgeOf = (): string => (env.MESH_LAB_FORGE ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE")).replace(/\/+$/, ""); + // The forge credential the mesh binds and seals to the lab (novox/hq issue 624), read at each call. + const credentialOf = (forge: string): string | undefined => forgeCredential(forge, + env.MESH_LAB_FORGE_BINDING ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING"), + env.MESH_LAB_FORGE_SECRET ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_SECRET")); return [ { name: "lab_check", @@ -29,7 +39,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every // repository it clones answers on the forge, and every one it installs has a lock file where the run // installs from. - const problems = setupProblems(forge, `${work}/check-setup`); + const problems = setupProblems(forge, `${work}/check-setup`, credentialOf(forge)); const output = [`${check.stdout}${check.stderr}`.trim(), ...problems].filter(Boolean).join("\n"); return { ok: check.status === 0 && problems.length === 0, output }; }, @@ -64,7 +74,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` }; const busy = running(work); if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy }; - return { started: true, run: start(work, forge, tests, refs) }; + return { started: true, run: start(work, forge, tests, refs, credentialOf(forge)) }; }, }, { @@ -99,30 +109,44 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { * answer for, and a checkout it installs with nothing installable (installPlan). Empty when a run's setup * would go through. */ -export function setupProblems(forge: string, dir: string): string[] { +export function setupProblems(forge: string, dir: string, credential?: string): string[] { const problems: string[] = []; spawnSync("rm", ["-rf", dir]); - for (const repo of REPOSITORIES) { - const url = `${forge}/novox/${repo}.git`; - const remote = spawnSync("git", ["ls-remote", "--heads", url, "main"], { encoding: "utf8" }); - if (remote.status !== 0 || !remote.stdout.includes("refs/heads/main")) { - problems.push(`a run checks out ${repo}, and ${url} answers no main branch`); - continue; - } - if (!NPM_REPOSITORIES.includes(repo)) continue; - const at = `${dir}/${repo}`; - const clone = spawnSync("git", ["clone", "--quiet", "--depth", "1", url, at], { encoding: "utf8" }); - if (clone.status !== 0) { - problems.push(`${repo} could not be cloned from ${url}: ${clone.stderr.trim()}`); - continue; - } - const plan = installPlan(at); - for (const d of plan.unlocked) problems.push(`${repo}: ${d.slice(at.length) || "/"} has a package.json and no package-lock.json, so a run's npm ci cannot install it`); - if (plan.install.length === 0 && plan.unlocked.length === 0) { - problems.push(`${repo}: no package-lock.json at its root or one directory down, so a run installs nothing of it`); + // The credential a run would offer, in a file of its own outside the check's clones, removed after. + const keep = mkdtempSync(join(tmpdir(), "lab-check-")); + const file = writeCredential(join(keep, "git-credentials"), credential); + const git = (args: string[]) => + spawnSync("sh", ["-c", `git ${credentialArgs(file)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], { + encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, + }); + try { + for (const repo of REPOSITORIES) { + const url = `${forge}/novox/${repo}.git`; + const remote = git(["ls-remote", "--heads", url, "main"]); + if (remote.status !== 0 || !remote.stdout.includes("refs/heads/main")) { + const said = remote.stderr.trim().split("\n")[0] ?? ""; + problems.push(`a run checks out ${repo}, and ${url} could not be read` + + (said ? `: ${said}` : ": no main branch") + + (credential ? "" : " — the lab holds no forge credential (its npm-package-registry binding and secret)")); + continue; + } + if (!NPM_REPOSITORIES.includes(repo)) continue; + const at = `${dir}/${repo}`; + const clone = git(["clone", "--quiet", "--depth", "1", url, at]); + if (clone.status !== 0) { + problems.push(`${repo} could not be cloned from ${url}: ${clone.stderr.trim()}`); + continue; + } + const plan = installPlan(at); + for (const d of plan.unlocked) problems.push(`${repo}: ${d.slice(at.length) || "/"} has a package.json and no package-lock.json, so a run's npm ci cannot install it`); + if (plan.install.length === 0 && plan.unlocked.length === 0) { + problems.push(`${repo}: no package-lock.json at its root or one directory down, so a run installs nothing of it`); + } } + } finally { + rmSync(keep, { recursive: true, force: true }); + spawnSync("rm", ["-rf", dir]); } - spawnSync("rm", ["-rf", dir]); return problems; } diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts index 834605fb..8170a53b 100644 --- a/modules/lab/tools/runs.ts +++ b/modules/lab/tools/runs.ts @@ -60,6 +60,58 @@ export function installStep(root: string, build: boolean): string { fi`; } +/** + * The forge credential a run's clones may present, as a git credential-store line — + * scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none. + * + * **The mesh's credential for the forge, the way the build agent holds it** (novox/hq issue 624): the lab + * requires the package registry, the forge's provisioner binds it an account (`as`) and seals that + * account's password, and the forge that answers npm is the forge that hosts the repositories. Private + * repositories (mesh-tools-go) are cloned with it; public ones clone as before. Written to a file git reads + * through its credential store, never into a URL, an argument, the run's script or its log, and offered + * only to the forge's own scheme, host and port. + */ +export function forgeCredential(forge: string, bindingFile: string, secretFile: string): string | undefined { + if (!forge || !bindingFile || !secretFile) return undefined; + let as = ""; + let secret = ""; + try { + as = String((JSON.parse(readFileSync(bindingFile, "utf8")) as { as?: string }).as ?? ""); + secret = readFileSync(secretFile, "utf8").trim(); + } catch { + return undefined; + } + if (!as || !secret) return undefined; + let origin: URL; + try { + origin = new URL(forge); + } catch { + return undefined; + } + const line = new URL(`${origin.protocol}//${origin.host}`); + line.username = as; + line.password = secret; + return line.toString().replace(/\/$/, ""); +} + +/** Where a run keeps its forge credential: beside its directory, never in it, and removed when it ends. */ +export function credentialPath(work: string, id: string): string { + return join(work, `.${id}.git-credentials`); +} + +/** writeCredential puts a credential-store line where git reads it, readable by this account alone. */ +export function writeCredential(path: string, line: string | undefined): string | undefined { + if (!line) return undefined; + writeFileSync(path, `${line}\n`, { mode: 0o600 }); + return path; +} + +/** The arguments that have git offer the credential in that file, and only that one. */ +export function credentialArgs(path: string | undefined): string { + if (!path) return ""; + return `-c credential.helper= -c ${shellQuote(`credential.helper=store --file=${path}`)} `; +} + export interface RunStatus { id: string; state: "checking-out" | "building" | "running" | "passed" | "failed" | "stopped" | "lost"; @@ -125,7 +177,8 @@ const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`; * The environment is the one the lab's README describes for a run against sibling checkouts, pointed * at this run's own tree, so what is built and claimed is exactly what was checked out. */ -export function script(work: string, id: string, forge: string, tests: string[], refs: Record): string { +export function script(work: string, id: string, forge: string, tests: string[], refs: Record, + credential?: string): string { const dir = runDir(work, id); const setState = (state: string) => `node -e ${shellQuote( @@ -134,12 +187,14 @@ export function script(work: string, id: string, forge: string, tests: string[], const clones = REPOSITORIES.map((repo) => { const ref = refs[repo] ?? "main"; return [ - `git clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, + `git ${credentialArgs(credential)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, `echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`, ].join("\n"); }).join("\n"); const bin = join(dir, "bin"); return `set -euo pipefail +# A repository the forge will not show fails its clone at once, rather than waiting on a prompt. +export GIT_TERMINAL_PROMPT=0 cd ${shellQuote(dir)} ${setState("checking-out")} ${clones} @@ -174,13 +229,15 @@ node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" } /** start begins a run and returns at once with its status. */ -export function start(work: string, forge: string, tests: string[], refs: Record): RunStatus { +export function start(work: string, forge: string, tests: string[], refs: Record, + credential?: string): RunStatus { const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`; const dir = runDir(work, id); mkdirSync(dir, { recursive: true }); const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs }; writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2)); - writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs), { mode: 0o700 }); + const credentials = writeCredential(credentialPath(work, id), credential); + writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials), { mode: 0o700 }); // The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a // run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172). @@ -188,6 +245,7 @@ export function start(work: string, forge: string, tests: string[], refs: Record node -e ${shellQuote( `const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, )} "$code" +rm -f ${shellQuote(credentialPath(work, id))} cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`; const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); child.unref();