From 789d2d9d649f6fdaa6ab2aa04e8301d18ba2b78d Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 13:54:16 +0200 Subject: [PATCH 1/2] lab: clone the forge with the credential the mesh binds and seals to the lab (hq issue 624) mesh-tools-go is private, and the lab cloned over HTTP with no credential: a run failed at its checkout, "could not read Username". The lab now requires the package registry, as the build agent does; the forge's provisioner binds it an account and seals that account's password, and a run offers them to the forge's own origin through git's credential store, from a file beside the run and removed when it ends: never in a URL, the run's script, an argument or the log. lab_check reads every repository a run clones with that same credential and names one it cannot read. --- modules/lab/module.json | 11 +++++- modules/lab/test/runs.test.ts | 51 +++++++++++++++++++++++++ modules/lab/tools/index.ts | 70 +++++++++++++++++++++++------------ modules/lab/tools/runs.ts | 66 +++++++++++++++++++++++++++++++-- 4 files changed, 170 insertions(+), 28 deletions(-) diff --git a/modules/lab/module.json b/modules/lab/module.json index fe523f30..dd9e8921 100644 --- a/modules/lab/module.json +++ b/modules/lab/module.json @@ -5,6 +5,15 @@ "container-runtime", "virtualisation" ], + "requires": [ + "npm-package-registry" + ], + "binds": { + "npm-package-registry": "${dir:state}/package-registry.json" + }, + "secrets": { + "npm-package-registry": "${dir:state}/package-registry.secret" + }, "data": { "own": [ { @@ -33,7 +42,7 @@ "type": "file", "path": "${dir:state}/lab.env", "mode": "0600", - "content": "MESH_LAB_FORGE=${setting:forge}\n" + "content": "MESH_LAB_FORGE=${setting:forge}\nMESH_LAB_FORGE_BINDING=${dir:state}/package-registry.json\nMESH_LAB_FORGE_SECRET=${dir:state}/package-registry.secret\n" }, { "id": "git", diff --git a/modules/lab/test/runs.test.ts b/modules/lab/test/runs.test.ts index 2389b122..aa230ec5 100644 --- a/modules/lab/test/runs.test.ts +++ b/modules/lab/test/runs.test.ts @@ -98,3 +98,54 @@ test("a run checks out mesh-tools-go and says where its checkouts are", () => { assert.doesNotMatch(s, /cd '\/work\/run-x'\/\$repo && npm ci/); for (const repo of NPM_REPOSITORIES) assert.ok(s.includes(`'/work/run-x/${repo}'/package-lock.json`), repo); }); + +// Defends novox/hq issue 624, its second half: mesh-tools-go is private, and the lab cloned over HTTP with +// no credential ("could not read Username"). A run clones with the forge credential the mesh binds and +// seals to the lab, through git's credential store, and the secret is in no script, URL or argument. +import { credentialArgs, credentialPath, forgeCredential } from "../tools/runs.ts"; + +function aBinding(as: string, secret: string): { binding: string; secret: string } { + const dir = mkdtempSync(join(tmpdir(), "lab-binding-")); + writeFileSync(join(dir, "package-registry.json"), JSON.stringify({ provision: "npm-package-registry", at: "10.42.0.1", as, + serves: { scheme: "http", port: 3000 } })); + writeFileSync(join(dir, "package-registry.secret"), `${secret}\n`); + return { binding: join(dir, "package-registry.json"), secret: join(dir, "package-registry.secret") }; +} + +test("the forge credential is the bound account and its sealed secret, for the forge's own origin", () => { + const b = aBinding("lab-shanks", "s3cret-of-forty-characters"); + assert.equal(forgeCredential("http://novox.internal:20000", b.binding, b.secret), + "http://lab-shanks:s3cret-of-forty-characters@novox.internal:20000"); + assert.equal(forgeCredential("http://novox.internal:20000/", b.binding, ""), undefined); + assert.equal(forgeCredential("", b.binding, b.secret), undefined); + assert.equal(forgeCredential("http://novox.internal:20000", "/nowhere", b.secret), undefined); +}); + +test("a run clones through git's credential store, and its script carries no secret", () => { + const b = aBinding("lab-shanks", "s3cret-of-forty-characters"); + const credential = forgeCredential("http://novox.internal:20000", b.binding, b.secret); + const path = credentialPath("/work", "run-x"); + const s = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, path); + assert.ok(credential); + assert.doesNotMatch(s, /s3cret/); + assert.match(s, /git -c credential\.helper= -c 'credential\.helper=store --file=\/work\/\.run-x\.git-credentials' clone --quiet --depth 50 --branch 'main' 'http:\/\/novox\.internal:20000\/novox\/mesh-tools-go\.git'/); + assert.match(s, /export GIT_TERMINAL_PROMPT=0/); + assert.equal(path.startsWith("/work/run-x/"), false, "the credential is kept out of the run's own directory"); + assert.equal(credentialArgs(undefined), ""); +}); + +// Held against git itself: the credential in the store is offered to the forge's origin when it asks, and +// a clone without it fails at once rather than prompting. +test("git presents the stored credential to the forge, and only when one is given", () => { + const dir = mkdtempSync(join(tmpdir(), "lab-git-")); + const store = join(dir, "git-credentials"); + writeFileSync(store, "http://lab-shanks:pw@forge.invalid:20000\n", { mode: 0o600 }); + const fill = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=forge.invalid:20000\\n\\n' | git ${credentialArgs(store)}credential fill`], + { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); + assert.match(fill.stdout, /username=lab-shanks/); + assert.match(fill.stdout, /password=pw/); + const other = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=elsewhere.invalid\\n\\n' | git ${credentialArgs(store)}credential fill`], + { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); + assert.doesNotMatch(other.stdout, /password=pw/); + rmSync(dir, { recursive: true, force: true }); +}); diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts index 718a271f..a9636b58 100644 --- a/modules/lab/tools/index.ts +++ b/modules/lab/tools/index.ts @@ -4,7 +4,13 @@ import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; -import { installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start, stop, tail } from "./runs.js"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + credentialArgs, forgeCredential, installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start, + stop, tail, writeCredential, +} from "./runs.js"; export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const work = env.MESH_LAB_WORK ?? "/var/lib/mesh-lab-runs"; @@ -12,6 +18,10 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // ADR 0192): read from the env-file the mesh fills, at each call, so a changed setting is used // without restarting the runtime. MESH_LAB_FORGE itself still wins, for a hand-run instance. const forgeOf = (): string => (env.MESH_LAB_FORGE ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE")).replace(/\/+$/, ""); + // The forge credential the mesh binds and seals to the lab (novox/hq issue 624), read at each call. + const credentialOf = (forge: string): string | undefined => forgeCredential(forge, + env.MESH_LAB_FORGE_BINDING ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING"), + env.MESH_LAB_FORGE_SECRET ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_SECRET")); return [ { name: "lab_check", @@ -29,7 +39,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every // repository it clones answers on the forge, and every one it installs has a lock file where the run // installs from. - const problems = setupProblems(forge, `${work}/check-setup`); + const problems = setupProblems(forge, `${work}/check-setup`, credentialOf(forge)); const output = [`${check.stdout}${check.stderr}`.trim(), ...problems].filter(Boolean).join("\n"); return { ok: check.status === 0 && problems.length === 0, output }; }, @@ -64,7 +74,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` }; const busy = running(work); if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy }; - return { started: true, run: start(work, forge, tests, refs) }; + return { started: true, run: start(work, forge, tests, refs, credentialOf(forge)) }; }, }, { @@ -99,30 +109,44 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { * answer for, and a checkout it installs with nothing installable (installPlan). Empty when a run's setup * would go through. */ -export function setupProblems(forge: string, dir: string): string[] { +export function setupProblems(forge: string, dir: string, credential?: string): string[] { const problems: string[] = []; spawnSync("rm", ["-rf", dir]); - for (const repo of REPOSITORIES) { - const url = `${forge}/novox/${repo}.git`; - const remote = spawnSync("git", ["ls-remote", "--heads", url, "main"], { encoding: "utf8" }); - if (remote.status !== 0 || !remote.stdout.includes("refs/heads/main")) { - problems.push(`a run checks out ${repo}, and ${url} answers no main branch`); - continue; - } - if (!NPM_REPOSITORIES.includes(repo)) continue; - const at = `${dir}/${repo}`; - const clone = spawnSync("git", ["clone", "--quiet", "--depth", "1", url, at], { encoding: "utf8" }); - if (clone.status !== 0) { - problems.push(`${repo} could not be cloned from ${url}: ${clone.stderr.trim()}`); - continue; - } - const plan = installPlan(at); - for (const d of plan.unlocked) problems.push(`${repo}: ${d.slice(at.length) || "/"} has a package.json and no package-lock.json, so a run's npm ci cannot install it`); - if (plan.install.length === 0 && plan.unlocked.length === 0) { - problems.push(`${repo}: no package-lock.json at its root or one directory down, so a run installs nothing of it`); + // The credential a run would offer, in a file of its own outside the check's clones, removed after. + const keep = mkdtempSync(join(tmpdir(), "lab-check-")); + const file = writeCredential(join(keep, "git-credentials"), credential); + const git = (args: string[]) => + spawnSync("sh", ["-c", `git ${credentialArgs(file)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], { + encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, + }); + try { + for (const repo of REPOSITORIES) { + const url = `${forge}/novox/${repo}.git`; + const remote = git(["ls-remote", "--heads", url, "main"]); + if (remote.status !== 0 || !remote.stdout.includes("refs/heads/main")) { + const said = remote.stderr.trim().split("\n")[0] ?? ""; + problems.push(`a run checks out ${repo}, and ${url} could not be read` + + (said ? `: ${said}` : ": no main branch") + + (credential ? "" : " — the lab holds no forge credential (its npm-package-registry binding and secret)")); + continue; + } + if (!NPM_REPOSITORIES.includes(repo)) continue; + const at = `${dir}/${repo}`; + const clone = git(["clone", "--quiet", "--depth", "1", url, at]); + if (clone.status !== 0) { + problems.push(`${repo} could not be cloned from ${url}: ${clone.stderr.trim()}`); + continue; + } + const plan = installPlan(at); + for (const d of plan.unlocked) problems.push(`${repo}: ${d.slice(at.length) || "/"} has a package.json and no package-lock.json, so a run's npm ci cannot install it`); + if (plan.install.length === 0 && plan.unlocked.length === 0) { + problems.push(`${repo}: no package-lock.json at its root or one directory down, so a run installs nothing of it`); + } } + } finally { + rmSync(keep, { recursive: true, force: true }); + spawnSync("rm", ["-rf", dir]); } - spawnSync("rm", ["-rf", dir]); return problems; } diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts index 834605fb..8170a53b 100644 --- a/modules/lab/tools/runs.ts +++ b/modules/lab/tools/runs.ts @@ -60,6 +60,58 @@ export function installStep(root: string, build: boolean): string { fi`; } +/** + * The forge credential a run's clones may present, as a git credential-store line — + * scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none. + * + * **The mesh's credential for the forge, the way the build agent holds it** (novox/hq issue 624): the lab + * requires the package registry, the forge's provisioner binds it an account (`as`) and seals that + * account's password, and the forge that answers npm is the forge that hosts the repositories. Private + * repositories (mesh-tools-go) are cloned with it; public ones clone as before. Written to a file git reads + * through its credential store, never into a URL, an argument, the run's script or its log, and offered + * only to the forge's own scheme, host and port. + */ +export function forgeCredential(forge: string, bindingFile: string, secretFile: string): string | undefined { + if (!forge || !bindingFile || !secretFile) return undefined; + let as = ""; + let secret = ""; + try { + as = String((JSON.parse(readFileSync(bindingFile, "utf8")) as { as?: string }).as ?? ""); + secret = readFileSync(secretFile, "utf8").trim(); + } catch { + return undefined; + } + if (!as || !secret) return undefined; + let origin: URL; + try { + origin = new URL(forge); + } catch { + return undefined; + } + const line = new URL(`${origin.protocol}//${origin.host}`); + line.username = as; + line.password = secret; + return line.toString().replace(/\/$/, ""); +} + +/** Where a run keeps its forge credential: beside its directory, never in it, and removed when it ends. */ +export function credentialPath(work: string, id: string): string { + return join(work, `.${id}.git-credentials`); +} + +/** writeCredential puts a credential-store line where git reads it, readable by this account alone. */ +export function writeCredential(path: string, line: string | undefined): string | undefined { + if (!line) return undefined; + writeFileSync(path, `${line}\n`, { mode: 0o600 }); + return path; +} + +/** The arguments that have git offer the credential in that file, and only that one. */ +export function credentialArgs(path: string | undefined): string { + if (!path) return ""; + return `-c credential.helper= -c ${shellQuote(`credential.helper=store --file=${path}`)} `; +} + export interface RunStatus { id: string; state: "checking-out" | "building" | "running" | "passed" | "failed" | "stopped" | "lost"; @@ -125,7 +177,8 @@ const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`; * The environment is the one the lab's README describes for a run against sibling checkouts, pointed * at this run's own tree, so what is built and claimed is exactly what was checked out. */ -export function script(work: string, id: string, forge: string, tests: string[], refs: Record): string { +export function script(work: string, id: string, forge: string, tests: string[], refs: Record, + credential?: string): string { const dir = runDir(work, id); const setState = (state: string) => `node -e ${shellQuote( @@ -134,12 +187,14 @@ export function script(work: string, id: string, forge: string, tests: string[], const clones = REPOSITORIES.map((repo) => { const ref = refs[repo] ?? "main"; return [ - `git clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, + `git ${credentialArgs(credential)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, `echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`, ].join("\n"); }).join("\n"); const bin = join(dir, "bin"); return `set -euo pipefail +# A repository the forge will not show fails its clone at once, rather than waiting on a prompt. +export GIT_TERMINAL_PROMPT=0 cd ${shellQuote(dir)} ${setState("checking-out")} ${clones} @@ -174,13 +229,15 @@ node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" } /** start begins a run and returns at once with its status. */ -export function start(work: string, forge: string, tests: string[], refs: Record): RunStatus { +export function start(work: string, forge: string, tests: string[], refs: Record, + credential?: string): RunStatus { const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`; const dir = runDir(work, id); mkdirSync(dir, { recursive: true }); const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs }; writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2)); - writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs), { mode: 0o700 }); + const credentials = writeCredential(credentialPath(work, id), credential); + writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials), { mode: 0o700 }); // The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a // run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172). @@ -188,6 +245,7 @@ export function start(work: string, forge: string, tests: string[], refs: Record node -e ${shellQuote( `const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, )} "$code" +rm -f ${shellQuote(credentialPath(work, id))} cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`; const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); child.unref(); From 8d8dcd5ebb8688621292db699b5e375d2b115e72 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 14:00:59 +0200 Subject: [PATCH 2/2] lab: a read-only forge account of its own, and no credential left on disk after a stop (hq issue 624, review 196) The lab's credential came from the package team, which also writes the @novox packages, and the branches a run clones are not reviewed yet. The lab now contributes {"access": "read"} to its npm-package-registry requirement, and gitea's provisioner puts such a consumer on a team of its own, readers: code read on the org's repositories, package read, no write, and off the package team. The build agent, which publishes, stays on packages. A stop's SIGTERM to the run's process group killed the wrapper before it removed the credential file. The run's script now removes it as soon as its clones are done and traps every way it ends, and a run's start and lab_check remove what a crash or a reboot left. --- modules/gitea/client.ts | 9 +++++ modules/gitea/package.json | 2 +- modules/gitea/provisioner/index.ts | 29 +++++--------- modules/gitea/provisioner/teams.ts | 64 ++++++++++++++++++++++++++++++ modules/gitea/test/teams.test.ts | 56 ++++++++++++++++++++++++++ modules/gitea/tsconfig.json | 2 +- modules/lab/module.json | 5 +++ modules/lab/test/runs.test.ts | 47 +++++++++++++++++++++- modules/lab/tools/index.ts | 9 ++++- modules/lab/tools/runs.ts | 37 ++++++++++++++++- 10 files changed, 234 insertions(+), 26 deletions(-) create mode 100644 modules/gitea/provisioner/teams.ts create mode 100644 modules/gitea/test/teams.test.ts diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 2a92d80d..4ac40fb9 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -905,6 +905,15 @@ export class GiteaAdmin { GiteaAdmin.fail(`/teams/${teamId}/members/${username}`, res); } + /** Take a user off a team; one not on it is already off. */ + async removeUserFromTeam(teamId: number, username: string): Promise { + const res = await this.request(`/teams/${teamId}/members/${encodeURIComponent(username)}`, { + method: "DELETE", + }); + if (res.status === 204 || res.status === 200 || res.status === 404) return; + GiteaAdmin.fail(`/teams/${teamId}/members/${username}`, res); + } + /** * Whether a consumer's user logs in with exactly this password and is still a member of the * package team. Read-only: the password is checked as the consumer presents it, basic auth on the diff --git a/modules/gitea/package.json b/modules/gitea/package.json index ec334404..21a6feb5 100644 --- a/modules/gitea/package.json +++ b/modules/gitea/package.json @@ -5,7 +5,7 @@ "type": "module", "private": true, "scripts": { - "build": "tsc client.ts token.ts index.ts provisioner/index.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist", + "build": "tsc client.ts token.ts index.ts provisioner/index.ts provisioner/teams.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist", "test": "npm run build && node --test --experimental-strip-types 'test/*.test.ts'" }, "dependencies": { diff --git a/modules/gitea/provisioner/index.ts b/modules/gitea/provisioner/index.ts index fdd8b319..9efc083b 100644 --- a/modules/gitea/provisioner/index.ts +++ b/modules/gitea/provisioner/index.ts @@ -11,8 +11,8 @@ // // The `npm-package-registry` interface: a consumer authenticates to the npm registry at // `/api/packages/novox/npm/` with basic auth, as `as` with the password the mesh minted, and can -// read and write packages under the `@novox` scope. The registry's npm owner is the gitea org -// `novox`; a consumer is a gitea *user* placed on that org's package team. +// read packages under the `@novox` scope, and write them unless it asked for read only (teams.ts). The +// registry's npm owner is the gitea org `novox`; a consumer is a gitea *user* on one of that org's teams. // // **The user name and password are the mesh's, not the provisioner's (ADR 0048).** The mesh derives // the login and hands it to both ends, and mints the password. gitea creates a user under exactly @@ -24,11 +24,9 @@ import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner"; import { GiteaAdmin } from "../client.js"; +import { grant, holds } from "./teams.js"; -// The npm registry owner: a gitea org named `novox`, whose package team every consumer joins so it -// can read and write packages under the `@novox` scope (ADR 0076). -const ORG = "novox"; -const PACKAGE_TEAM = "packages"; +// Which team a consumer joins — `packages` to publish, `readers` to read only — is teams.ts's. const gitea = GiteaAdmin.fromEnv(); @@ -38,19 +36,10 @@ const gitea = GiteaAdmin.fromEnv(); // module needs the mesh to say where each provision's file is — not yet possible, and not faked. runProvisioner("npm-package-registry", { async create(p: Provision): Promise { - // The org and its package team are the same for every consumer; ensuring them per-create is - // idempotent and needs no separate bootstrap step. - await gitea.ensureOrg(ORG); - const teamId = await gitea.ensureTeam(ORG, PACKAGE_TEAM, true); - // The user carries the consumer's login and the mesh's minted password, set every run so a - // rotation takes. Membership of the package team is what grants read+write on packages. - // - // The address is gitea's own convention for one that is not real: its email validation - // requires a dotted domain, so `@localhost` was refused at create — the fault that had this - // grant retrying for a day — while `@noreply.localhost` is the shape gitea itself gives - // hidden addresses. - await gitea.ensureUser(p.as, p.password, `${p.as}@noreply.localhost`); - await gitea.addUserToTeam(teamId, p.as); + // The org and its teams are ensured per create: idempotent, and no separate bootstrap step. The + // user carries the consumer's login and the mesh's minted password, set every run so a rotation + // takes; membership of its team is what grants it what it may do. + await grant(gitea, p); }, async remove(p: { as: string }): Promise { @@ -60,6 +49,6 @@ runProvisioner("npm-package-registry", { // Asked every minute by the harness: whether the backend still holds this consumer exactly as // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120). async holds(p: Provision): Promise { - return gitea.holdsTeamMember(ORG, PACKAGE_TEAM, p.as, p.password); + return holds(gitea, p); }, }); diff --git a/modules/gitea/provisioner/teams.ts b/modules/gitea/provisioner/teams.ts new file mode 100644 index 00000000..3183af79 --- /dev/null +++ b/modules/gitea/provisioner/teams.ts @@ -0,0 +1,64 @@ +// Which team of the forge's npm owner a consumer of `npm-package-registry` joins, and the per-consumer +// half of the provisioner around it — pure of the forge's address, so it is tested with a stand-in. +// +// **Two teams, by what the consumer said it needs** (novox/hq issue 624). A consumer that publishes — +// the build agent, whose builds publish the SDK and every TypeScript module's packages — joins `packages`: +// code read on the org's repositories and write on its packages. A consumer that contributes +// `{"access": "read"}` joins `readers`: code read and package read, and no write. The lab is the first: it +// clones branches that are not yet reviewed and runs their code beside its credential, and code that is +// not reviewed must not be able to publish the mesh's packages with it. + +/** The npm registry's owner: the gitea org every consumer's packages live under (ADR 0076). */ +export const ORG = "novox"; +/** The team whose members read and write the org's packages. */ +export const PACKAGE_TEAM = "packages"; +/** The team whose members read the org's code and packages, and write nothing. */ +export const READ_TEAM = "readers"; + +/** The team a consumer joins, from what it contributed; an access it may not ask for is refused. */ +export function teamFor(values: Readonly> | undefined): { team: string; packageWrite: boolean } { + const access = values?.access; + if (access === undefined || access === "write") return { team: PACKAGE_TEAM, packageWrite: true }; + if (access === "read") return { team: READ_TEAM, packageWrite: false }; + throw new Error(`npm-package-registry: access ${JSON.stringify(access)} is not "read" or "write"`); +} + +/** What the adapter asks of the forge's admin API (client.ts GiteaAdmin). */ +export interface Forge { + ensureOrg(org: string): Promise; + ensureTeam(org: string, team: string, packageWrite: boolean): Promise; + ensureUser(username: string, password: string, email: string): Promise; + addUserToTeam(teamId: number, username: string): Promise; + removeUserFromTeam(teamId: number, username: string): Promise; + holdsTeamMember(org: string, team: string, username: string, password: string): Promise; +} + +export interface Consumer { + readonly as: string; + readonly password: string; + readonly values: Readonly>; +} + +/** + * grant makes the consumer's user, with exactly the mesh's password, a member of its one team. A consumer + * that asks for read only is also taken out of the package team, so one that asked for write before and + * read now loses the write at once. + */ +export async function grant(forge: Forge, p: Consumer): Promise { + const { team, packageWrite } = teamFor(p.values); + await forge.ensureOrg(ORG); + const teamId = await forge.ensureTeam(ORG, team, packageWrite); + // The address is gitea's own convention for one that is not real: its email validation requires a + // dotted domain, so `@localhost` was refused at create, while `@noreply.localhost` is the shape gitea + // itself gives hidden addresses. + await forge.ensureUser(p.as, p.password, `${p.as}@noreply.localhost`); + await forge.addUserToTeam(teamId, p.as); + if (!packageWrite) { + await forge.removeUserFromTeam(await forge.ensureTeam(ORG, PACKAGE_TEAM, true), p.as); + } +} + +/** holds is whether the forge still holds the consumer as the mesh gave it: its login, in its team. */ +export async function holds(forge: Forge, p: Consumer): Promise { + return forge.holdsTeamMember(ORG, teamFor(p.values).team, p.as, p.password); +} diff --git a/modules/gitea/test/teams.test.ts b/modules/gitea/test/teams.test.ts new file mode 100644 index 00000000..11def05e --- /dev/null +++ b/modules/gitea/test/teams.test.ts @@ -0,0 +1,56 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { grant, holds, PACKAGE_TEAM, READ_TEAM, teamFor, type Forge } from "../provisioner/teams.ts"; + +// Defends novox/hq issue 624: the lab reads the forge with a credential of its own that can write no +// package. A consumer that contributes {"access": "read"} joins a team with code and package read only. + +function aForge(): { forge: Forge; did: string[] } { + const did: string[] = []; + const teams: Record = {}; + const forge: Forge = { + async ensureOrg(org) { did.push(`org ${org}`); }, + async ensureTeam(org, team, packageWrite) { + did.push(`team ${org}/${team} packages:${packageWrite ? "write" : "read"}`); + return (teams[team] ??= Object.keys(teams).length + 1); + }, + async ensureUser(username) { did.push(`user ${username}`); }, + async addUserToTeam(id, username) { did.push(`add ${username} to ${id}`); }, + async removeUserFromTeam(id, username) { did.push(`remove ${username} from ${id}`); }, + async holdsTeamMember(org, team, username) { did.push(`holds ${username} in ${org}/${team}`); return true; }, + }; + return { forge, did }; +} + +test("a consumer that asks nothing, or write, publishes: the package team", () => { + assert.deepEqual(teamFor({}), { team: PACKAGE_TEAM, packageWrite: true }); + assert.deepEqual(teamFor({ access: "write" }), { team: PACKAGE_TEAM, packageWrite: true }); +}); + +test("a consumer that asks to read joins the readers, who write no package", () => { + assert.deepEqual(teamFor({ access: "read" }), { team: READ_TEAM, packageWrite: false }); + assert.throws(() => teamFor({ access: "admin" }), /not "read" or "write"/); +}); + +test("a reader is made, put on the readers team, and taken off the package team", async () => { + const { forge, did } = aForge(); + await grant(forge, { as: "lab-shanks", password: "pw", values: { access: "read" } }); + assert.deepEqual(did, [ + "org novox", + "team novox/readers packages:read", + "user lab-shanks", + "add lab-shanks to 1", + "team novox/packages packages:write", + "remove lab-shanks from 2", + ]); + did.length = 0; + await holds(forge, { as: "lab-shanks", password: "pw", values: { access: "read" } }); + assert.deepEqual(did, ["holds lab-shanks in novox/readers"]); +}); + +test("a publisher is made and put on the package team, as before", async () => { + const { forge, did } = aForge(); + await grant(forge, { as: "build-agent-ace", password: "pw", values: {} }); + assert.deepEqual(did, ["org novox", "team novox/packages packages:write", "user build-agent-ace", "add build-agent-ace to 1"]); +}); diff --git a/modules/gitea/tsconfig.json b/modules/gitea/tsconfig.json index e416b008..7b5440b7 100644 --- a/modules/gitea/tsconfig.json +++ b/modules/gitea/tsconfig.json @@ -8,5 +8,5 @@ "skipLibCheck": true, "noEmit": true }, - "include": ["client.ts", "token.ts", "pulls.ts", "protection.ts", "index.ts", "provisioner/index.ts", "tools/index.ts"] + "include": ["client.ts", "token.ts", "pulls.ts", "protection.ts", "index.ts", "provisioner/index.ts", "provisioner/teams.ts", "tools/index.ts"] } diff --git a/modules/lab/module.json b/modules/lab/module.json index dd9e8921..5406579d 100644 --- a/modules/lab/module.json +++ b/modules/lab/module.json @@ -8,6 +8,11 @@ "requires": [ "npm-package-registry" ], + "contributes": { + "npm-package-registry": { + "access": "read" + } + }, "binds": { "npm-package-registry": "${dir:state}/package-registry.json" }, diff --git a/modules/lab/test/runs.test.ts b/modules/lab/test/runs.test.ts index aa230ec5..816d1baa 100644 --- a/modules/lab/test/runs.test.ts +++ b/modules/lab/test/runs.test.ts @@ -1,6 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -149,3 +149,48 @@ test("git presents the stored credential to the forge, and only when one is give assert.doesNotMatch(other.stdout, /password=pw/); rmSync(dir, { recursive: true, force: true }); }); + +// Review 196 on mesh-catalog #272: the credential file outlived a stop, because SIGTERM to the run's +// process group killed the wrapper before its cleanup. It is now removed right after the clones, on +// every way the script ends, and at the next run's start and in lab_check for what a crash left. +import { cleanCredentials, credentialGuard } from "../tools/runs.ts"; + +test("a stopped script removes the credential, whatever ends it", () => { + for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"] as const) { + const dir = mkdtempSync(join(tmpdir(), "lab-guard-")); + const file = join(dir, ".run-x.git-credentials"); + writeFileSync(file, "http://u:p@h\n", { mode: 0o600 }); + const ran = spawnSync("bash", ["-c", `${credentialGuard(file)}\nkill -${signal.slice(3)} $$\nsleep 5`], { encoding: "utf8", timeout: 10_000 }); + assert.notEqual(ran.status, 0, `${signal}: the script was not stopped`); + assert.equal(existsSync(file), false, `${signal} left the credential on disk`); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("the script removes the credential as soon as its clones are done", () => { + const s = script("/work", "run-x", "http://forge", ["t.test.ts"], {}, "/work/.run-x.git-credentials"); + const lastClone = s.lastIndexOf("clone --quiet"); + const removed = s.indexOf("rm -f '/work/.run-x.git-credentials'", lastClone); + const building = s.indexOf("npm ci"); + assert.ok(lastClone > 0 && removed > lastClone && removed < building, "not removed between the clones and the build"); + assert.match(s, /trap 'rm -f '\\''\/work\/\.run-x\.git-credentials'\\''' EXIT/); +}); + +test("credentials a crashed run left are removed, but not the running one's", () => { + const work = mkdtempSync(join(tmpdir(), "lab-work-")); + for (const n of [".run-a.git-credentials", ".run-b.git-credentials", "run-a", "other"]) writeFileSync(join(work, n), "x"); + const removed = cleanCredentials(work, join(work, ".run-b.git-credentials")); + assert.deepEqual(removed, [join(work, ".run-a.git-credentials")]); + assert.ok(existsSync(join(work, ".run-b.git-credentials")) && existsSync(join(work, "run-a")) && existsSync(join(work, "other"))); + rmSync(work, { recursive: true, force: true }); +}); + +// The lab asks the forge for read only (review 196): its credential is for clones of branches nobody has +// reviewed yet, and must not be able to publish the mesh's packages. +test("the lab asks the package registry for read only", () => { + const m = JSON.parse(readFileSync(join(import.meta.dirname, "..", "module.json"), "utf8")) as { + requires: unknown[]; contributes?: Record>; + }; + assert.ok(m.requires.includes("npm-package-registry")); + assert.equal(m.contributes?.["npm-package-registry"]?.access, "read"); +}); diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts index a9636b58..ce665cbf 100644 --- a/modules/lab/tools/index.ts +++ b/modules/lab/tools/index.ts @@ -8,7 +8,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { - credentialArgs, forgeCredential, installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start, + cleanCredentials, credentialArgs, forgeCredential, installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start, stop, tail, writeCredential, } from "./runs.js"; @@ -39,8 +39,13 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every // repository it clones answers on the forge, and every one it installs has a lock file where the run // installs from. + // A credential a stopped, crashed or rebooted run left behind is removed, and said; one still in use + // by a running run is not. + const busy = running(work); + const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined); const problems = setupProblems(forge, `${work}/check-setup`, credentialOf(forge)); - const output = [`${check.stdout}${check.stderr}`.trim(), ...problems].filter(Boolean).join("\n"); + const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : ""; + const output = [`${check.stdout}${check.stderr}`.trim(), removed, ...problems].filter(Boolean).join("\n"); return { ok: check.status === 0 && problems.length === 0, output }; }, }, diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts index 8170a53b..debc0bc0 100644 --- a/modules/lab/tools/runs.ts +++ b/modules/lab/tools/runs.ts @@ -7,7 +7,7 @@ // for it, and says it was lost rather than pretending it is still going. import { spawn } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; /** @@ -106,6 +106,37 @@ export function writeCredential(path: string, line: string | undefined): string return path; } +/** + * credentialGuard is the shell that removes the credential however the script ends — its end, an error, + * or a stop's SIGTERM to the run's process group, which kills the wrapper before its own cleanup runs + * (review 196 on mesh-catalog #272). The script also removes it as soon as its clones are done. + */ +export function credentialGuard(path: string | undefined): string { + if (!path) return ""; + const rm = `rm -f ${shellQuote(path)}`; + return `trap ${shellQuote(rm)} EXIT +trap ${shellQuote(`${rm}; exit 143`)} TERM +trap ${shellQuote(`${rm}; exit 130`)} INT +trap ${shellQuote(`${rm}; exit 129`)} HUP`; +} + +/** + * cleanCredentials removes every run's credential file left in the work directory but the one named — what + * a crash or a reboot left behind, which no trap could remove. Answers what it removed. + */ +export function cleanCredentials(work: string, keep?: string): string[] { + if (!existsSync(work)) return []; + const removed: string[] = []; + for (const name of readdirSync(work)) { + if (!/^\.run-.*\.git-credentials$/.test(name)) continue; + const path = join(work, name); + if (path === keep) continue; + rmSync(path, { force: true }); + removed.push(path); + } + return removed; +} + /** The arguments that have git offer the credential in that file, and only that one. */ export function credentialArgs(path: string | undefined): string { if (!path) return ""; @@ -195,9 +226,11 @@ export function script(work: string, id: string, forge: string, tests: string[], return `set -euo pipefail # A repository the forge will not show fails its clone at once, rather than waiting on a prompt. export GIT_TERMINAL_PROMPT=0 +${credentialGuard(credential)} cd ${shellQuote(dir)} ${setState("checking-out")} ${clones} +${credential ? `# The forge credential is for the clones, and nothing after them runs with it on disk.\nrm -f ${shellQuote(credential)}` : ""} node -e ${shellQuote( `const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`, )} @@ -236,6 +269,8 @@ export function start(work: string, forge: string, tests: string[], refs: Record mkdirSync(dir, { recursive: true }); const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs }; writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2)); + // What a crashed or rebooted run left behind goes before this run writes its own. + cleanCredentials(work); const credentials = writeCredential(credentialPath(work, id), credential); writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials), { mode: 0o700 });