diff --git a/modules/lab/test/runs.test.ts b/modules/lab/test/runs.test.ts index f87bf3d2..87dc1d12 100644 --- a/modules/lab/test/runs.test.ts +++ b/modules/lab/test/runs.test.ts @@ -251,3 +251,71 @@ test("the lab's tools bundle names its bound credential and binding in its env", assert.equal(env.MESH_LAB_FORGE_SECRET, m.secrets["npm-package-registry"]); assert.equal(env.MESH_LAB_FORGE_BINDING, m.binds["npm-package-registry"]); }); + +// The next setup failure of novox/hq issue 624 (run-2026-10-13T13-15-26-634Z on shanks): npm 12 refused +// mesh-tools' locked @novox/mesh-sdk with EALLOWREMOTE. The lab named the registry by its forge setting's +// address (http://novox.internal:20000/…), the lock file holds the address the forge writes every tarball +// URL at (https://git.novox.be/…), and npm 12 installs a locked tarball only from its configured registry. +// A run names the registry by the address the registry names itself by. +import { npmRegistry } from "../tools/runs.ts"; + +function aRegistry(answer: () => Response | Promise): { get: typeof fetch; asked: string[] } { + const asked: string[] = []; + const get = (async (url: string | URL | Request) => { + asked.push(String(url)); + return answer(); + }) as typeof fetch; + return { get, asked }; +} + +const packument = (tarball: string) => new Response(JSON.stringify({ + name: "@novox/mesh-sdk", "dist-tags": { latest: "0.1.14" }, + versions: { "0.1.1": { dist: { tarball: "http://old.example/api/packages/novox/npm/x.tgz" } }, "0.1.14": { dist: { tarball } } }, +}), { status: 200, headers: { "content-type": "application/json" } }); + +test("a run names the package registry by the address the registry gives its tarballs", async () => { + const r = aRegistry(() => packument("https://forge.example.org/api/packages/novox/npm/%40novox%2Fmesh-sdk/-/0.1.14/mesh-sdk-0.1.14.tgz")); + const got = await npmRegistry("http://forge.example.org:3000/", r.get); + assert.deepEqual(got, { registry: "https://forge.example.org/api/packages/novox/npm/" }); + assert.deepEqual(r.asked, ["http://forge.example.org:3000/api/packages/novox/npm/@novox%2fmesh-sdk"]); + // The forge's own scheme is taken too, with its port. + const plain = await npmRegistry("http://forge.example.org:3000", + aRegistry(() => packument("http://forge.example.org:3000/api/packages/novox/npm/%40novox%2Fmesh-sdk/-/0.1.14/mesh-sdk-0.1.14.tgz")).get); + assert.deepEqual(plain, { registry: "http://forge.example.org:3000/api/packages/novox/npm/" }); +}); + +// What the registry answers may come over plain HTTP, so it never decides where packages come from: only +// the forge's own host, at https or the forge's scheme, under the npm path, with nothing after it +// (review of mesh-catalog #275). +test("a registry that does not give an address the lab takes leaves the forge's, and the reason is said", async () => { + const sdk = "/api/packages/novox/npm/%40novox%2Fmesh-sdk/-/0.1.14/mesh-sdk-0.1.14.tgz"; + for (const [answer, why] of [ + [() => new Response("not found", { status: 404 }), /it answered 404/], + [() => new Response("{}", { status: 200 }), /no tarball/], + [() => { throw new Error("connect ECONNREFUSED"); }, /ECONNREFUSED/], + [() => packument(`https://forge.example.org/elsewhere/mesh-sdk-0.1.14.tgz`), /not under \/api\/packages\/novox\/npm\//], + [() => packument(`https://attacker.example.net${sdk}`), /attacker\.example\.net, which is not the forge's host novox\.internal/], + [() => packument(`https://attacker.example.net/x?u=https://novox.internal${sdk}`), /query or a fragment/], + [() => packument(`https://novox.internal${sdk}?x=1`), /query or a fragment/], + [() => packument(`https://novox.internal${sdk}#frag`), /query or a fragment/], + [() => packument(`https://attacker.example.net/#https://novox.internal${sdk}`), /query or a fragment/], + [() => packument(`ftp://novox.internal${sdk}`), /ftp: — neither https: nor the forge's http:/], + [() => packument(`file:///api/packages/novox/npm/x.tgz`), /file: — neither/], + [() => packument(`https://novox.internal/api/packages/novox/npm/\nregistry=https://attacker.example.net/\nallow-remote=all`), /whitespace or a control character/], + [() => packument(`https://user:pw@novox.internal${sdk}`), /user or password/], + ] as [() => Response, RegExp][]) { + const got = await npmRegistry("http://novox.internal:20000", aRegistry(answer).get); + assert.equal(got.registry, "http://novox.internal:20000/api/packages/novox/npm/", String(why)); + assert.match(got.problem ?? "", /did not give an address the lab takes/); + assert.match(got.problem ?? "", why); + } +}); + +test("a run's npm configuration names the registry it was given, and the forge's own address only without one", () => { + const given = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, undefined, + "https://forge.example.org/api/packages/novox/npm/"); + assert.match(given, / '@novox:registry=https:\/\/forge\.example\.org\/api\/packages\/novox\/npm\/' > '\/work\/run-x\/\.npmrc'\n/); + assert.doesNotMatch(given, /@novox:registry=http:\/\/novox\.internal/); + const none = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}); + assert.match(none, /'@novox:registry=http:\/\/novox\.internal:20000\/api\/packages\/novox\/npm\/'/); +}); diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts index d13d5cc7..7a3013e9 100644 --- a/modules/lab/tools/index.ts +++ b/modules/lab/tools/index.ts @@ -8,7 +8,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { - checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start, + checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start, stop, tail, writeCredential, } from "./runs.js"; @@ -42,6 +42,9 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" }); check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" }); problems = setupProblems(forge, `${dir}/setup`, credentialOf(forge)); + // And the address a run names the package registry by (npmRegistry): said when the registry does not say it. + const registry = await npmRegistry(forge); + if (registry.problem) problems.push(registry.problem); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -87,7 +90,9 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` }; const busy = running(work); if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy }; - return { started: true, run: start(work, forge, tests, refs, credentialOf(forge)) }; + const registry = await npmRegistry(forge); + return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry), + registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) }; }, }, { diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts index 9d06f18f..5572b843 100644 --- a/modules/lab/tools/runs.ts +++ b/modules/lab/tools/runs.ts @@ -60,6 +60,70 @@ export function installStep(root: string, build: boolean): string { fi`; } +/** Where the forge serves the mesh's npm packages, under its own address (gitea's npm registry for the novox org). */ +export const NPM_PATH = "/api/packages/novox/npm/"; + +/** + * The address a run names the @novox registry by: the one the registry names itself by, when that is the + * forge's own host (novox/hq issue 624). + * + * **npm 12 installs a locked tarball only from the registry it is configured with.** Its `allow-remote` + * defaults to `none`, and a lock file's `resolved` URL counts as the registry's only when its origin and + * path are the configured scope registry's; any other is a "remote" tarball and refused (EALLOWREMOTE). + * The forge writes every tarball URL — in what it answers, and so in every lock file made against it — at + * its own public address, whichever address it was asked at. So the lab asks the registry, at the forge, + * for the SDK's newest tarball and names the registry by the origin of that URL. + * + * **What the registry answers is not trusted for where packages come from** (review of mesh-catalog #275): + * it may be fetched over plain HTTP. The tarball URL is parsed, and taken only when it is https (or the + * forge's own scheme), its path is under the npm path, it carries no query or fragment, and its host is + * the forge's own host. The registry is then built from + * the parsed origin and the npm path alone, never from the string answered, so nothing the registry says + * reaches the run's npm configuration but a scheme, a host and a port. Anything else falls back to the + * forge's own address and says why, so a run still starts and lab_check names it. + * + * So the lab's forge setting is the address the forge names itself by (its public https name), not a + * second name for it: a forge set as another name of the same server is refused here, named by lab_check, + * and its runs stop at the first locked @novox package. + */ +export async function npmRegistry(forge: string, get: typeof fetch = fetch): Promise<{ registry: string; problem?: string }> { + const asked = `${forge.replace(/\/+$/, "")}${NPM_PATH}`; + const url = `${asked}@novox%2fmesh-sdk`; + const fallback = (why: string) => ({ + registry: asked, + problem: `the package registry at ${url} did not give an address the lab takes (${why}), so a run names it ${asked}, and npm 12 refuses a locked @novox tarball at any other`, + }); + let own: URL; + try { + own = new URL(forge); + } catch { + return fallback(`the forge ${JSON.stringify(forge)} is not a URL`); + } + try { + const res = await get(url, { signal: AbortSignal.timeout(15_000) }); + if (!res.ok) return fallback(`it answered ${res.status}`); + const doc = await res.json() as { "dist-tags"?: { latest?: string }; versions?: Record }; + const latest = doc["dist-tags"]?.latest ?? ""; + const tarball = doc.versions?.[latest]?.dist?.tarball; + if (typeof tarball !== "string" || !tarball) return fallback("no tarball for its latest version"); + if (/[\s\u0000-\u001f\u007f]/.test(tarball)) return fallback("its tarball URL holds whitespace or a control character"); + let t: URL; + try { + t = new URL(tarball); + } catch { + return fallback("its tarball is not a URL"); + } + if (t.protocol !== "https:" && t.protocol !== own.protocol) return fallback(`its tarball is ${t.protocol} — neither https: nor the forge's ${own.protocol}`); + if (t.username || t.password) return fallback("its tarball URL carries a user or password"); + if (t.search || t.hash) return fallback("its tarball URL carries a query or a fragment"); + if (!t.pathname.startsWith(NPM_PATH)) return fallback(`its tarball is not under ${NPM_PATH}`); + if (t.hostname !== own.hostname) return fallback(`its tarball is at ${t.hostname}, which is not the forge's host ${own.hostname}`); + return { registry: `${t.origin}${NPM_PATH}` }; + } catch (e) { + return fallback(e instanceof Error ? e.message : String(e)); + } +} + /** * The forge credential a run's clones may present, as a git credential-store line — * scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none. @@ -240,7 +304,7 @@ const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`; * at this run's own tree, so what is built and claimed is exactly what was checked out. */ export function script(work: string, id: string, forge: string, tests: string[], refs: Record, - credential?: string): string { + credential?: string, registry = `${forge}${NPM_PATH}`): string { const dir = runDir(work, id); const setState = (state: string) => `node -e ${shellQuote( @@ -266,9 +330,9 @@ node -e ${shellQuote( `const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`, )} ${setState("building")} -# The @novox scope resolves from the mesh's own package registry on the forge, as the builder -# resolves it; nothing else is asked of it. -printf '%s\n' ${shellQuote(`@novox:registry=${forge}/api/packages/novox/npm/`)} > ${shellQuote(join(dir, ".npmrc"))} +# The @novox scope resolves from the mesh's own package registry on the forge, named by the address it +# names itself by (npmRegistry), which is the address its lock files hold; nothing else is asked of it. +printf '%s\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))} export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))} ${NPM_REPOSITORIES.map((repo) => installStep(join(dir, repo), repo !== "mesh-lab")).join("\n")} mkdir -p ${shellQuote(bin)} @@ -294,7 +358,7 @@ node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" /** start begins a run and returns at once with its status. */ export function start(work: string, forge: string, tests: string[], refs: Record, - credential?: string): RunStatus { + credential?: string, registry?: string): RunStatus { const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`; const dir = runDir(work, id); mkdirSync(dir, { recursive: true }); @@ -303,7 +367,7 @@ export function start(work: string, forge: string, tests: string[], refs: Record // What a crashed or rebooted run left behind goes before this run writes its own. cleanCredentials(work); const credentials = writeCredential(credentialPath(work, id), credential); - writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials), { mode: 0o700 }); + writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials, registry), { mode: 0o700 }); // The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a // run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172).