Merge pull request 'lab: name the package registry by the address it names itself by, so npm 12 installs locked @novox packages (issue 624)' (#275) from fix/624-lab-names-the-registry-as-it-names-itself into main

modules/lab/test/runs.test.ts

@@ -251,3 +251,71 @@ test("the lab's tools bundle names its bound credential and binding in its env",
   assert.equal(env.MESH_LAB_FORGE_SECRET, m.secrets["npm-package-registry"]);
   assert.equal(env.MESH_LAB_FORGE_BINDING, m.binds["npm-package-registry"]);
 });
+
+// The next setup failure of novox/hq issue 624 (run-2026-10-13T13-15-26-634Z on shanks): npm 12 refused
+// mesh-tools' locked @novox/mesh-sdk with EALLOWREMOTE. The lab named the registry by its forge setting's
+// address (http://novox.internal:20000/…), the lock file holds the address the forge writes every tarball
+// URL at (https://git.novox.be/…), and npm 12 installs a locked tarball only from its configured registry.
+// A run names the registry by the address the registry names itself by.
+import { npmRegistry } from "../tools/runs.ts";
+
+function aRegistry(answer: () => Response | Promise<Response>): { get: typeof fetch; asked: string[] } {
+  const asked: string[] = [];
+  const get = (async (url: string | URL | Request) => {
+    asked.push(String(url));
+    return answer();
+  }) as typeof fetch;
+  return { get, asked };
+}
+
+const packument = (tarball: string) => new Response(JSON.stringify({
+  name: "@novox/mesh-sdk", "dist-tags": { latest: "0.1.14" },
+  versions: { "0.1.1": { dist: { tarball: "http://old.example/api/packages/novox/npm/x.tgz" } }, "0.1.14": { dist: { tarball } } },
+}), { status: 200, headers: { "content-type": "application/json" } });
+
+test("a run names the package registry by the address the registry gives its tarballs", async () => {
+  const r = aRegistry(() => packument("https://forge.example.org/api/packages/novox/npm/%40novox%2Fmesh-sdk/-/0.1.14/mesh-sdk-0.1.14.tgz"));
+  const got = await npmRegistry("http://forge.example.org:3000/", r.get);
+  assert.deepEqual(got, { registry: "https://forge.example.org/api/packages/novox/npm/" });
+  assert.deepEqual(r.asked, ["http://forge.example.org:3000/api/packages/novox/npm/@novox%2fmesh-sdk"]);
+  // The forge's own scheme is taken too, with its port.
+  const plain = await npmRegistry("http://forge.example.org:3000",
+    aRegistry(() => packument("http://forge.example.org:3000/api/packages/novox/npm/%40novox%2Fmesh-sdk/-/0.1.14/mesh-sdk-0.1.14.tgz")).get);
+  assert.deepEqual(plain, { registry: "http://forge.example.org:3000/api/packages/novox/npm/" });
+});
+
+// What the registry answers may come over plain HTTP, so it never decides where packages come from: only
+// the forge's own host, at https or the forge's scheme, under the npm path, with nothing after it
+// (review of mesh-catalog #275).
+test("a registry that does not give an address the lab takes leaves the forge's, and the reason is said", async () => {
+  const sdk = "/api/packages/novox/npm/%40novox%2Fmesh-sdk/-/0.1.14/mesh-sdk-0.1.14.tgz";
+  for (const [answer, why] of [
+    [() => new Response("not found", { status: 404 }), /it answered 404/],
+    [() => new Response("{}", { status: 200 }), /no tarball/],
+    [() => { throw new Error("connect ECONNREFUSED"); }, /ECONNREFUSED/],
+    [() => packument(`https://forge.example.org/elsewhere/mesh-sdk-0.1.14.tgz`), /not under \/api\/packages\/novox\/npm\//],
+    [() => packument(`https://attacker.example.net${sdk}`), /attacker\.example\.net, which is not the forge's host novox\.internal/],
+    [() => packument(`https://attacker.example.net/x?u=https://novox.internal${sdk}`), /query or a fragment/],
+    [() => packument(`https://novox.internal${sdk}?x=1`), /query or a fragment/],
+    [() => packument(`https://novox.internal${sdk}#frag`), /query or a fragment/],
+    [() => packument(`https://attacker.example.net/#https://novox.internal${sdk}`), /query or a fragment/],
+    [() => packument(`ftp://novox.internal${sdk}`), /ftp: — neither https: nor the forge's http:/],
+    [() => packument(`file:///api/packages/novox/npm/x.tgz`), /file: — neither/],
+    [() => packument(`https://novox.internal/api/packages/novox/npm/\nregistry=https://attacker.example.net/\nallow-remote=all`), /whitespace or a control character/],
+    [() => packument(`https://user:pw@novox.internal${sdk}`), /user or password/],
+  ] as [() => Response, RegExp][]) {
+    const got = await npmRegistry("http://novox.internal:20000", aRegistry(answer).get);
+    assert.equal(got.registry, "http://novox.internal:20000/api/packages/novox/npm/", String(why));
+    assert.match(got.problem ?? "", /did not give an address the lab takes/);
+    assert.match(got.problem ?? "", why);
+  }
+});
+
+test("a run's npm configuration names the registry it was given, and the forge's own address only without one", () => {
+  const given = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, undefined,
+    "https://forge.example.org/api/packages/novox/npm/");
+  assert.match(given, / '@novox:registry=https:\/\/forge\.example\.org\/api\/packages\/novox\/npm\/' > '\/work\/run-x\/\.npmrc'\n/);
+  assert.doesNotMatch(given, /@novox:registry=http:\/\/novox\.internal/);
+  const none = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {});
+  assert.match(none, /'@novox:registry=http:\/\/novox\.internal:20000\/api\/packages\/novox\/npm\/'/);
+});

modules/lab/tools/index.ts

@@ -8,7 +8,7 @@ import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
-  checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start,
+  checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, readStatus, REPOSITORIES, running, start,
   stop, tail, writeCredential,
 } from "./runs.js";
 
@@ -42,6 +42,9 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
           spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" });
           check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" });
           problems = setupProblems(forge, `${dir}/setup`, credentialOf(forge));
+          // And the address a run names the package registry by (npmRegistry): said when the registry does not say it.
+          const registry = await npmRegistry(forge);
+          if (registry.problem) problems.push(registry.problem);
         } finally {
           rmSync(dir, { recursive: true, force: true });
         }
@@ -87,7 +90,9 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
         if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` };
         const busy = running(work);
         if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy };
-        return { started: true, run: start(work, forge, tests, refs, credentialOf(forge)) };
+        const registry = await npmRegistry(forge);
+        return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry),
+          registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) };
       },
     },
     {

modules/lab/tools/runs.ts

@@ -60,6 +60,70 @@ export function installStep(root: string, build: boolean): string {
 fi`;
 }
 
+/** Where the forge serves the mesh's npm packages, under its own address (gitea's npm registry for the novox org). */
+export const NPM_PATH = "/api/packages/novox/npm/";
+
+/**
+ * The address a run names the @novox registry by: the one the registry names itself by, when that is the
+ * forge's own host (novox/hq issue 624).
+ *
+ * **npm 12 installs a locked tarball only from the registry it is configured with.** Its `allow-remote`
+ * defaults to `none`, and a lock file's `resolved` URL counts as the registry's only when its origin and
+ * path are the configured scope registry's; any other is a "remote" tarball and refused (EALLOWREMOTE).
+ * The forge writes every tarball URL — in what it answers, and so in every lock file made against it — at
+ * its own public address, whichever address it was asked at. So the lab asks the registry, at the forge,
+ * for the SDK's newest tarball and names the registry by the origin of that URL.
+ *
+ * **What the registry answers is not trusted for where packages come from** (review of mesh-catalog #275):
+ * it may be fetched over plain HTTP. The tarball URL is parsed, and taken only when it is https (or the
+ * forge's own scheme), its path is under the npm path, it carries no query or fragment, and its host is
+ * the forge's own host. The registry is then built from
+ * the parsed origin and the npm path alone, never from the string answered, so nothing the registry says
+ * reaches the run's npm configuration but a scheme, a host and a port. Anything else falls back to the
+ * forge's own address and says why, so a run still starts and lab_check names it.
+ *
+ * So the lab's forge setting is the address the forge names itself by (its public https name), not a
+ * second name for it: a forge set as another name of the same server is refused here, named by lab_check,
+ * and its runs stop at the first locked @novox package.
+ */
+export async function npmRegistry(forge: string, get: typeof fetch = fetch): Promise<{ registry: string; problem?: string }> {
+  const asked = `${forge.replace(/\/+$/, "")}${NPM_PATH}`;
+  const url = `${asked}@novox%2fmesh-sdk`;
+  const fallback = (why: string) => ({
+    registry: asked,
+    problem: `the package registry at ${url} did not give an address the lab takes (${why}), so a run names it ${asked}, and npm 12 refuses a locked @novox tarball at any other`,
+  });
+  let own: URL;
+  try {
+    own = new URL(forge);
+  } catch {
+    return fallback(`the forge ${JSON.stringify(forge)} is not a URL`);
+  }
+  try {
+    const res = await get(url, { signal: AbortSignal.timeout(15_000) });
+    if (!res.ok) return fallback(`it answered ${res.status}`);
+    const doc = await res.json() as { "dist-tags"?: { latest?: string }; versions?: Record<string, { dist?: { tarball?: unknown } }> };
+    const latest = doc["dist-tags"]?.latest ?? "";
+    const tarball = doc.versions?.[latest]?.dist?.tarball;
+    if (typeof tarball !== "string" || !tarball) return fallback("no tarball for its latest version");
+    if (/[\s\u0000-\u001f\u007f]/.test(tarball)) return fallback("its tarball URL holds whitespace or a control character");
+    let t: URL;
+    try {
+      t = new URL(tarball);
+    } catch {
+      return fallback("its tarball is not a URL");
+    }
+    if (t.protocol !== "https:" && t.protocol !== own.protocol) return fallback(`its tarball is ${t.protocol} — neither https: nor the forge's ${own.protocol}`);
+    if (t.username || t.password) return fallback("its tarball URL carries a user or password");
+    if (t.search || t.hash) return fallback("its tarball URL carries a query or a fragment");
+    if (!t.pathname.startsWith(NPM_PATH)) return fallback(`its tarball is not under ${NPM_PATH}`);
+    if (t.hostname !== own.hostname) return fallback(`its tarball is at ${t.hostname}, which is not the forge's host ${own.hostname}`);
+    return { registry: `${t.origin}${NPM_PATH}` };
+  } catch (e) {
+    return fallback(e instanceof Error ? e.message : String(e));
+  }
+}
+
 /**
  * The forge credential a run's clones may present, as a git credential-store line —
  * scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none.
@@ -240,7 +304,7 @@ const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`;
  * at this run's own tree, so what is built and claimed is exactly what was checked out.
  */
 export function script(work: string, id: string, forge: string, tests: string[], refs: Record<string, string>,
-  credential?: string): string {
+  credential?: string, registry = `${forge}${NPM_PATH}`): string {
   const dir = runDir(work, id);
   const setState = (state: string) =>
     `node -e ${shellQuote(
@@ -266,9 +330,9 @@ node -e ${shellQuote(
     `const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`,
   )}
 ${setState("building")}
-# The @novox scope resolves from the mesh's own package registry on the forge, as the builder
-# resolves it; nothing else is asked of it.
-printf '%s\n' ${shellQuote(`@novox:registry=${forge}/api/packages/novox/npm/`)} > ${shellQuote(join(dir, ".npmrc"))}
+# The @novox scope resolves from the mesh's own package registry on the forge, named by the address it
+# names itself by (npmRegistry), which is the address its lock files hold; nothing else is asked of it.
+printf '%s\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))}
 export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))}
 ${NPM_REPOSITORIES.map((repo) => installStep(join(dir, repo), repo !== "mesh-lab")).join("\n")}
 mkdir -p ${shellQuote(bin)}
@@ -294,7 +358,7 @@ node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join("
 
 /** start begins a run and returns at once with its status. */
 export function start(work: string, forge: string, tests: string[], refs: Record<string, string>,
-  credential?: string): RunStatus {
+  credential?: string, registry?: string): RunStatus {
   const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`;
   const dir = runDir(work, id);
   mkdirSync(dir, { recursive: true });
@@ -303,7 +367,7 @@ export function start(work: string, forge: string, tests: string[], refs: Record
   // What a crashed or rebooted run left behind goes before this run writes its own.
   cleanCredentials(work);
   const credentials = writeCredential(credentialPath(work, id), credential);
-  writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials), { mode: 0o700 });
+  writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials, registry), { mode: 0o700 });
 
   // The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a
   // run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172).