CAT-LAB1: the lab clones through the forge account, and holds no registry secret

npm-package-registry is form none, so the registry pair the lab's clones of
mesh-tools-go used is no longer minted. The lab requires forge-account as the
build modules do; its clones go over ssh through a git and ssh configuration the
mesh writes from that binding. The secret, contributes and MESH_LAB_FORGE_SECRET go.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
2026-11-04 15:39:17 +01:00
parent af280050d6
commit e40ace34a6
10 changed files with 190 additions and 203 deletions
+11 -1
View File
@@ -1,6 +1,16 @@
# lab
The `lab` module of the mesh catalogue: installs the package `git`; installs the package `make`; installs the package `python`; installs the package `file`; installs the package `iproute2`; installs the package `incus`; requires `go-toolchain` and `node-toolchain`, so the `go` module and the `nodejs` module are assigned to a machine running it (novox/hq ADR 0477 clause 4).
The `lab` module of the mesh catalogue: installs the package `git`; installs the package `openssh`; installs the package `make`; installs the package `python`; installs the package `file`; installs the package `iproute2`; installs the package `incus`; requires `go-toolchain` and `node-toolchain`, so the `go` module and the `nodejs` module are assigned to a machine running it (novox/hq ADR 0477 clause 4).
## How its runs clone
A run clones the mesh's repositories, private ones too (mesh-tools-go), as the build modules do: the lab requires
`forge-account`, the forge's provider registers the account's key (`~/.ssh/mesh_ed25519`) for it, and the mesh writes
`forge.gitconfig`, `forge.ssh` and `forge.known_hosts` into the lab's state from that binding. Every clone reads
`forge.gitconfig` (`git -c include.path=...`), which sends the forge's address, as the lab's `forge` setting or the
forge itself names it, over ssh to the forge with that key and only the forge's host key. No password is bound,
sealed or written for a run. npm reads from `npm-package-registry` are anonymous (novox/hq ADR 0346): the lab
contributes nothing to it and holds no secret of it; its binding gives the registry's public root only.
## Files it does not know
+5
View File
@@ -0,0 +1,5 @@
[url "ssh://${bound:forge-account:ssh_alias}/"]
insteadOf = ${bound:forge-account:https_base}
insteadOf = ${config:forge}/
[core]
sshCommand = ssh -F ${dir:state}/forge.ssh
+1
View File
@@ -0,0 +1 @@
${bound:forge-account:ssh_alias} ${bound:forge-account:host_key}
+10
View File
@@ -0,0 +1,10 @@
Host ${bound:forge-account:ssh_alias}
HostName ${bound:forge-account:ssh_host}
Port ${bound:forge-account:ssh_port}
HostKeyAlias ${bound:forge-account:ssh_alias}
User git
IdentityFile ~/.ssh/mesh_ed25519
IdentitiesOnly yes
UserKnownHostsFile ${dir:state}/forge.known_hosts
StrictHostKeyChecking yes
BatchMode yes
+1 -1
View File
@@ -1,3 +1,3 @@
MESH_LAB_FORGE=${config:forge}
MESH_LAB_FORGE_BINDING=${dir:state}/package-registry.json
MESH_LAB_FORGE_SECRET=${dir:state}/package-registry.secret
MESH_LAB_FORGE_GIT=${dir:state}/forge.gitconfig
+28 -9
View File
@@ -7,20 +7,13 @@
],
"requires": [
"npm-package-registry",
"forge-account",
"go-toolchain",
"node-toolchain"
],
"contributes": {
"npm-package-registry": {
"access": "read"
}
},
"binds": {
"npm-package-registry": "${dir:state}/package-registry.json"
},
"secrets": {
"npm-package-registry": "${dir:state}/package-registry.secret"
},
"data": {
"own": [
{
@@ -61,11 +54,37 @@
"mode": "0600",
"template": "files/lab.env"
},
{
"id": "forge-git",
"type": "file",
"path": "${dir:state}/forge.gitconfig",
"mode": "0600",
"template": "files/forge.gitconfig"
},
{
"id": "forge-ssh",
"type": "file",
"path": "${dir:state}/forge.ssh",
"mode": "0600",
"template": "files/forge.ssh"
},
{
"id": "forge-known-hosts",
"type": "file",
"path": "${dir:state}/forge.known_hosts",
"mode": "0644",
"template": "files/forge.known_hosts"
},
{
"id": "git",
"type": "package",
"package": "git"
},
{
"id": "openssh",
"type": "package",
"package": "openssh"
},
{
"id": "make",
"type": "package",
@@ -108,7 +127,7 @@
"MESH_LAB_WORK": "${dir:work}",
"MESH_LAB_ENV_FILE": "${dir:state}/lab.env",
"MESH_LAB_FORGE_BINDING": "${dir:state}/package-registry.json",
"MESH_LAB_FORGE_SECRET": "${dir:state}/package-registry.secret"
"MESH_LAB_FORGE_GIT": "${dir:state}/forge.gitconfig"
}
},
{
+3 -3
View File
@@ -60,10 +60,10 @@ test("no run starts while the base image is rebuilt", () => {
});
test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => {
const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials");
assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/);
const s = baseScript("/work/base-x", "https://forge.invalid", "/state/forge.gitconfig");
assert.match(s, /git -c include\.path='\/state\/forge\.gitconfig' clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/);
assert.match(s, /src\/cli.ts base build/);
assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone");
assert.doesNotMatch(s, /credential|^trap |rm -f '\/state/m, "the forge account's configuration is the mesh's, never removed by a rebuild");
});
test("a rebuild is known by its status, and is not a run", () => {
+92 -97
View File
@@ -5,7 +5,7 @@ import { spawnSync } from "node:child_process";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { installPlan, installStep, NPM_REPOSITORIES, outcomeOf, readStatus, REPOSITORIES, script } from "../tools/runs.ts";
import { baseScript, installPlan, installStep, NPM_REPOSITORIES, outcomeOf, readStatus, REPOSITORIES, script } from "../tools/runs.ts";
// Defends novox/hq issue 624: lab_run ran `npm ci` at the root of every checkout it installs, and
// mesh-tools keeps its package in node-tools/ with no lock file at its root, so every run failed at its
@@ -100,101 +100,109 @@ test("a run checks out mesh-tools-go and says where its checkouts are", () => {
});
// Defends novox/hq issue 624, its second half: mesh-tools-go is private, and the lab cloned over HTTP with
// no credential ("could not read Username"). A run clones with the forge credential the mesh binds and
// seals to the lab, through git's credential store, and the secret is in no script, URL or argument.
import { credentialArgs, credentialPath, forgeCredential } from "../tools/runs.ts";
// no credential ("could not read Username"). The lab clones as the build modules do (issue 1966): it requires
// the forge account, the mesh writes the account's git and ssh configuration into the lab's state, and a run's
// clones go through it over ssh with the account's key. No password is bound, sealed or written for a run.
import { cleanCredentials, forgeArgs, forgeConfig } from "../tools/runs.ts";
function aBinding(as: string, secret: string): { binding: string; secret: string } {
const dir = mkdtempSync(join(tmpdir(), "lab-binding-"));
writeFileSync(join(dir, "package-registry.json"), JSON.stringify({ provision: "npm-package-registry", at: "10.42.0.1", as,
serves: { scheme: "http", port: 3000 } }));
writeFileSync(join(dir, "package-registry.secret"), `${secret}\n`);
return { binding: join(dir, "package-registry.json"), secret: join(dir, "package-registry.secret") };
const labDir = join(import.meta.dirname, "..");
/** The lab's forge files as the mesh writes them for a bound forge account, into a state directory of its own. */
function renderedForgeFiles(forge: string): string {
const state = mkdtempSync(join(tmpdir(), "lab-forge-"));
const values: Record<string, string> = {
ssh_alias: "mesh-forge", ssh_host: "forge.invalid", ssh_port: "222", host_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOn",
https_base: "https://git.example.org/",
};
for (const f of ["forge.gitconfig", "forge.ssh", "forge.known_hosts"]) {
const text = readFileSync(join(labDir, "files", f), "utf8")
.replace(/\$\{bound:forge-account:([a-z_]+)\}/g, (_, k: string) => values[k] ?? `<unknown ${k}>`)
.replace(/\$\{config:forge\}/g, forge).replace(/\$\{dir:state\}/g, state);
writeFileSync(join(state, f), text);
}
return state;
}
test("the forge credential is the bound account and its sealed secret, for the forge's own origin", () => {
const b = aBinding("lab-shanks", "s3cret-of-forty-characters");
assert.equal(forgeCredential("http://novox.internal:20000", b.binding, b.secret),
"http://lab-shanks:s3cret-of-forty-characters@novox.internal:20000");
assert.equal(forgeCredential("http://novox.internal:20000/", b.binding, ""), undefined);
assert.equal(forgeCredential("", b.binding, b.secret), undefined);
assert.equal(forgeCredential("http://novox.internal:20000", "/nowhere", b.secret), undefined);
});
test("a run clones through git's credential store, and its script carries no secret", () => {
const b = aBinding("lab-shanks", "s3cret-of-forty-characters");
const credential = forgeCredential("http://novox.internal:20000", b.binding, b.secret);
const path = credentialPath("/work", "run-x");
const s = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, path);
assert.ok(credential);
assert.doesNotMatch(s, /s3cret/);
assert.match(s, /git -c credential\.helper= -c 'credential\.helper=store --file=\/work\/\.run-x\.git-credentials' clone --quiet --depth 50 --branch 'main' 'http:\/\/novox\.internal:20000\/novox\/mesh-tools-go\.git'/);
assert.match(s, /export GIT_TERMINAL_PROMPT=0/);
assert.equal(path.startsWith("/work/run-x/"), false, "the credential is kept out of the run's own directory");
assert.equal(credentialArgs(undefined), "");
});
// Held against git itself: the credential in the store is offered to the forge's origin when it asks, and
// a clone without it fails at once rather than prompting.
test("git presents the stored credential to the forge, and only when one is given", () => {
const dir = mkdtempSync(join(tmpdir(), "lab-git-"));
const store = join(dir, "git-credentials");
writeFileSync(store, "http://lab-shanks:pw@forge.invalid:20000\n", { mode: 0o600 });
const fill = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=forge.invalid:20000\\n\\n' | git ${credentialArgs(store)}credential fill`],
{ encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } });
assert.match(fill.stdout, /username=lab-shanks/);
assert.match(fill.stdout, /password=pw/);
const other = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=elsewhere.invalid\\n\\n' | git ${credentialArgs(store)}credential fill`],
{ encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } });
assert.doesNotMatch(other.stdout, /password=pw/);
rmSync(dir, { recursive: true, force: true });
});
// Review 196 on mesh-catalog #272: the credential file outlived a stop, because SIGTERM to the run's
// process group killed the wrapper before its cleanup. It is now removed right after the clones, on
// every way the script ends, and at the next run's start and in lab_check for what a crash left.
import { cleanCredentials, credentialGuard } from "../tools/runs.ts";
test("a stopped script removes the credential, whatever ends it", () => {
for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"] as const) {
const dir = mkdtempSync(join(tmpdir(), "lab-guard-"));
const file = join(dir, ".run-x.git-credentials");
writeFileSync(file, "http://u:p@h\n", { mode: 0o600 });
const ran = spawnSync("bash", ["-c", `${credentialGuard(file)}\nkill -${signal.slice(3)} $$\nsleep 5`], { encoding: "utf8", timeout: 10_000 });
assert.notEqual(ran.status, 0, `${signal}: the script was not stopped`);
assert.equal(existsSync(file), false, `${signal} left the credential on disk`);
rmSync(dir, { recursive: true, force: true });
test("the lab requires the forge account for its clones, and holds no registry credential", () => {
const m = JSON.parse(readFileSync(join(labDir, "module.json"), "utf8")) as {
requires: string[]; contributes?: unknown; secrets?: unknown; resources: { id: string; path?: string; template?: string; package?: string }[];
build: { artifacts: { name: string; env?: Record<string, string> }[] };
};
assert.ok(m.requires.includes("forge-account") && m.requires.includes("npm-package-registry"));
assert.equal(m.contributes, undefined, "npm-package-registry reads are anonymous: nothing is contributed");
assert.equal(m.secrets, undefined, "no registry secret is sealed to the lab");
const env = m.build.artifacts.find((a) => a.name === "code")?.env ?? {};
assert.equal(env.MESH_LAB_FORGE_SECRET, undefined);
assert.equal(env.MESH_LAB_FORGE_GIT, m.resources.find((r) => r.id === "forge-git")?.path);
assert.match(readFileSync(join(labDir, "files", "lab.env"), "utf8"), /^MESH_LAB_FORGE_GIT=\$\{dir:state\}\/forge\.gitconfig$/m);
assert.doesNotMatch(readFileSync(join(labDir, "files", "lab.env"), "utf8"), /SECRET/);
assert.ok(m.resources.some((r) => r.package === "openssh"), "the clones need ssh");
for (const id of ["forge-git", "forge-ssh", "forge-known-hosts"]) {
const r = m.resources.find((x) => x.id === id);
assert.match(readFileSync(join(labDir, String(r?.template)), "utf8"), /\$\{bound:forge-account:/, id);
}
});
test("the script removes the credential as soon as its clones are done", () => {
const s = script("/work", "run-x", "http://forge", ["t.test.ts"], {}, "/work/.run-x.git-credentials");
const lastClone = s.lastIndexOf("clone --quiet");
const removed = s.indexOf("rm -f '/work/.run-x.git-credentials'", lastClone);
const building = s.indexOf("npm ci");
assert.ok(lastClone > 0 && removed > lastClone && removed < building, "not removed between the clones and the build");
assert.match(s, /trap 'rm -f '\\''\/work\/\.run-x\.git-credentials'\\''' EXIT/);
test("a run clones through the forge account's git configuration, and its script holds no credential", () => {
const s = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, "/state/forge.gitconfig");
assert.match(s, /git -c include\.path='\/state\/forge\.gitconfig' clone --quiet --depth 50 --branch 'main' 'http:\/\/novox\.internal:20000\/novox\/mesh-tools-go\.git'/);
assert.match(s, /export GIT_TERMINAL_PROMPT=0/);
assert.doesNotMatch(s, /credential|^trap /m);
assert.match(baseScript("/work/base-x", "http://forge", "/state/forge.gitconfig"), /git -c include\.path='\/state\/forge\.gitconfig' clone/);
assert.equal(forgeArgs(undefined), "");
});
test("credentials a crashed run left are removed, but not the running one's", () => {
test("the forge account's configuration is taken only when the mesh wrote it", () => {
const state = renderedForgeFiles("http://novox.internal:20000");
assert.equal(forgeConfig(join(state, "forge.gitconfig")), join(state, "forge.gitconfig"));
assert.equal(forgeConfig(join(state, "nowhere")), undefined);
assert.equal(forgeConfig(undefined), undefined);
writeFileSync(join(state, "empty"), "");
assert.equal(forgeConfig(join(state, "empty")), undefined);
rmSync(state, { recursive: true, force: true });
});
function withoutSsh(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
const { GIT_SSH_COMMAND: _c, GIT_SSH: _s, ...rest } = env;
return rest;
}
// Held against git and ssh themselves: a clone of the forge's address, as the lab's setting names it or as the
// forge names itself, goes over ssh to the forge account's host with the account's key and the forge's host key.
test("git sends a clone of the forge's address over ssh, as the forge account's configuration says", () => {
const state = renderedForgeFiles("http://novox.internal:20000");
const bin = mkdtempSync(join(tmpdir(), "lab-ssh-"));
writeFileSync(join(bin, "ssh"), `#!/bin/sh\nprintf '%s\\n' "$@" >> ${JSON.stringify(join(bin, "asked"))}\nexit 1\n`, { mode: 0o755 });
for (const url of ["http://novox.internal:20000/novox/mesh-tools-go.git", "https://git.example.org/novox/mesh-tools-go.git"]) {
rmSync(join(bin, "asked"), { force: true });
spawnSync("sh", ["-c", `git ${forgeArgs(join(state, "forge.gitconfig"))}ls-remote ${url}`],
{ encoding: "utf8", env: { ...withoutSsh(process.env), PATH: `${bin}:${process.env.PATH}`, GIT_TERMINAL_PROMPT: "0" } });
const asked = readFileSync(join(bin, "asked"), "utf8").split("\n");
assert.ok(asked.includes("-F") && asked.includes(join(state, "forge.ssh")), `${url}: ${asked.join(" ")}`);
assert.ok(asked.includes("mesh-forge"), url);
assert.ok(asked.some((a) => a.includes("novox/mesh-tools-go.git")), url);
}
const g = spawnSync("ssh", ["-G", "-F", join(state, "forge.ssh"), "mesh-forge"], { encoding: "utf8" });
for (const line of ["hostname forge.invalid", "port 222", "user git", "identitiesonly yes", "batchmode yes",
"stricthostkeychecking true", `userknownhostsfile ${join(state, "forge.known_hosts")}`, "hostkeyalias mesh-forge"]) {
assert.ok(g.stdout.split("\n").includes(line), `ssh -G lacks ${line}`);
}
assert.match(g.stdout, /^identityfile ~\/\.ssh\/mesh_ed25519$/m);
assert.equal(readFileSync(join(state, "forge.known_hosts"), "utf8"), "mesh-forge ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOn\n");
rmSync(state, { recursive: true, force: true });
rmSync(bin, { recursive: true, force: true });
});
// What a run of before left on disk, its credential line, is removed at the next run's start and in lab_check.
test("credentials an earlier run left are removed", () => {
const work = mkdtempSync(join(tmpdir(), "lab-work-"));
for (const n of [".run-a.git-credentials", ".run-b.git-credentials", "run-a", "other"]) writeFileSync(join(work, n), "x");
const removed = cleanCredentials(work, join(work, ".run-b.git-credentials"));
assert.deepEqual(removed, [join(work, ".run-a.git-credentials")]);
assert.ok(existsSync(join(work, ".run-b.git-credentials")) && existsSync(join(work, "run-a")) && existsSync(join(work, "other")));
for (const n of [".run-a.git-credentials", ".base-b.git-credentials", "run-a", "other"]) writeFileSync(join(work, n), "x");
const removed = cleanCredentials(work).sort();
assert.deepEqual(removed, [join(work, ".base-b.git-credentials"), join(work, ".run-a.git-credentials")]);
assert.ok(existsSync(join(work, "run-a")) && existsSync(join(work, "other")));
rmSync(work, { recursive: true, force: true });
});
// The lab asks the forge for read only (review 196): its credential is for clones of branches nobody has
// reviewed yet, and must not be able to publish the mesh's packages.
test("the lab asks the package registry for read only", () => {
const m = JSON.parse(readFileSync(join(import.meta.dirname, "..", "module.json"), "utf8")) as {
requires: unknown[]; contributes?: Record<string, Record<string, unknown>>;
};
assert.ok(m.requires.includes("npm-package-registry"));
assert.equal(m.contributes?.["npm-package-registry"]?.access, "read");
});
// lab_check failed for ever on "destination path '<work>/check' already exists": it cloned into one fixed
// path and ignored whether removing the last one worked (novox/hq issue 624). Each check now works in a
// fresh directory, and what earlier checks left is cleared or named.
@@ -239,19 +247,6 @@ test("leftover checks are cleared, whoever runs the check", () => {
rmSync(work, { recursive: true, force: true });
});
// The lab's tools read the bound credential, so the runtime is given that file by the tools bundle's env
// (ADR 0192), as gitea and umami are given theirs: the runtime's account may then read exactly it, and no
// other credential changes hands (issue 624).
test("the lab's tools bundle names its bound credential and binding in its env", () => {
const m = JSON.parse(readFileSync(join(import.meta.dirname, "..", "module.json"), "utf8")) as {
secrets: Record<string, string>; binds: Record<string, string>;
build: { artifacts: { name: string; env?: Record<string, string> }[] };
};
const env = m.build.artifacts.find((a) => a.name === "code")?.env ?? {};
assert.equal(env.MESH_LAB_FORGE_SECRET, m.secrets["npm-package-registry"]);
assert.equal(env.MESH_LAB_FORGE_BINDING, m.binds["npm-package-registry"]);
});
// The next setup failure of novox/hq issue 624 (run-2026-10-13T13-15-26-634Z on shanks): npm 12 refused
// mesh-tools' locked @novox/mesh-sdk with EALLOWREMOTE. The lab named the registry by its forge setting's
// address (http://novox.internal:20000/…), the lock file holds the address the forge writes every tarball
+15 -23
View File
@@ -4,12 +4,10 @@
import { spawnSync } from "node:child_process";
import { readFileSync } from "node:fs";
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { rmSync } from "node:fs";
import {
checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start,
stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun,
checkDir, cleanCredentials, clearChecks, forgeArgs, forgeConfig, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, start,
stop, tail, baseBuildLog, listBaseBuilds, startBaseBuild, launchRun,
launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem,
} from "./runs.js";
@@ -19,7 +17,6 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
// ADR 0192): read from the env-file the mesh fills, at each call, so a changed setting is used
// without restarting the runtime. MESH_LAB_FORGE itself still wins, for a hand-run instance.
const forgeOf = (): string => (env.MESH_LAB_FORGE ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE")).replace(/\/+$/, "");
// The forge credential the mesh binds and seals to the lab (novox/hq issue 624), read at each call.
// The registry's public root the forge serves in the lab's binding (novox/hq ADR 0346), read at each call.
const bindingOf = (): string | undefined =>
env.MESH_LAB_FORGE_BINDING ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING") || undefined);
@@ -28,9 +25,9 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
const registry = await npmRegistry(forge, fetch, root.root);
return registry.problem && root.problem ? { ...registry, problem: `${registry.problem}; ${root.problem}` } : registry;
};
const credentialOf = (forge: string): string | undefined => forgeCredential(forge,
env.MESH_LAB_FORGE_BINDING ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING"),
env.MESH_LAB_FORGE_SECRET ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_SECRET"));
// The forge account's git configuration the mesh writes for the lab's clones (forgeConfig), read at each call.
const forgeGitOf = (): string | undefined =>
forgeConfig(env.MESH_LAB_FORGE_GIT ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_GIT") || undefined));
return [
{
name: "lab_check",
@@ -50,7 +47,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
if (clone.status !== 0) return { ok: false, output: [String(clone.stderr).trim(), ...leftovers].join("\n") };
spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" });
check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" });
problems = setupProblems(forge, `${dir}/setup`, credentialOf(forge));
problems = setupProblems(forge, `${dir}/setup`, forgeGitOf());
// And the address a run names the package registry by (npmRegistry): said when the registry does not say it.
const registry = await registryOf(forge);
if (registry.problem) problems.push(registry.problem);
@@ -60,10 +57,8 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
// **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every
// repository it clones answers on the forge, and every one it installs has a lock file where the run
// installs from.
// A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in
// use by a running run or rebuild is not.
const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id);
const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id)));
// A credential line a run or rebuild of before the forge account left behind is removed, and said.
const left = cleanCredentials(work);
const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : "";
// **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot
// be read is a problem here, though lab_run refuses only the first.
@@ -106,7 +101,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
// **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698,
// review 256): the registry asked first, then the check and the start with nothing awaited between.
const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() },
(registry) => start(work, forge, tests, refs, credentialOf(forge), registry));
(registry) => start(work, forge, tests, refs, forgeGitOf(), registry));
if (!launched.started) return launched;
return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry,
...(launched.problem ? { problem: launched.problem } : {}) };
@@ -137,7 +132,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
// Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with
// nothing awaited between (review 256).
const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before },
(registry) => startBaseBuild(work, forge, credentialOf(forge), registry));
(registry) => startBaseBuild(work, forge, forgeGitOf(), registry));
if (launched.started === false) return { ...reading, started: false, reason: launched.reason };
return { ...reading, started: true, build: launched.job };
},
@@ -174,14 +169,12 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
* answer for, and a checkout it installs with nothing installable (installPlan). Empty when a run's setup
* would go through.
*/
export function setupProblems(forge: string, dir: string, credential?: string): string[] {
export function setupProblems(forge: string, dir: string, forgeGit?: string): string[] {
const problems: string[] = [];
spawnSync("rm", ["-rf", dir]);
// The credential a run would offer, in a file of its own outside the check's clones, removed after.
const keep = mkdtempSync(join(tmpdir(), "lab-check-"));
const file = writeCredential(join(keep, "git-credentials"), credential);
// The forge account's configuration a run's clones go through (forgeConfig).
const git = (args: string[]) =>
spawnSync("sh", ["-c", `git ${credentialArgs(file)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], {
spawnSync("sh", ["-c", `git ${forgeArgs(forgeGit)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], {
encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
});
try {
@@ -192,7 +185,7 @@ export function setupProblems(forge: string, dir: string, credential?: string):
const said = remote.stderr.trim().split("\n")[0] ?? "";
problems.push(`a run checks out ${repo}, and ${url} could not be read` +
(said ? `: ${said}` : ": no main branch") +
(credential ? "" : " — the lab holds no forge credential (its npm-package-registry binding and secret)"));
(forgeGit ? "" : " — the lab holds no forge account yet (its forge-account binding, written as forge.gitconfig in its state)"));
continue;
}
if (!NPM_REPOSITORIES.includes(repo)) continue;
@@ -209,7 +202,6 @@ export function setupProblems(forge: string, dir: string, credential?: string):
}
}
} finally {
rmSync(keep, { recursive: true, force: true });
spawnSync("rm", ["-rf", dir]);
}
return problems;
+24 -69
View File
@@ -163,68 +163,31 @@ export async function npmRegistry(forge: string, get: typeof fetch = fetch, publ
}
/**
* The forge credential a run's clones may present, as a git credential-store line —
* scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none.
* The forge account's git configuration a run's clones go through, or undefined when the mesh has not written it.
*
* **The mesh's credential for the forge, the way the build modules hold it** (novox/hq issue 624): the lab
* requires the package registry, the forge's provisioner binds it an account (`as`) and seals that
* account's password, and the forge that answers npm is the forge that hosts the repositories. Private
* repositories (mesh-tools-go) are cloned with it; public ones clone as before. Written to a file git reads
* through its credential store, never into a URL, an argument, the run's script or its log, and offered
* only to the forge's own scheme, host and port.
* **The mesh's account on the forge, the way the build modules hold it** (novox/hq issue 624, issue 1966): the lab
* requires `forge-account`, the forge's provider registers the account's key (`~/.ssh/mesh_ed25519`) for it, and
* the mesh writes into the lab's state a git configuration that sends every clone of the forge's address over ssh
* to the forge, with an ssh configuration that offers that key and takes only the forge's host key. Private
* repositories (mesh-tools-go) clone with it; no password is bound, sealed or written for a run.
*/
export function forgeCredential(forge: string, bindingFile: string, secretFile: string): string | undefined {
if (!forge || !bindingFile || !secretFile) return undefined;
let as = "";
let secret = "";
export function forgeConfig(file: string | undefined): string | undefined {
if (!file) return undefined;
try {
as = String((JSON.parse(readFileSync(bindingFile, "utf8")) as { as?: string }).as ?? "");
secret = readFileSync(secretFile, "utf8").trim();
return /^\s*insteadOf\s*=\s*\S/m.test(readFileSync(file, "utf8")) ? file : undefined;
} catch {
return undefined;
}
if (!as || !secret) return undefined;
let origin: URL;
try {
origin = new URL(forge);
} catch {
return undefined;
}
const line = new URL(`${origin.protocol}//${origin.host}`);
line.username = as;
line.password = secret;
return line.toString().replace(/\/$/, "");
}
/** Where a run keeps its forge credential: beside its directory, never in it, and removed when it ends. */
/** Where a run of before kept its forge credential line, beside its directory: removed wherever one is left. */
export function credentialPath(work: string, id: string): string {
return join(work, `.${id}.git-credentials`);
}
/** writeCredential puts a credential-store line where git reads it, readable by this account alone. */
export function writeCredential(path: string, line: string | undefined): string | undefined {
if (!line) return undefined;
writeFileSync(path, `${line}\n`, { mode: 0o600 });
return path;
}
/**
* credentialGuard is the shell that removes the credential however the script ends — its end, an error,
* or a stop's SIGTERM to the run's process group, which kills the wrapper before its own cleanup runs
* (review 196 on mesh-catalog #272). The script also removes it as soon as its clones are done.
*/
export function credentialGuard(path: string | undefined): string {
if (!path) return "";
const rm = `rm -f ${shellQuote(path)}`;
return `trap ${shellQuote(rm)} EXIT
trap ${shellQuote(`${rm}; exit 143`)} TERM
trap ${shellQuote(`${rm}; exit 130`)} INT
trap ${shellQuote(`${rm}; exit 129`)} HUP`;
}
/**
* cleanCredentials removes every run's and every base rebuild's credential file left in the work directory
* but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it
* but the ones named — what runs of before the forge account, a crash or a reboot left behind. Answers what it
* removed.
*/
export function cleanCredentials(work: string, keep?: string | string[]): string[] {
@@ -272,10 +235,10 @@ export function clearChecks(work: string): string[] {
return said;
}
/** The arguments that have git offer the credential in that file, and only that one. */
export function credentialArgs(path: string | undefined): string {
/** The arguments that have git read the forge account's configuration (forgeConfig) for one command. */
export function forgeArgs(path: string | undefined): string {
if (!path) return "";
return `-c credential.helper= -c ${shellQuote(`credential.helper=store --file=${path}`)} `;
return `-c include.path=${shellQuote(path)} `;
}
export interface RunStatus {
@@ -462,7 +425,7 @@ const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`;
* at this run's own tree, so what is built and claimed is exactly what was checked out.
*/
export function script(work: string, id: string, forge: string, tests: string[], refs: Record<string, string>,
credential?: string, registry = `${forge}${NPM_PATH}`): string {
forgeGit?: string, registry = `${forge}${NPM_PATH}`): string {
const dir = runDir(work, id);
const setState = (state: string) =>
`node -e ${shellQuote(
@@ -471,7 +434,7 @@ export function script(work: string, id: string, forge: string, tests: string[],
const clones = REPOSITORIES.map((repo) => {
const ref = refs[repo] ?? "main";
return [
`git ${credentialArgs(credential)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`,
`git ${forgeArgs(forgeGit)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`,
`echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`,
].join("\n");
}).join("\n");
@@ -479,11 +442,9 @@ export function script(work: string, id: string, forge: string, tests: string[],
return `set -euo pipefail
# A repository the forge will not show fails its clone at once, rather than waiting on a prompt.
export GIT_TERMINAL_PROMPT=0
${credentialGuard(credential)}
cd ${shellQuote(dir)}
${setState("checking-out")}
${clones}
${credential ? `# The forge credential is for the clones, and nothing after them runs with it on disk.\nrm -f ${shellQuote(credential)}` : ""}
node -e ${shellQuote(
`const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`,
)}
@@ -516,16 +477,15 @@ node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join("
/** start begins a run and returns at once with its status. */
export function start(work: string, forge: string, tests: string[], refs: Record<string, string>,
credential?: string, registry?: string): RunStatus {
forgeGit?: string, registry?: string): RunStatus {
const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`;
const dir = runDir(work, id);
mkdirSync(dir, { recursive: true });
const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs };
writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2));
// What a crashed or rebooted run left behind goes before this run writes its own.
// A credential line a run of before left behind goes before this run starts.
cleanCredentials(work);
const credentials = writeCredential(credentialPath(work, id), credential);
writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials, registry), { mode: 0o700 });
writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, forgeGit, registry), { mode: 0o700 });
// The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a
// run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172).
@@ -533,7 +493,6 @@ export function start(work: string, forge: string, tests: string[], refs: Record
node -e ${shellQuote(
`const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
)} "$code"
rm -f ${shellQuote(credentialPath(work, id))}
cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`;
const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
child.unref();
@@ -715,14 +674,12 @@ export function baseBuildLog(work: string, id: string, lines = 40): string {
* baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it,
* then its `base build`, which replaces the image and reads back that it works before publishing it.
*/
export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string {
export function baseScript(dir: string, forge: string, forgeGit?: string, registry = `${forge}${NPM_PATH}`): string {
const lab = join(dir, "mesh-lab");
return `set -euo pipefail
export GIT_TERMINAL_PROMPT=0
${credentialGuard(credential)}
cd ${shellQuote(dir)}
git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)}
${credential ? `rm -f ${shellQuote(credential)}` : ""}
git ${forgeArgs(forgeGit)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)}
echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)"
printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))}
export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))}
@@ -733,21 +690,19 @@ node --experimental-strip-types src/cli.ts base build
}
/** startBaseBuild rebuilds the base image in the background and answers at once with the build. */
export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild {
export function startBaseBuild(work: string, forge: string, forgeGit?: string, registry?: string): BaseBuild {
const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`;
const dir = join(work, id);
mkdirSync(dir, { recursive: true });
const build: BaseBuild = { id, state: "building", started: new Date().toISOString() };
writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
// What a crashed or rebooted run or rebuild left behind goes before this one writes its own.
// A credential line a run or rebuild of before left behind goes before this one starts.
cleanCredentials(work);
const credentials = writeCredential(credentialPath(work, id), credential);
writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 });
writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, forgeGit, registry), { mode: 0o700 });
const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$?
node -e ${shellQuote(
`const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
)} "$code"
rm -f ${shellQuote(credentialPath(work, id))}
rm -rf ${shellQuote(join(dir, "mesh-lab"))}`;
const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
child.unref();