From e40ace34a6a1d45def1588f11b45e2a648dc9f6a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 15:39:17 +0100 Subject: [PATCH] CAT-LAB1: the lab clones through the forge account, and holds no registry secret npm-package-registry is form none, so the registry pair the lab's clones of mesh-tools-go used is no longer minted. The lab requires forge-account as the build modules do; its clones go over ssh through a git and ssh configuration the mesh writes from that binding. The secret, contributes and MESH_LAB_FORGE_SECRET go. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- modules/lab/README.md | 12 +- modules/lab/files/forge.gitconfig | 5 + modules/lab/files/forge.known_hosts | 1 + modules/lab/files/forge.ssh | 10 ++ modules/lab/files/lab.env | 2 +- modules/lab/module.json | 37 ++++-- modules/lab/test/base.test.ts | 6 +- modules/lab/test/runs.test.ts | 189 ++++++++++++++-------------- modules/lab/tools/index.ts | 38 +++--- modules/lab/tools/runs.ts | 93 ++++---------- 10 files changed, 190 insertions(+), 203 deletions(-) create mode 100644 modules/lab/files/forge.gitconfig create mode 100644 modules/lab/files/forge.known_hosts create mode 100644 modules/lab/files/forge.ssh diff --git a/modules/lab/README.md b/modules/lab/README.md index 6ce0f8d5..87bb7bd9 100644 --- a/modules/lab/README.md +++ b/modules/lab/README.md @@ -1,6 +1,16 @@ # lab -The `lab` module of the mesh catalogue: installs the package `git`; installs the package `make`; installs the package `python`; installs the package `file`; installs the package `iproute2`; installs the package `incus`; requires `go-toolchain` and `node-toolchain`, so the `go` module and the `nodejs` module are assigned to a machine running it (novox/hq ADR 0477 clause 4). +The `lab` module of the mesh catalogue: installs the package `git`; installs the package `openssh`; installs the package `make`; installs the package `python`; installs the package `file`; installs the package `iproute2`; installs the package `incus`; requires `go-toolchain` and `node-toolchain`, so the `go` module and the `nodejs` module are assigned to a machine running it (novox/hq ADR 0477 clause 4). + +## How its runs clone + +A run clones the mesh's repositories, private ones too (mesh-tools-go), as the build modules do: the lab requires +`forge-account`, the forge's provider registers the account's key (`~/.ssh/mesh_ed25519`) for it, and the mesh writes +`forge.gitconfig`, `forge.ssh` and `forge.known_hosts` into the lab's state from that binding. Every clone reads +`forge.gitconfig` (`git -c include.path=...`), which sends the forge's address, as the lab's `forge` setting or the +forge itself names it, over ssh to the forge with that key and only the forge's host key. No password is bound, +sealed or written for a run. npm reads from `npm-package-registry` are anonymous (novox/hq ADR 0346): the lab +contributes nothing to it and holds no secret of it; its binding gives the registry's public root only. ## Files it does not know diff --git a/modules/lab/files/forge.gitconfig b/modules/lab/files/forge.gitconfig new file mode 100644 index 00000000..d61dc537 --- /dev/null +++ b/modules/lab/files/forge.gitconfig @@ -0,0 +1,5 @@ +[url "ssh://${bound:forge-account:ssh_alias}/"] + insteadOf = ${bound:forge-account:https_base} + insteadOf = ${config:forge}/ +[core] + sshCommand = ssh -F ${dir:state}/forge.ssh diff --git a/modules/lab/files/forge.known_hosts b/modules/lab/files/forge.known_hosts new file mode 100644 index 00000000..e0df6d81 --- /dev/null +++ b/modules/lab/files/forge.known_hosts @@ -0,0 +1 @@ +${bound:forge-account:ssh_alias} ${bound:forge-account:host_key} diff --git a/modules/lab/files/forge.ssh b/modules/lab/files/forge.ssh new file mode 100644 index 00000000..8723cbf2 --- /dev/null +++ b/modules/lab/files/forge.ssh @@ -0,0 +1,10 @@ +Host ${bound:forge-account:ssh_alias} + HostName ${bound:forge-account:ssh_host} + Port ${bound:forge-account:ssh_port} + HostKeyAlias ${bound:forge-account:ssh_alias} + User git + IdentityFile ~/.ssh/mesh_ed25519 + IdentitiesOnly yes + UserKnownHostsFile ${dir:state}/forge.known_hosts + StrictHostKeyChecking yes + BatchMode yes diff --git a/modules/lab/files/lab.env b/modules/lab/files/lab.env index 35aa1a29..6b0a8e77 100644 --- a/modules/lab/files/lab.env +++ b/modules/lab/files/lab.env @@ -1,3 +1,3 @@ MESH_LAB_FORGE=${config:forge} MESH_LAB_FORGE_BINDING=${dir:state}/package-registry.json -MESH_LAB_FORGE_SECRET=${dir:state}/package-registry.secret +MESH_LAB_FORGE_GIT=${dir:state}/forge.gitconfig diff --git a/modules/lab/module.json b/modules/lab/module.json index ef82b87c..ccd89740 100644 --- a/modules/lab/module.json +++ b/modules/lab/module.json @@ -7,20 +7,13 @@ ], "requires": [ "npm-package-registry", + "forge-account", "go-toolchain", "node-toolchain" ], - "contributes": { - "npm-package-registry": { - "access": "read" - } - }, "binds": { "npm-package-registry": "${dir:state}/package-registry.json" }, - "secrets": { - "npm-package-registry": "${dir:state}/package-registry.secret" - }, "data": { "own": [ { @@ -61,11 +54,37 @@ "mode": "0600", "template": "files/lab.env" }, + { + "id": "forge-git", + "type": "file", + "path": "${dir:state}/forge.gitconfig", + "mode": "0600", + "template": "files/forge.gitconfig" + }, + { + "id": "forge-ssh", + "type": "file", + "path": "${dir:state}/forge.ssh", + "mode": "0600", + "template": "files/forge.ssh" + }, + { + "id": "forge-known-hosts", + "type": "file", + "path": "${dir:state}/forge.known_hosts", + "mode": "0644", + "template": "files/forge.known_hosts" + }, { "id": "git", "type": "package", "package": "git" }, + { + "id": "openssh", + "type": "package", + "package": "openssh" + }, { "id": "make", "type": "package", @@ -108,7 +127,7 @@ "MESH_LAB_WORK": "${dir:work}", "MESH_LAB_ENV_FILE": "${dir:state}/lab.env", "MESH_LAB_FORGE_BINDING": "${dir:state}/package-registry.json", - "MESH_LAB_FORGE_SECRET": "${dir:state}/package-registry.secret" + "MESH_LAB_FORGE_GIT": "${dir:state}/forge.gitconfig" } }, { diff --git a/modules/lab/test/base.test.ts b/modules/lab/test/base.test.ts index 93a6f8ec..36d53331 100644 --- a/modules/lab/test/base.test.ts +++ b/modules/lab/test/base.test.ts @@ -60,10 +60,10 @@ test("no run starts while the base image is rebuilt", () => { }); test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => { - const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials"); - assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/); + const s = baseScript("/work/base-x", "https://forge.invalid", "/state/forge.gitconfig"); + assert.match(s, /git -c include\.path='\/state\/forge\.gitconfig' clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/); assert.match(s, /src\/cli.ts base build/); - assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone"); + assert.doesNotMatch(s, /credential|^trap |rm -f '\/state/m, "the forge account's configuration is the mesh's, never removed by a rebuild"); }); test("a rebuild is known by its status, and is not a run", () => { diff --git a/modules/lab/test/runs.test.ts b/modules/lab/test/runs.test.ts index b2071208..5383a3ea 100644 --- a/modules/lab/test/runs.test.ts +++ b/modules/lab/test/runs.test.ts @@ -5,7 +5,7 @@ import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { installPlan, installStep, NPM_REPOSITORIES, outcomeOf, readStatus, REPOSITORIES, script } from "../tools/runs.ts"; +import { baseScript, installPlan, installStep, NPM_REPOSITORIES, outcomeOf, readStatus, REPOSITORIES, script } from "../tools/runs.ts"; // Defends novox/hq issue 624: lab_run ran `npm ci` at the root of every checkout it installs, and // mesh-tools keeps its package in node-tools/ with no lock file at its root, so every run failed at its @@ -100,101 +100,109 @@ test("a run checks out mesh-tools-go and says where its checkouts are", () => { }); // Defends novox/hq issue 624, its second half: mesh-tools-go is private, and the lab cloned over HTTP with -// no credential ("could not read Username"). A run clones with the forge credential the mesh binds and -// seals to the lab, through git's credential store, and the secret is in no script, URL or argument. -import { credentialArgs, credentialPath, forgeCredential } from "../tools/runs.ts"; +// no credential ("could not read Username"). The lab clones as the build modules do (issue 1966): it requires +// the forge account, the mesh writes the account's git and ssh configuration into the lab's state, and a run's +// clones go through it over ssh with the account's key. No password is bound, sealed or written for a run. +import { cleanCredentials, forgeArgs, forgeConfig } from "../tools/runs.ts"; -function aBinding(as: string, secret: string): { binding: string; secret: string } { - const dir = mkdtempSync(join(tmpdir(), "lab-binding-")); - writeFileSync(join(dir, "package-registry.json"), JSON.stringify({ provision: "npm-package-registry", at: "10.42.0.1", as, - serves: { scheme: "http", port: 3000 } })); - writeFileSync(join(dir, "package-registry.secret"), `${secret}\n`); - return { binding: join(dir, "package-registry.json"), secret: join(dir, "package-registry.secret") }; +const labDir = join(import.meta.dirname, ".."); + +/** The lab's forge files as the mesh writes them for a bound forge account, into a state directory of its own. */ +function renderedForgeFiles(forge: string): string { + const state = mkdtempSync(join(tmpdir(), "lab-forge-")); + const values: Record = { + ssh_alias: "mesh-forge", ssh_host: "forge.invalid", ssh_port: "222", host_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOn", + https_base: "https://git.example.org/", + }; + for (const f of ["forge.gitconfig", "forge.ssh", "forge.known_hosts"]) { + const text = readFileSync(join(labDir, "files", f), "utf8") + .replace(/\$\{bound:forge-account:([a-z_]+)\}/g, (_, k: string) => values[k] ?? ``) + .replace(/\$\{config:forge\}/g, forge).replace(/\$\{dir:state\}/g, state); + writeFileSync(join(state, f), text); + } + return state; } -test("the forge credential is the bound account and its sealed secret, for the forge's own origin", () => { - const b = aBinding("lab-shanks", "s3cret-of-forty-characters"); - assert.equal(forgeCredential("http://novox.internal:20000", b.binding, b.secret), - "http://lab-shanks:s3cret-of-forty-characters@novox.internal:20000"); - assert.equal(forgeCredential("http://novox.internal:20000/", b.binding, ""), undefined); - assert.equal(forgeCredential("", b.binding, b.secret), undefined); - assert.equal(forgeCredential("http://novox.internal:20000", "/nowhere", b.secret), undefined); -}); - -test("a run clones through git's credential store, and its script carries no secret", () => { - const b = aBinding("lab-shanks", "s3cret-of-forty-characters"); - const credential = forgeCredential("http://novox.internal:20000", b.binding, b.secret); - const path = credentialPath("/work", "run-x"); - const s = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, path); - assert.ok(credential); - assert.doesNotMatch(s, /s3cret/); - assert.match(s, /git -c credential\.helper= -c 'credential\.helper=store --file=\/work\/\.run-x\.git-credentials' clone --quiet --depth 50 --branch 'main' 'http:\/\/novox\.internal:20000\/novox\/mesh-tools-go\.git'/); - assert.match(s, /export GIT_TERMINAL_PROMPT=0/); - assert.equal(path.startsWith("/work/run-x/"), false, "the credential is kept out of the run's own directory"); - assert.equal(credentialArgs(undefined), ""); -}); - -// Held against git itself: the credential in the store is offered to the forge's origin when it asks, and -// a clone without it fails at once rather than prompting. -test("git presents the stored credential to the forge, and only when one is given", () => { - const dir = mkdtempSync(join(tmpdir(), "lab-git-")); - const store = join(dir, "git-credentials"); - writeFileSync(store, "http://lab-shanks:pw@forge.invalid:20000\n", { mode: 0o600 }); - const fill = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=forge.invalid:20000\\n\\n' | git ${credentialArgs(store)}credential fill`], - { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); - assert.match(fill.stdout, /username=lab-shanks/); - assert.match(fill.stdout, /password=pw/); - const other = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=elsewhere.invalid\\n\\n' | git ${credentialArgs(store)}credential fill`], - { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); - assert.doesNotMatch(other.stdout, /password=pw/); - rmSync(dir, { recursive: true, force: true }); -}); - -// Review 196 on mesh-catalog #272: the credential file outlived a stop, because SIGTERM to the run's -// process group killed the wrapper before its cleanup. It is now removed right after the clones, on -// every way the script ends, and at the next run's start and in lab_check for what a crash left. -import { cleanCredentials, credentialGuard } from "../tools/runs.ts"; - -test("a stopped script removes the credential, whatever ends it", () => { - for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"] as const) { - const dir = mkdtempSync(join(tmpdir(), "lab-guard-")); - const file = join(dir, ".run-x.git-credentials"); - writeFileSync(file, "http://u:p@h\n", { mode: 0o600 }); - const ran = spawnSync("bash", ["-c", `${credentialGuard(file)}\nkill -${signal.slice(3)} $$\nsleep 5`], { encoding: "utf8", timeout: 10_000 }); - assert.notEqual(ran.status, 0, `${signal}: the script was not stopped`); - assert.equal(existsSync(file), false, `${signal} left the credential on disk`); - rmSync(dir, { recursive: true, force: true }); +test("the lab requires the forge account for its clones, and holds no registry credential", () => { + const m = JSON.parse(readFileSync(join(labDir, "module.json"), "utf8")) as { + requires: string[]; contributes?: unknown; secrets?: unknown; resources: { id: string; path?: string; template?: string; package?: string }[]; + build: { artifacts: { name: string; env?: Record }[] }; + }; + assert.ok(m.requires.includes("forge-account") && m.requires.includes("npm-package-registry")); + assert.equal(m.contributes, undefined, "npm-package-registry reads are anonymous: nothing is contributed"); + assert.equal(m.secrets, undefined, "no registry secret is sealed to the lab"); + const env = m.build.artifacts.find((a) => a.name === "code")?.env ?? {}; + assert.equal(env.MESH_LAB_FORGE_SECRET, undefined); + assert.equal(env.MESH_LAB_FORGE_GIT, m.resources.find((r) => r.id === "forge-git")?.path); + assert.match(readFileSync(join(labDir, "files", "lab.env"), "utf8"), /^MESH_LAB_FORGE_GIT=\$\{dir:state\}\/forge\.gitconfig$/m); + assert.doesNotMatch(readFileSync(join(labDir, "files", "lab.env"), "utf8"), /SECRET/); + assert.ok(m.resources.some((r) => r.package === "openssh"), "the clones need ssh"); + for (const id of ["forge-git", "forge-ssh", "forge-known-hosts"]) { + const r = m.resources.find((x) => x.id === id); + assert.match(readFileSync(join(labDir, String(r?.template)), "utf8"), /\$\{bound:forge-account:/, id); } }); -test("the script removes the credential as soon as its clones are done", () => { - const s = script("/work", "run-x", "http://forge", ["t.test.ts"], {}, "/work/.run-x.git-credentials"); - const lastClone = s.lastIndexOf("clone --quiet"); - const removed = s.indexOf("rm -f '/work/.run-x.git-credentials'", lastClone); - const building = s.indexOf("npm ci"); - assert.ok(lastClone > 0 && removed > lastClone && removed < building, "not removed between the clones and the build"); - assert.match(s, /trap 'rm -f '\\''\/work\/\.run-x\.git-credentials'\\''' EXIT/); +test("a run clones through the forge account's git configuration, and its script holds no credential", () => { + const s = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, "/state/forge.gitconfig"); + assert.match(s, /git -c include\.path='\/state\/forge\.gitconfig' clone --quiet --depth 50 --branch 'main' 'http:\/\/novox\.internal:20000\/novox\/mesh-tools-go\.git'/); + assert.match(s, /export GIT_TERMINAL_PROMPT=0/); + assert.doesNotMatch(s, /credential|^trap /m); + assert.match(baseScript("/work/base-x", "http://forge", "/state/forge.gitconfig"), /git -c include\.path='\/state\/forge\.gitconfig' clone/); + assert.equal(forgeArgs(undefined), ""); }); -test("credentials a crashed run left are removed, but not the running one's", () => { +test("the forge account's configuration is taken only when the mesh wrote it", () => { + const state = renderedForgeFiles("http://novox.internal:20000"); + assert.equal(forgeConfig(join(state, "forge.gitconfig")), join(state, "forge.gitconfig")); + assert.equal(forgeConfig(join(state, "nowhere")), undefined); + assert.equal(forgeConfig(undefined), undefined); + writeFileSync(join(state, "empty"), ""); + assert.equal(forgeConfig(join(state, "empty")), undefined); + rmSync(state, { recursive: true, force: true }); +}); + +function withoutSsh(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const { GIT_SSH_COMMAND: _c, GIT_SSH: _s, ...rest } = env; + return rest; +} + +// Held against git and ssh themselves: a clone of the forge's address, as the lab's setting names it or as the +// forge names itself, goes over ssh to the forge account's host with the account's key and the forge's host key. +test("git sends a clone of the forge's address over ssh, as the forge account's configuration says", () => { + const state = renderedForgeFiles("http://novox.internal:20000"); + const bin = mkdtempSync(join(tmpdir(), "lab-ssh-")); + writeFileSync(join(bin, "ssh"), `#!/bin/sh\nprintf '%s\\n' "$@" >> ${JSON.stringify(join(bin, "asked"))}\nexit 1\n`, { mode: 0o755 }); + for (const url of ["http://novox.internal:20000/novox/mesh-tools-go.git", "https://git.example.org/novox/mesh-tools-go.git"]) { + rmSync(join(bin, "asked"), { force: true }); + spawnSync("sh", ["-c", `git ${forgeArgs(join(state, "forge.gitconfig"))}ls-remote ${url}`], + { encoding: "utf8", env: { ...withoutSsh(process.env), PATH: `${bin}:${process.env.PATH}`, GIT_TERMINAL_PROMPT: "0" } }); + const asked = readFileSync(join(bin, "asked"), "utf8").split("\n"); + assert.ok(asked.includes("-F") && asked.includes(join(state, "forge.ssh")), `${url}: ${asked.join(" ")}`); + assert.ok(asked.includes("mesh-forge"), url); + assert.ok(asked.some((a) => a.includes("novox/mesh-tools-go.git")), url); + } + const g = spawnSync("ssh", ["-G", "-F", join(state, "forge.ssh"), "mesh-forge"], { encoding: "utf8" }); + for (const line of ["hostname forge.invalid", "port 222", "user git", "identitiesonly yes", "batchmode yes", + "stricthostkeychecking true", `userknownhostsfile ${join(state, "forge.known_hosts")}`, "hostkeyalias mesh-forge"]) { + assert.ok(g.stdout.split("\n").includes(line), `ssh -G lacks ${line}`); + } + assert.match(g.stdout, /^identityfile ~\/\.ssh\/mesh_ed25519$/m); + assert.equal(readFileSync(join(state, "forge.known_hosts"), "utf8"), "mesh-forge ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOn\n"); + rmSync(state, { recursive: true, force: true }); + rmSync(bin, { recursive: true, force: true }); +}); + +// What a run of before left on disk, its credential line, is removed at the next run's start and in lab_check. +test("credentials an earlier run left are removed", () => { const work = mkdtempSync(join(tmpdir(), "lab-work-")); - for (const n of [".run-a.git-credentials", ".run-b.git-credentials", "run-a", "other"]) writeFileSync(join(work, n), "x"); - const removed = cleanCredentials(work, join(work, ".run-b.git-credentials")); - assert.deepEqual(removed, [join(work, ".run-a.git-credentials")]); - assert.ok(existsSync(join(work, ".run-b.git-credentials")) && existsSync(join(work, "run-a")) && existsSync(join(work, "other"))); + for (const n of [".run-a.git-credentials", ".base-b.git-credentials", "run-a", "other"]) writeFileSync(join(work, n), "x"); + const removed = cleanCredentials(work).sort(); + assert.deepEqual(removed, [join(work, ".base-b.git-credentials"), join(work, ".run-a.git-credentials")]); + assert.ok(existsSync(join(work, "run-a")) && existsSync(join(work, "other"))); rmSync(work, { recursive: true, force: true }); }); -// The lab asks the forge for read only (review 196): its credential is for clones of branches nobody has -// reviewed yet, and must not be able to publish the mesh's packages. -test("the lab asks the package registry for read only", () => { - const m = JSON.parse(readFileSync(join(import.meta.dirname, "..", "module.json"), "utf8")) as { - requires: unknown[]; contributes?: Record>; - }; - assert.ok(m.requires.includes("npm-package-registry")); - assert.equal(m.contributes?.["npm-package-registry"]?.access, "read"); -}); - // lab_check failed for ever on "destination path '/check' already exists": it cloned into one fixed // path and ignored whether removing the last one worked (novox/hq issue 624). Each check now works in a // fresh directory, and what earlier checks left is cleared or named. @@ -239,19 +247,6 @@ test("leftover checks are cleared, whoever runs the check", () => { rmSync(work, { recursive: true, force: true }); }); -// The lab's tools read the bound credential, so the runtime is given that file by the tools bundle's env -// (ADR 0192), as gitea and umami are given theirs: the runtime's account may then read exactly it, and no -// other credential changes hands (issue 624). -test("the lab's tools bundle names its bound credential and binding in its env", () => { - const m = JSON.parse(readFileSync(join(import.meta.dirname, "..", "module.json"), "utf8")) as { - secrets: Record; binds: Record; - build: { artifacts: { name: string; env?: Record }[] }; - }; - const env = m.build.artifacts.find((a) => a.name === "code")?.env ?? {}; - assert.equal(env.MESH_LAB_FORGE_SECRET, m.secrets["npm-package-registry"]); - assert.equal(env.MESH_LAB_FORGE_BINDING, m.binds["npm-package-registry"]); -}); - // The next setup failure of novox/hq issue 624 (run-2026-10-13T13-15-26-634Z on shanks): npm 12 refused // mesh-tools' locked @novox/mesh-sdk with EALLOWREMOTE. The lab named the registry by its forge setting's // address (http://novox.internal:20000/…), the lock file holds the address the forge writes every tarball diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts index 63c7c41f..20bae115 100644 --- a/modules/lab/tools/index.ts +++ b/modules/lab/tools/index.ts @@ -4,12 +4,10 @@ import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; -import { mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { rmSync } from "node:fs"; import { - checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start, - stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun, + checkDir, cleanCredentials, clearChecks, forgeArgs, forgeConfig, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, start, + stop, tail, baseBuildLog, listBaseBuilds, startBaseBuild, launchRun, launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem, } from "./runs.js"; @@ -19,7 +17,6 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // ADR 0192): read from the env-file the mesh fills, at each call, so a changed setting is used // without restarting the runtime. MESH_LAB_FORGE itself still wins, for a hand-run instance. const forgeOf = (): string => (env.MESH_LAB_FORGE ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE")).replace(/\/+$/, ""); - // The forge credential the mesh binds and seals to the lab (novox/hq issue 624), read at each call. // The registry's public root the forge serves in the lab's binding (novox/hq ADR 0346), read at each call. const bindingOf = (): string | undefined => env.MESH_LAB_FORGE_BINDING ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING") || undefined); @@ -28,9 +25,9 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const registry = await npmRegistry(forge, fetch, root.root); return registry.problem && root.problem ? { ...registry, problem: `${registry.problem}; ${root.problem}` } : registry; }; - const credentialOf = (forge: string): string | undefined => forgeCredential(forge, - env.MESH_LAB_FORGE_BINDING ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING"), - env.MESH_LAB_FORGE_SECRET ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_SECRET")); + // The forge account's git configuration the mesh writes for the lab's clones (forgeConfig), read at each call. + const forgeGitOf = (): string | undefined => + forgeConfig(env.MESH_LAB_FORGE_GIT ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_GIT") || undefined)); return [ { name: "lab_check", @@ -50,7 +47,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { if (clone.status !== 0) return { ok: false, output: [String(clone.stderr).trim(), ...leftovers].join("\n") }; spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" }); check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" }); - problems = setupProblems(forge, `${dir}/setup`, credentialOf(forge)); + problems = setupProblems(forge, `${dir}/setup`, forgeGitOf()); // And the address a run names the package registry by (npmRegistry): said when the registry does not say it. const registry = await registryOf(forge); if (registry.problem) problems.push(registry.problem); @@ -60,10 +57,8 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every // repository it clones answers on the forge, and every one it installs has a lock file where the run // installs from. - // A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in - // use by a running run or rebuild is not. - const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id); - const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id))); + // A credential line a run or rebuild of before the forge account left behind is removed, and said. + const left = cleanCredentials(work); const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : ""; // **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot // be read is a problem here, though lab_run refuses only the first. @@ -106,7 +101,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698, // review 256): the registry asked first, then the check and the start with nothing awaited between. const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() }, - (registry) => start(work, forge, tests, refs, credentialOf(forge), registry)); + (registry) => start(work, forge, tests, refs, forgeGitOf(), registry)); if (!launched.started) return launched; return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry, ...(launched.problem ? { problem: launched.problem } : {}) }; @@ -137,7 +132,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with // nothing awaited between (review 256). const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before }, - (registry) => startBaseBuild(work, forge, credentialOf(forge), registry)); + (registry) => startBaseBuild(work, forge, forgeGitOf(), registry)); if (launched.started === false) return { ...reading, started: false, reason: launched.reason }; return { ...reading, started: true, build: launched.job }; }, @@ -174,14 +169,12 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { * answer for, and a checkout it installs with nothing installable (installPlan). Empty when a run's setup * would go through. */ -export function setupProblems(forge: string, dir: string, credential?: string): string[] { +export function setupProblems(forge: string, dir: string, forgeGit?: string): string[] { const problems: string[] = []; spawnSync("rm", ["-rf", dir]); - // The credential a run would offer, in a file of its own outside the check's clones, removed after. - const keep = mkdtempSync(join(tmpdir(), "lab-check-")); - const file = writeCredential(join(keep, "git-credentials"), credential); + // The forge account's configuration a run's clones go through (forgeConfig). const git = (args: string[]) => - spawnSync("sh", ["-c", `git ${credentialArgs(file)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], { + spawnSync("sh", ["-c", `git ${forgeArgs(forgeGit)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, }); try { @@ -192,7 +185,7 @@ export function setupProblems(forge: string, dir: string, credential?: string): const said = remote.stderr.trim().split("\n")[0] ?? ""; problems.push(`a run checks out ${repo}, and ${url} could not be read` + (said ? `: ${said}` : ": no main branch") + - (credential ? "" : " — the lab holds no forge credential (its npm-package-registry binding and secret)")); + (forgeGit ? "" : " — the lab holds no forge account yet (its forge-account binding, written as forge.gitconfig in its state)")); continue; } if (!NPM_REPOSITORIES.includes(repo)) continue; @@ -209,7 +202,6 @@ export function setupProblems(forge: string, dir: string, credential?: string): } } } finally { - rmSync(keep, { recursive: true, force: true }); spawnSync("rm", ["-rf", dir]); } return problems; diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts index a253c069..b1795d42 100644 --- a/modules/lab/tools/runs.ts +++ b/modules/lab/tools/runs.ts @@ -163,68 +163,31 @@ export async function npmRegistry(forge: string, get: typeof fetch = fetch, publ } /** - * The forge credential a run's clones may present, as a git credential-store line — - * scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none. + * The forge account's git configuration a run's clones go through, or undefined when the mesh has not written it. * - * **The mesh's credential for the forge, the way the build modules hold it** (novox/hq issue 624): the lab - * requires the package registry, the forge's provisioner binds it an account (`as`) and seals that - * account's password, and the forge that answers npm is the forge that hosts the repositories. Private - * repositories (mesh-tools-go) are cloned with it; public ones clone as before. Written to a file git reads - * through its credential store, never into a URL, an argument, the run's script or its log, and offered - * only to the forge's own scheme, host and port. + * **The mesh's account on the forge, the way the build modules hold it** (novox/hq issue 624, issue 1966): the lab + * requires `forge-account`, the forge's provider registers the account's key (`~/.ssh/mesh_ed25519`) for it, and + * the mesh writes into the lab's state a git configuration that sends every clone of the forge's address over ssh + * to the forge, with an ssh configuration that offers that key and takes only the forge's host key. Private + * repositories (mesh-tools-go) clone with it; no password is bound, sealed or written for a run. */ -export function forgeCredential(forge: string, bindingFile: string, secretFile: string): string | undefined { - if (!forge || !bindingFile || !secretFile) return undefined; - let as = ""; - let secret = ""; +export function forgeConfig(file: string | undefined): string | undefined { + if (!file) return undefined; try { - as = String((JSON.parse(readFileSync(bindingFile, "utf8")) as { as?: string }).as ?? ""); - secret = readFileSync(secretFile, "utf8").trim(); + return /^\s*insteadOf\s*=\s*\S/m.test(readFileSync(file, "utf8")) ? file : undefined; } catch { return undefined; } - if (!as || !secret) return undefined; - let origin: URL; - try { - origin = new URL(forge); - } catch { - return undefined; - } - const line = new URL(`${origin.protocol}//${origin.host}`); - line.username = as; - line.password = secret; - return line.toString().replace(/\/$/, ""); } -/** Where a run keeps its forge credential: beside its directory, never in it, and removed when it ends. */ +/** Where a run of before kept its forge credential line, beside its directory: removed wherever one is left. */ export function credentialPath(work: string, id: string): string { return join(work, `.${id}.git-credentials`); } -/** writeCredential puts a credential-store line where git reads it, readable by this account alone. */ -export function writeCredential(path: string, line: string | undefined): string | undefined { - if (!line) return undefined; - writeFileSync(path, `${line}\n`, { mode: 0o600 }); - return path; -} - -/** - * credentialGuard is the shell that removes the credential however the script ends — its end, an error, - * or a stop's SIGTERM to the run's process group, which kills the wrapper before its own cleanup runs - * (review 196 on mesh-catalog #272). The script also removes it as soon as its clones are done. - */ -export function credentialGuard(path: string | undefined): string { - if (!path) return ""; - const rm = `rm -f ${shellQuote(path)}`; - return `trap ${shellQuote(rm)} EXIT -trap ${shellQuote(`${rm}; exit 143`)} TERM -trap ${shellQuote(`${rm}; exit 130`)} INT -trap ${shellQuote(`${rm}; exit 129`)} HUP`; -} - /** * cleanCredentials removes every run's and every base rebuild's credential file left in the work directory - * but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it + * but the ones named — what runs of before the forge account, a crash or a reboot left behind. Answers what it * removed. */ export function cleanCredentials(work: string, keep?: string | string[]): string[] { @@ -272,10 +235,10 @@ export function clearChecks(work: string): string[] { return said; } -/** The arguments that have git offer the credential in that file, and only that one. */ -export function credentialArgs(path: string | undefined): string { +/** The arguments that have git read the forge account's configuration (forgeConfig) for one command. */ +export function forgeArgs(path: string | undefined): string { if (!path) return ""; - return `-c credential.helper= -c ${shellQuote(`credential.helper=store --file=${path}`)} `; + return `-c include.path=${shellQuote(path)} `; } export interface RunStatus { @@ -462,7 +425,7 @@ const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`; * at this run's own tree, so what is built and claimed is exactly what was checked out. */ export function script(work: string, id: string, forge: string, tests: string[], refs: Record, - credential?: string, registry = `${forge}${NPM_PATH}`): string { + forgeGit?: string, registry = `${forge}${NPM_PATH}`): string { const dir = runDir(work, id); const setState = (state: string) => `node -e ${shellQuote( @@ -471,7 +434,7 @@ export function script(work: string, id: string, forge: string, tests: string[], const clones = REPOSITORIES.map((repo) => { const ref = refs[repo] ?? "main"; return [ - `git ${credentialArgs(credential)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, + `git ${forgeArgs(forgeGit)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, `echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`, ].join("\n"); }).join("\n"); @@ -479,11 +442,9 @@ export function script(work: string, id: string, forge: string, tests: string[], return `set -euo pipefail # A repository the forge will not show fails its clone at once, rather than waiting on a prompt. export GIT_TERMINAL_PROMPT=0 -${credentialGuard(credential)} cd ${shellQuote(dir)} ${setState("checking-out")} ${clones} -${credential ? `# The forge credential is for the clones, and nothing after them runs with it on disk.\nrm -f ${shellQuote(credential)}` : ""} node -e ${shellQuote( `const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`, )} @@ -516,16 +477,15 @@ node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" /** start begins a run and returns at once with its status. */ export function start(work: string, forge: string, tests: string[], refs: Record, - credential?: string, registry?: string): RunStatus { + forgeGit?: string, registry?: string): RunStatus { const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`; const dir = runDir(work, id); mkdirSync(dir, { recursive: true }); const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs }; writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2)); - // What a crashed or rebooted run left behind goes before this run writes its own. + // A credential line a run of before left behind goes before this run starts. cleanCredentials(work); - const credentials = writeCredential(credentialPath(work, id), credential); - writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials, registry), { mode: 0o700 }); + writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, forgeGit, registry), { mode: 0o700 }); // The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a // run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172). @@ -533,7 +493,6 @@ export function start(work: string, forge: string, tests: string[], refs: Record node -e ${shellQuote( `const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, )} "$code" -rm -f ${shellQuote(credentialPath(work, id))} cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`; const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); child.unref(); @@ -715,14 +674,12 @@ export function baseBuildLog(work: string, id: string, lines = 40): string { * baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it, * then its `base build`, which replaces the image and reads back that it works before publishing it. */ -export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string { +export function baseScript(dir: string, forge: string, forgeGit?: string, registry = `${forge}${NPM_PATH}`): string { const lab = join(dir, "mesh-lab"); return `set -euo pipefail export GIT_TERMINAL_PROMPT=0 -${credentialGuard(credential)} cd ${shellQuote(dir)} -git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)} -${credential ? `rm -f ${shellQuote(credential)}` : ""} +git ${forgeArgs(forgeGit)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)} echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)" printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))} export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))} @@ -733,21 +690,19 @@ node --experimental-strip-types src/cli.ts base build } /** startBaseBuild rebuilds the base image in the background and answers at once with the build. */ -export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild { +export function startBaseBuild(work: string, forge: string, forgeGit?: string, registry?: string): BaseBuild { const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`; const dir = join(work, id); mkdirSync(dir, { recursive: true }); const build: BaseBuild = { id, state: "building", started: new Date().toISOString() }; writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2)); - // What a crashed or rebooted run or rebuild left behind goes before this one writes its own. + // A credential line a run or rebuild of before left behind goes before this one starts. cleanCredentials(work); - const credentials = writeCredential(credentialPath(work, id), credential); - writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 }); + writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, forgeGit, registry), { mode: 0o700 }); const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$? node -e ${shellQuote( `const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, )} "$code" -rm -f ${shellQuote(credentialPath(work, id))} rm -rf ${shellQuote(join(dir, "mesh-lab"))}`; const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); child.unref();