From d94764f00168471ffa2f825f462f1383a5894737 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 21:07:26 +0200 Subject: [PATCH 01/11] Build build-agent and route-proxy without a build ID (hq issue 721) Go's build ID hashes every input package, so a change mesh-builder never reaches still changed its bytes; without it, a rebuild of the same program is the same file, and the check can say the module is unchanged. --- modules/build-agent/Dockerfile | 2 +- modules/route-proxy/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/build-agent/Dockerfile b/modules/build-agent/Dockerfile index 8abeaa19..55f388d9 100644 --- a/modules/build-agent/Dockerfile +++ b/modules/build-agent/Dockerfile @@ -14,7 +14,7 @@ WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /mesh-builder ./cmd/mesh-builder +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -buildvcs=false -ldflags=-buildid= -o /mesh-builder ./cmd/mesh-builder # Unlike mesh-controller's own FROM scratch (ADR 0006: nothing to audit but one binary), the build # machine's whole job is shelling out to git and docker — it needs a real userland to do that in, diff --git a/modules/route-proxy/Dockerfile b/modules/route-proxy/Dockerfile index b556c2f6..7e7fe648 100644 --- a/modules/route-proxy/Dockerfile +++ b/modules/route-proxy/Dockerfile @@ -20,7 +20,7 @@ WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /mesh-route-proxy ./examples/route-proxy +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -buildvcs=false -ldflags=-buildid= -o /mesh-route-proxy ./examples/route-proxy # A small runtime with the public CA roots the ACME client needs to reach a real authority, and run # as root so it can bind :80 and :443 — the two privileged ports a public front door listens on. From 1d8a244ba42bcea8def1751b75fb6de5c8bc28a1 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 22:16:50 +0200 Subject: [PATCH 02/11] Say every forge event on the bus through a signed system webhook, and announce a head or merge at once (novox/hq issue 729) The gitea module keeps one system webhook for every event family, made or corrected at start with the admin account (POST /admin/hooks, forge 1.27.3) and a secret it mints and keeps. Its receiver, at the docker bridge's gateway, refuses an unsigned request before reading it, a body over 5 MB, and a signature that does not verify; it answers a verified delivery at once, drops one already heard by its delivery id, and publishes each as forge.. on the module's events (its own stream, ADR 0363, follows issue 725). A pull request opened, pushed to or merged is announced at once. The polls look every five minutes only while deliveries arrive, and every 30 s again, saying so loudly, when one finds what no event said or ten minutes pass without a delivery. The announced records keep both paths from announcing twice. --- modules/gitea/client.ts | 5 + modules/gitea/index.ts | 216 ++++++++++++++------ modules/gitea/module.json | 58 +++++- modules/gitea/test/webhook.test.ts | 120 +++++++++++ modules/gitea/webhook.ts | 314 +++++++++++++++++++++++++++++ 5 files changed, 645 insertions(+), 68 deletions(-) create mode 100644 modules/gitea/test/webhook.test.ts create mode 100644 modules/gitea/webhook.ts diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 55bba9dc..389f3b62 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -914,6 +914,11 @@ export class GiteaAdmin { return { status: res.status, body }; } + /** One admin-API call with a JSON body, answering the status and body (the system webhook, novox/hq issue 729). */ + call(method: string, path: string, body?: unknown): Promise { + return this.request(path, { method, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); + } + /** Fail with the forge's own message when a status the caller did not expect comes back. */ private static fail(path: string, res: AdminResponse): never { const detail = typeof res.body === "string" ? res.body : JSON.stringify(res.body); diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index d64045a7..8efe7c73 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -40,10 +40,11 @@ import { emit, on } from "@novox/mesh-sdk/events"; import { registerModuleTools } from "@novox/mesh-sdk/tools"; import { commitStatusTool } from "./delivery.js"; import { mergeTool } from "./merge.js"; -import { GiteaClient, movedSince } from "./client.js"; +import { GiteaAdmin, GiteaClient, movedSince } from "./client.js"; import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked, type RepoCheckFacts, reposToLook } from "./pulls.js"; import { GateWriter, clonedAs, type BuildStarted } from "./gate.js"; +import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, keptSecret, receiver, type ForgeEvent } from "./webhook.js"; import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js"; // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is @@ -145,57 +146,9 @@ async function pollMerged(client: GiteaClient): Promise { // at once. const fresh = !!pull.merged_at && !!since && pull.merged_at > since; if (primedMerges && fresh) { - // What it changed, asked for only now: a module is rebuilt because a file inside its own - // directory moved, and without this every module built from a repository is rebuilt for a - // change to any of them (novox/hq 04-ISSUES/131). - const changed = await client.listPullFiles(repo.owner, repo.name, pull.number); - // Which of the directories they are in hold a module at the merge commit (novox/hq issue 278): a - // change inside one is that module's, held or not, and only a file in none is shared code. Not - // said when the list is cut or the forge could not be asked; the mesh then keeps its old rule. - let moduleDirs: string[] | null = null; - if (!changed.truncated) { - try { - moduleDirs = await client.moduleDirsAt(repo.owner, repo.name, pull.merge_commit_sha, changed.paths); - } catch (err) { - console.error(`[gitea] ${repo.full_name}#${pull.number}: which directories hold a module could not be read, ` + - `so the mesh reads its files by the old rule — ${err instanceof Error ? err.message : String(err)}`); - } - } - // The head it merged, and what its head was checked as (novox/hq ADR 0239): the delivery it was, made - // from the forge's word when its owner never heard the head. - let headChecks: Record | null = null; - if (pull.head_sha) { - try { - headChecks = await client.commitStatuses(repo.owner, repo.name, pull.head_sha); - } catch (err) { - console.error(`[gitea] ${repo.full_name}#${pull.number}: its head's statuses could not be read — ${err instanceof Error ? err.message : err}`); - } - } - await emit("pull.merged", { - owner: repo.owner, - repo: repo.name, - number: pull.number, - title: pull.title, - body: pull.body, - head_sha: pull.head_sha, - ...(headChecks ? { head_checks: headChecks } : {}), - head: pull.head, - base: pull.base, - merge_commit_sha: pull.merge_commit_sha, - merged_at: pull.merged_at, - clone_url: repo.clone_url, - html_url: pull.html_url, - paths: changed.paths, - paths_truncated: changed.truncated, - // Which of them the merge deleted (novox/hq ADR 0236): a module whose manifest went is forgotten, - // not built. - removed: changed.removed, - ...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}), - }); - // Said, because a trigger that fires silently is indistinguishable from one that did not - // fire (novox/hq 04-ISSUES/131) — this line is how an operator knows the mesh was told. - console.log(`[gitea] announced merge ${repo.full_name}#${pull.number} (${pull.merge_commit_sha.slice(0, 8)}) into ${pull.base}`); + if (await announceMerge(client, repo, pull)) pace.miss(`the merge of ${repo.full_name}#${pull.number}`); } + else announced.add(pull.merge_commit_sha); announced.add(pull.merge_commit_sha); changed = true; } @@ -206,6 +159,73 @@ async function pollMerged(client: GiteaClient): Promise { lastLook = began; } +type RepoOf = { owner: string; name: string; full_name: string; clone_url?: string }; +type PullOf = Awaited>[number]; + +/** **A merge, announced once, whichever path saw it first** (novox/hq issue 729): the poll or the forge's event. */ +async function announceMerge(client: GiteaClient, repo: RepoOf, pull: PullOf): Promise { + if (!pull.merged || !pull.merge_commit_sha || announced.has(pull.merge_commit_sha)) return false; + // Recorded before the first wait, so the other path finds it taken while this one is still asking the forge. + announced.add(pull.merge_commit_sha); + try { + // What it changed, asked for only now: a module is rebuilt because a file inside its own + // directory moved, and without this every module built from a repository is rebuilt for a + // change to any of them (novox/hq 04-ISSUES/131). + const changed = await client.listPullFiles(repo.owner, repo.name, pull.number); + // Which of the directories they are in hold a module at the merge commit (novox/hq issue 278): a + // change inside one is that module's, held or not, and only a file in none is shared code. Not + // said when the list is cut or the forge could not be asked; the mesh then keeps its old rule. + let moduleDirs: string[] | null = null; + if (!changed.truncated) { + try { + moduleDirs = await client.moduleDirsAt(repo.owner, repo.name, pull.merge_commit_sha, changed.paths); + } catch (err) { + console.error(`[gitea] ${repo.full_name}#${pull.number}: which directories hold a module could not be read, ` + + `so the mesh reads its files by the old rule — ${err instanceof Error ? err.message : String(err)}`); + } + } + // The head it merged, and what its head was checked as (novox/hq ADR 0239): the delivery it was, made + // from the forge's word when its owner never heard the head. + let headChecks: Record | null = null; + if (pull.head_sha) { + try { + headChecks = await client.commitStatuses(repo.owner, repo.name, pull.head_sha); + } catch (err) { + console.error(`[gitea] ${repo.full_name}#${pull.number}: its head's statuses could not be read — ${err instanceof Error ? err.message : err}`); + } + } + await emit("pull.merged", { + owner: repo.owner, + repo: repo.name, + number: pull.number, + title: pull.title, + body: pull.body, + head_sha: pull.head_sha, + ...(headChecks ? { head_checks: headChecks } : {}), + head: pull.head, + base: pull.base, + merge_commit_sha: pull.merge_commit_sha, + merged_at: pull.merged_at, + clone_url: repo.clone_url, + html_url: pull.html_url, + paths: changed.paths, + paths_truncated: changed.truncated, + // Which of them the merge deleted (novox/hq ADR 0236): a module whose manifest went is forgotten, + // not built. + removed: changed.removed, + ...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}), + }); + // Said, because a trigger that fires silently is indistinguishable from one that did not + // fire (novox/hq 04-ISSUES/131) — this line is how an operator knows the mesh was told. + console.log(`[gitea] announced merge ${repo.full_name}#${pull.number} (${pull.merge_commit_sha.slice(0, 8)}) into ${pull.base}`); + } catch (err) { + announced.delete(pull.merge_commit_sha); + throw err; + } + keepAnnounced(); + return true; +} + // **Every new head of an open pull request is announced, once** (novox/hq to-be 45 §9): the mesh checks it // against every machine of its facts before it merges. Kept beside the merges' record, so a restart // announces nothing twice; the first look on a machine with no record announces only what moved in the @@ -228,6 +248,31 @@ function keepHeads(): void { writeFileSync(tmp, JSON.stringify({ heads })); renameSync(tmp, pullsRecord); } +/** **A head, announced once, whichever path saw it first** (novox/hq issue 729): the poll or the forge's event. */ +async function announceHead(client: GiteaClient, repo: RepoOf, pull: OpenOf): Promise { + const key = `${repo.full_name}#${pull.number}`; + const sha = String(pull.head_sha ?? ""); + const before = heads[key]; + // Claimed before the first wait, so the other path finds it taken while this one is still announcing. + if (!claimHead(heads, key, sha)) return false; + try { + // The announcement the merge tool makes too when it asks a check (trunk.ts, novox/hq ADR 0323 §5). + await emit("pull.updated", await announcementOf(client, repo.owner, repo.name, repo.clone_url ?? "", pull)); + } catch (err) { + if (before === undefined) delete heads[key]; + else heads[key] = before; + throw err; + } + keepHeads(); + // Said, so an operator knows the mesh was asked to check it. + console.log(`[gitea] announced ${key} at ${sha.slice(0, 8)} to be checked before it merges`); + // Pending until the verdict comes, so the pull request says a check is running rather than nothing — and + // where it is: in the build queue from now, until a build machine takes it (novox/hq issue 446). + await gates.pending(repo.owner, repo.name, sha); + return true; +} +type OpenOf = Awaited>[number]; + let lastPullLook = ""; // **A sweep now and then looks at every repository** (novox/hq issue 347), whatever moved: the floor is a // heuristic, and a heuristic that misses is caught by the next sweep rather than never. @@ -259,15 +304,8 @@ async function pollPulls(client: GiteaClient): Promise { const key = `${repo.full_name}#${pull.number}`; const fresh = primedPulls || (!!pull.updated_at && pull.updated_at > dayAgo); if (fresh) { - // The announcement the merge tool makes too when it asks a check (trunk.ts, novox/hq ADR 0323 §5). - await emit("pull.updated", await announcementOf(client, repo.owner, repo.name, repo.clone_url, pull)); - // Said, so an operator knows the mesh was asked to check it. - console.log(`[gitea] announced ${key} at ${String(pull.head_sha).slice(0, 8)} to be checked before it merges`); - // Pending until the verdict comes, so the pull request says a check is running rather than nothing — and - // where it is: in the build queue from now, until a build machine takes it (novox/hq issue 446). - await gates.pending(repo.owner, repo.name, String(pull.head_sha)); - } - heads[key] = String(pull.head_sha); + if (await announceHead(client, repo, pull)) pace.miss(`${key} at ${String(pull.head_sha).slice(0, 8)}`); + } else heads[key] = String(pull.head_sha); changed = true; } if (await trunksMoved(client, repo.owner, repo.name)) changedTrunks = true; @@ -413,12 +451,59 @@ async function buildStarted(event: { body: unknown }): Promise { } } +/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are (webhook.ts). */ +const pace = new Pace(); + +/** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */ +async function actOn(client: GiteaClient, event: ForgeEvent): Promise { + const b = event.body as { owner?: string; repo?: string; number?: number }; + if (!b.owner || !b.repo || !b.number) return; + const head = ["forge.pull-request.opened", "forge.pull-request.synchronized", "forge.pull-request.reopened"].includes(event.name); + const merged = event.name === "forge.pull-request.merged"; + if (!head && !merged) return; + const [repo, pull] = await Promise.all([client.getRepo(b.owner, b.repo), client.getPullRequest(b.owner, b.repo, b.number)]); + if (head && pull.state === "open") await announceHead(client, repo, pull); + if (merged) await announceMerge(client, repo, pull); +} + +/** + * **The receiver, and the forge's one system webhook pointing at it** (issue 729). The secret is minted and kept beside + * the token; the webhook is made or corrected with the admin account (ADR 0266: the tools' token holds no admin scope). + * Answers whether events are heard; a forge that refuses is said, and the polls keep their pace. + */ +async function listenForForge(client: GiteaClient): Promise { + const stateDir = process.env.MESH_GITEA_STATE_DIR; + const url = process.env.MESH_GITEA_HOOK_URL; + if (!stateDir || !url) { + console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge"); + return false; + } + try { + const secret = keptSecret(join(stateDir, "webhook.secret")); + const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT); + const server = receiver({ secret, port, heard: () => pace.heard(), + publish: (event) => emit(event.name, event.body), + act: (event) => actOn(client, event) }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0", () => resolve()); + }); + const did = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret); + console.log(`[gitea] listening for the forge's events on port ${port}; its system webhook ${did}, every event, signed`); + return true; + } catch (err) { + console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${err instanceof Error ? err.message : err}`); + return false; + } +} + if (gitea) { const client = gitea; // A poll that fails says so once, not once a minute: the same reason repeating (the forge not up // yet, the admin account refused on a restored forge) is one fact, and a recovery is worth a line. let failing: string | null = null; - const tick = (fn: () => Promise, everyMs: number): void => { + const tick = (fn: () => Promise, every: number | (() => number)): void => { + const everyMs = (): number => (typeof every === "number" ? every : every()); // **One pass at a time** (novox/hq issue 250): the next pass is scheduled when this one has ended, so a // pass that outlasts its interval delays the next instead of running beside it — two passes at once // could each announce the same merge before either recorded it. @@ -433,12 +518,15 @@ if (gitea) { if (why !== failing) console.error(`[gitea] not watching until this clears — ${why}`); failing = why; }) - .finally(() => setTimeout(run, everyMs)); + .finally(() => setTimeout(run, everyMs())); run(); }; + // **The forge's events first, the polls as the safety net** (novox/hq issue 729): while verified deliveries arrive the + // polls slow to every five minutes and announce only what no event said (Pace); otherwise they keep 30 s. + await listenForForge(client); tick(() => pollRepos(client), 60_000); - tick(() => pollMerged(client), 30_000); - tick(() => pollPulls(client), 30_000); + tick(() => pollMerged(client), () => pace.every()); + tick(() => pollPulls(client), () => pace.every()); await on("mesh-controller.checked", (event) => { const c = (event.body ?? {}) as Checked; // A head's own verdict takes its turn after every write of that head's gate already asked for (gate.ts). diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 069d7163..210c2437 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -42,7 +42,47 @@ "issue.opened", "pull.merged", "pull.updated", - "pull.closed" + "pull.closed", + "forge.push", + "forge.create", + "forge.delete", + "forge.fork", + "forge.status", + "forge.pull-request.opened", + "forge.pull-request.reopened", + "forge.pull-request.synchronized", + "forge.pull-request.edited", + "forge.pull-request.closed", + "forge.pull-request.merged", + "forge.pull-request.labeled", + "forge.pull-request.unlabeled", + "forge.pull-request.assigned", + "forge.pull-request.unassigned", + "forge.pull-request.reviewed", + "forge.pull-request.review-requested", + "forge.pull-request.commented", + "forge.pull-request.changed", + "forge.issue-comment.created", + "forge.issue-comment.edited", + "forge.issue-comment.deleted", + "forge.issues.opened", + "forge.issues.reopened", + "forge.issues.closed", + "forge.issues.edited", + "forge.issues.labeled", + "forge.issues.unlabeled", + "forge.issues.assigned", + "forge.issues.unassigned", + "forge.issues.changed", + "forge.release.published", + "forge.release.updated", + "forge.release.deleted", + "forge.release.changed", + "forge.repository.created", + "forge.repository.deleted", + "forge.repository.changed", + "forge.wiki.changed", + "forge.other" ], "consumes": [ "mesh-controller.checked", @@ -61,7 +101,14 @@ "port": 22, "protocol": "tcp", "from": "mesh", - "why": "git over ssh, gitea's own unmodified sshd. Published on the machine's own side at 222, the mesh's fixed public convention \u2014 not 22, which the machine's own daemon holds and a module does not take" + "why": "git over ssh, gitea's own unmodified sshd. Published on the machine's own side at 222, the mesh's fixed public convention — not 22, which the machine's own daemon holds and a module does not take" + }, + { + "name": "forge-events", + "port": 3047, + "protocol": "tcp", + "from": "machine", + "why": "the forge's system webhook, every event signed with a secret the module keeps, from the forge's container on the docker bridge to the module's receiver at the bridge's gateway 172.17.0.1 (novox/hq issue 729)" } ], "serves": { @@ -153,7 +200,7 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "GITEA__security__INTERNAL_TOKEN=${secret:internal-token}\nGITEA__database__DB_TYPE=postgres\nGITEA__database__HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nGITEA__database__NAME=${bound:postgres-database:as}\nGITEA__database__USER=${bound:postgres-database:as}\nGITEA__database__PASSWD=${secret:postgres-database}\n" + "content": "GITEA__security__INTERNAL_TOKEN=${secret:internal-token}\nGITEA__database__DB_TYPE=postgres\nGITEA__database__HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nGITEA__database__NAME=${bound:postgres-database:as}\nGITEA__database__USER=${bound:postgres-database:as}\nGITEA__database__PASSWD=${secret:postgres-database}\nGITEA__webhook__ALLOWED_HOST_LIST=172.17.0.1\n" }, { "id": "data", @@ -263,7 +310,10 @@ "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", "MESH_GITEA_STATE_DIR": "${dir:runtime-state}", "MESH_RECEIVES": "${dir:grants}/npm.json", - "MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors" + "MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors", + "MESH_GITEA_HOOK_URL": "http://172.17.0.1:3047", + "MESH_GITEA_HOOK_PORT": "3047", + "MESH_GITEA_HOOK_HOST": "172.17.0.1" } }, { diff --git a/modules/gitea/test/webhook.test.ts b/modules/gitea/test/webhook.test.ts new file mode 100644 index 00000000..453ff2d0 --- /dev/null +++ b/modules/gitea/test/webhook.test.ts @@ -0,0 +1,120 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createHmac } from "node:crypto"; +import type { AddressInfo } from "node:net"; + +// **The forge says every event on the bus** (novox/hq issue 729): a signed delivery is published once and acted on, +// an unsigned or unverified one is refused, and an event and the poll seeing the same head announce it once. + +const secret = "s".repeat(64); +const sign = (body: string, key = secret) => createHmac("sha256", key).update(body).digest("hex"); +const opened = JSON.stringify({ action: "opened", number: 7, pull_request: { number: 7, title: "t", user: { login: "jo" }, + head: { ref: "feat/x", sha: "a".repeat(40) }, base: { ref: "main" }, merged: false }, + repository: { name: "hq", full_name: "novox/hq", owner: { login: "novox" } }, sender: { login: "jo" } }); + +async function serve(published: { name: string }[], acted: { name: string }[]) { + const { receiver, HOOK_PATH } = await import("../webhook.ts"); + const server = receiver({ secret, port: 0, log: () => {}, + publish: async (e) => { published.push(e); }, act: async (e) => { acted.push(e); } }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`; + return { server, url }; +} + +test("a signed event is published on the bus and acted on once", async () => { + const published: { name: string }[] = [], acted: { name: string }[] = []; + const { server, url } = await serve(published, acted); + try { + const headers = { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-1" }; + const res = await fetch(url, { method: "POST", body: opened, headers }); + assert.equal(res.status, 200); + // The same delivery sent again is answered and published nowhere. + const again = await fetch(url, { method: "POST", body: opened, headers }); + assert.equal(again.status, 200); + await server.idle(); + assert.deepEqual(published.map((e) => e.name), ["forge.pull-request.opened"]); + assert.deepEqual(acted.map((e) => e.name), ["forge.pull-request.opened"]); + } finally { server.close(); } +}); + +test("an unsigned event, or one whose signature does not verify, is refused and published nowhere", async () => { + const published: { name: string }[] = [], acted: { name: string }[] = []; + const { server, url } = await serve(published, acted); + try { + const unsigned = await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request" } }); + assert.equal(unsigned.status, 401); + assert.match(await unsigned.text(), /unsigned/); + const forged = await fetch(url, { method: "POST", body: opened, + headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "another secret") } }); + assert.equal(forged.status, 401); + assert.match(await forged.text(), /does not verify/); + const big = "x".repeat(5 * 1024 * 1024 + 1); + const large = await fetch(url, { method: "POST", body: big, + headers: { "X-Gitea-Event": "push", "X-Gitea-Signature": sign(big) } }); + assert.equal(large.status, 413); + await server.idle(); + assert.deepEqual(published, []); + assert.deepEqual(acted, []); + } finally { server.close(); } +}); + +test("an event and the poll seeing the same head announce it once", async () => { + const { claimHead } = await import("../webhook.ts"); + const heads: Record = {}; + const sha = "a".repeat(40); + assert.equal(claimHead(heads, "novox/hq#7", sha), true, "the event claims the head and announces it"); + assert.equal(claimHead(heads, "novox/hq#7", sha), false, "the poll finds it taken"); + assert.equal(claimHead(heads, "novox/hq#7", "b".repeat(40)), true, "a new head is announced again"); +}); + +test("each forge event is named in the module's stream and trimmed", async () => { + const { eventOf } = await import("../webhook.ts"); + const merged = eventOf("pull_request", { action: "closed", pull_request: { number: 3, merged: true, merge_commit_sha: "c".repeat(40), + head: { sha: "a".repeat(40) }, base: { ref: "main" } }, repository: { name: "hq", owner: { login: "novox" } } }); + assert.equal(merged.name, "forge.pull-request.merged"); + assert.equal(merged.body.merge_commit_sha, "c".repeat(40)); + assert.equal(eventOf("pull_request", { action: "label_updated", pull_request: { number: 3 }, label: { name: "mesh/ready" } }).name, + "forge.pull-request.labeled"); + const comment = eventOf("issue_comment", { action: "created", is_pull: true, issue: { number: 3 }, + comment: { id: 9, body: "ready at abc", user: { login: "jo" } } }); + assert.equal(comment.name, "forge.pull-request.commented"); + assert.equal(comment.body.comment, "ready at abc"); + assert.equal(eventOf("push", { ref: "refs/heads/main", after: "d".repeat(40) }).name, "forge.push"); + assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other"); +}); + +test("the system webhook is made when none points at the receiver, and corrected when one does", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const calls: string[] = []; + let hooks: unknown[] = []; + const admin = { call: async (method: string, path: string, body?: any) => { + calls.push(`${method} ${path}`); + if (method === "GET") return { status: 200, body: hooks }; + if (method === "POST") { assert.ok(body.events.includes("pull_request") && body.events.includes("push")); assert.equal(body.config.secret, secret); return { status: 201, body: {} }; } + return { status: 200, body: {} }; + } }; + assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "created"); + hooks = [{ id: 4, config: { url: "http://h:3047/forge-event" } }]; + assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "corrected"); + assert.ok(calls.includes("PATCH /admin/hooks/4")); +}); + +test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const pace = new Pace(() => now, (l) => said.push(l)); + assert.equal(pace.every(), 30_000, "no delivery yet"); + pace.heard(); + assert.equal(pace.every(), 300_000, "a delivery heard"); + now += 11 * 60_000; + assert.equal(pace.every(), 30_000, "ten minutes without a delivery"); + pace.heard(); + now += 1000; + pace.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(pace.every(), 30_000, "the poll found what no event said"); + assert.match(said.join("\n"), /MISSED novox\/hq#7/); + now += 1000; + pace.heard(); + assert.equal(pace.every(), 300_000, "the next delivery"); +}); diff --git a/modules/gitea/webhook.ts b/modules/gitea/webhook.ts new file mode 100644 index 00000000..130a4e01 --- /dev/null +++ b/modules/gitea/webhook.ts @@ -0,0 +1,314 @@ +// **The forge says every event on the bus** (novox/hq issue 729; to-be 10: an event makes it fast, nothing makes it +// necessary). The module keeps one system webhook on its forge for every event, pointing at its own receiver, signed +// with a secret it mints and keeps beside its token; the receiver refuses whatever is unsigned or does not verify, and +// publishes each event, trimmed, on the module's events as forge.. (forge.pull-request.opened, forge.push, …), +// on the shared EVENTS path its other events take; its own stream (ADR 0363) is a follow-up once issue 725 ships. +// The polls stay as the safety net, slowed; the announced-heads and announced-merges records keep anything from +// being announced twice, whichever path saw it first. +// +// The secret is never logged: lines say it was minted or reused and where it is kept, never what it is. + +import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; +import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import { dirname } from "node:path"; + +/** The name the receiver answers on, and the port it listens on unless the manifest says otherwise. */ +export const HOOK_PATH = "/forge-event"; +export const HOOK_PORT = 3047; + +/** The secret the forge signs with: read from its 0600 file, or minted and kept there. */ +export function keptSecret(file: string, log: (line: string) => void = (l) => console.log(`[gitea] ${l}`)): string { + try { + const kept = readFileSync(file, "utf8").trim(); + if (kept.length >= 32) { + log(`reusing the webhook secret kept at ${file}`); + return kept; + } + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; + } + const secret = randomBytes(32).toString("hex"); + mkdirSync(dirname(file), { recursive: true, mode: 0o700 }); + const tmp = `${file}.tmp`; + writeFileSync(tmp, secret + "\n", { mode: 0o600 }); + chmodSync(tmp, 0o600); + renameSync(tmp, file); + log(`minted a webhook secret, kept at ${file}`); + return secret; +} + +/** Whether a body carries the forge's HMAC-SHA256 signature (X-Gitea-Signature, hex) under the secret. */ +export function verified(secret: string, body: Buffer | string, signature: string | undefined): boolean { + if (!secret || !signature || !/^[0-9a-f]{64}$/i.test(signature.trim())) return false; + const want = createHmac("sha256", secret).update(body).digest(); + const got = Buffer.from(signature.trim(), "hex"); + return got.length === want.length && timingSafeEqual(got, want); +} + +/** One forge event as the bus carries it: its name inside the stream, and what is needed of it — no secrets. */ +export interface ForgeEvent { + name: string; + body: Record; +} + +/** Every event name the module emits for the forge, without its forge. prefix (module.json `emits`). */ +export const EVENTS: readonly string[] = ["push", "create", "delete", "fork", "status", "pull-request.opened", "pull-request.reopened", "pull-request.synchronized", "pull-request.edited", "pull-request.closed", "pull-request.merged", "pull-request.labeled", "pull-request.unlabeled", "pull-request.assigned", "pull-request.unassigned", "pull-request.reviewed", "pull-request.review-requested", "pull-request.commented", "pull-request.changed", "issue-comment.created", "issue-comment.edited", "issue-comment.deleted", "issues.opened", "issues.reopened", "issues.closed", "issues.edited", "issues.labeled", "issues.unlabeled", "issues.assigned", "issues.unassigned", "issues.changed", "release.published", "release.updated", "release.deleted", "release.changed", "repository.created", "repository.deleted", "repository.changed", "wiki.changed", "other"]; + +/** The families the module's stream carries (module.json `streams`); any other kind is said as other.; underscores become hyphens. */ +export const FAMILIES = ["push", "create", "delete", "fork", "status", "pull_request", "issue_comment", "issues", + "release", "repository", "wiki"] as const; + +const ACTIONS: Record = { label_updated: "labeled", label_cleared: "unlabeled", synchronized: "synchronized" }; + +function str(v: unknown): string | undefined { + return typeof v === "string" && v ? v : undefined; +} +function num(v: unknown): number | undefined { + return typeof v === "number" ? v : undefined; +} +function login(u: any): string | undefined { + return str(u?.login) ?? str(u?.username); +} + +/** + * The event a forge delivery is: its kind (X-Gitea-Event) and the payload's action, trimmed to ids, repository, refs, + * shas, author, labels and a comment's body (a ready mark is one, ADR 0364). A merge is pull-request.merged, a comment on + * a pull request pull-request.commented. + */ +export function eventOf(kind: string, payload: any): ForgeEvent { + // The bus's event names take lower-case letters, digits and hyphens: the forge's underscores become hyphens. + const event = forgeEventOf(kind, payload); + const name = event.name.toLowerCase().replace(/_/g, "-").replace(/[^a-z0-9.-]/g, "-"); + // Published on the module's events as it publishes its others, each name declared in its `emits`: one the manifest + // does not name is forge.other, saying what the forge called it. Its own stream (ADR 0363) follows issue 725. + if (EVENTS.includes(name)) return { name: `forge.${name}`, body: event.body }; + return { name: "forge.other", body: { ...event.body, kind, said: name } }; +} + +function forgeEventOf(said: string, payload: any): ForgeEvent { + // The forge names some kinds finer than their family (pull_request_label, pull_request_sync): read by the payload. + let kind = said; + if (said.startsWith("pull_request") && payload?.pull_request) kind = "pull_request"; + if ((said.startsWith("issue") || said === "pull_request_comment") && payload?.comment) kind = "issue_comment"; + if (said.startsWith("issue") && !payload?.comment && payload?.issue) kind = "issues"; + const repo = payload?.repository ?? {}; + const base: Record = { + owner: login(repo.owner), repo: str(repo.name), full_name: str(repo.full_name), clone_url: str(repo.clone_url), + sender: login(payload?.sender), + }; + const family = (FAMILIES as readonly string[]).includes(kind) ? kind : `other`; + let action = str(payload?.action); + const pull = payload?.pull_request; + if (kind === "push") { + return { name: "push", body: { ...base, ref: str(payload.ref), before: str(payload.before), after: str(payload.after), + commits: Array.isArray(payload.commits) ? payload.commits.length : undefined, pusher: login(payload.pusher) } }; + } + if (kind === "pull_request" && pull) { + if (action === "closed" && pull.merged) action = "merged"; + action = ACTIONS[action ?? ""] ?? action ?? "changed"; + return { name: `pull_request.${action}`, body: { ...base, number: num(pull.number ?? payload.number), + title: str(pull.title), author: login(pull.user), head: str(pull.head?.ref), head_sha: str(pull.head?.sha), + base: str(pull.base?.ref), merged: !!pull.merged, merge_commit_sha: str(pull.merge_commit_sha), + merged_at: str(pull.merged_at), html_url: str(pull.html_url), + labels: Array.isArray(pull.labels) ? pull.labels.map((l: any) => str(l?.name)).filter(Boolean) : undefined, + label: str(payload.label?.name) } }; + } + if (kind === "issue_comment") { + const issue = payload.issue ?? {}; + const onPull = !!payload.is_pull || !!issue.pull_request; + return { name: onPull ? "pull_request.commented" : `issue_comment.${action ?? "created"}`, body: { ...base, + number: num(issue.number), comment_id: num(payload.comment?.id), author: login(payload.comment?.user), + comment: str(payload.comment?.body), action } }; + } + if (kind === "create" || kind === "delete") { + return { name: kind, body: { ...base, ref: str(payload.ref), ref_type: str(payload.ref_type), sha: str(payload.sha) } }; + } + if (kind === "status") { + return { name: "status", body: { ...base, sha: str(payload.sha), context: str(payload.context), state: str(payload.state) } }; + } + const name = family === "other" ? `other.${kind.replace(/[^a-z0-9_-]/gi, "_") || "unknown"}` : + ["fork"].includes(family) ? family : `${family}.${action ?? "changed"}`; + return { name, body: { ...base, number: num(payload?.number ?? payload?.issue?.number), action } }; +} + +export interface ReceiverOptions { + secret: string; + port: number; + host?: string; + /** Publishes one event on the bus. */ + publish: (event: ForgeEvent) => Promise; + /** Acts on it at once, beside publishing it (the announcer). */ + act?: (event: ForgeEvent) => Promise; + /** Told of every verified delivery, so the polls know events are heard. */ + heard?: () => void; + log?: (line: string) => void; +} + +/** The most a delivery may carry: the forge's push of many commits stays well under it. */ +export const MAX_BODY = 5 * 1024 * 1024; +/** How many deliveries' ids are remembered, so a delivery sent again is published once. */ +export const REMEMBERED = 2000; + +/** + * The receiver. A request without a signature is refused before its body is read, a body over MAX_BODY is refused, and + * one whose signature does not verify is refused, each with 401 or 413 and a line saying so. A verified delivery is + * answered 200 at once and queued: it is published and acted on after the reply, one at a time, so the forge's + * delivery timeout is never reached. A delivery whose X-Gitea-Delivery id was already heard is answered and dropped. + */ +export function receiver(opts: ReceiverOptions): Server & { idle(): Promise } { + const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`)); + const seen = new Set(); + let queue: Promise = Promise.resolve(); + const server = createServer((req, res) => { + if (req.method !== "POST" || (req.url ?? "").split("?")[0] !== HOOK_PATH) { + res.statusCode = 404; + return res.end(); + } + const from = req.socket.remoteAddress ?? "?"; + const signature = String(req.headers["x-gitea-signature"] ?? "").trim(); + if (!signature) { + log(`refused a forge event from ${from}: it is unsigned`); + res.writeHead(401, { Connection: "close" }); + res.end("unsigned\n"); + req.destroy(); + return; + } + if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) { + log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`); + res.writeHead(413, { Connection: "close" }); + res.end("too large\n"); + req.destroy(); + return; + } + const chunks: Buffer[] = []; + let size = 0; + let over = false; + req.on("data", (c: Buffer) => { + size += c.length; + if (size > MAX_BODY) { + if (!over) { + over = true; + log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`); + res.writeHead(413, { Connection: "close" }); + res.end("too large\n"); + req.destroy(); + } + } else chunks.push(c); + }); + req.on("end", () => { + if (over) return; + const body = Buffer.concat(chunks); + if (!verified(opts.secret, body, signature)) { + log(`refused a forge event from ${from}: its signature does not verify`); + res.statusCode = 401; + return res.end("signature does not verify\n"); + } + let payload: unknown; + try { + payload = JSON.parse(body.toString("utf8")); + } catch { + res.statusCode = 400; + return res.end("not JSON\n"); + } + opts.heard?.(); + const delivery = String(req.headers["x-gitea-delivery"] ?? ""); + if (delivery && seen.has(delivery)) { + res.statusCode = 200; + return res.end("already heard\n"); + } + if (delivery) { + seen.add(delivery); + if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string); + } + const event = eventOf(String(req.headers["x-gitea-event"] ?? "other"), payload); + res.statusCode = 200; + res.end("queued\n"); + queue = queue.then(async () => { + try { + await opts.publish(event); + if (opts.act) await opts.act(event); + } catch (err) { + log(`a forge event ${event.name} could not be published or acted on; the polls catch it — ${err instanceof Error ? err.message : err}`); + } + }); + }); + }); + return Object.assign(server, { idle: () => queue }); +} + +/** What ensuring the system webhook needs of the forge's administration (GiteaAdmin's login). */ +export interface HookAdmin { + call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>; +} + +/** + * **How fast the polls look** (issue 729): every five minutes only while the forge's events are heard — a verified + * delivery within the last ten minutes, and none missed since. Until the first delivery, after ten minutes without one, + * or once the poll found something no event said, every thirty seconds. + */ +export class Pace { + private last = 0; + private missed = 0; + private readonly now: () => number; + private readonly log: (l: string) => void; + constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) { + this.now = now; + this.log = log; + } + heard(): void { + this.last = this.now(); + } + /** The poll found a head or merge no forge event announced. */ + miss(what: string): void { + if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`); + this.missed = this.now(); + } + hearing(): boolean { + const t = this.now(); + return this.last > 0 && t - this.last < 10 * 60_000 && this.last > this.missed; + } + every(): number { + return this.hearing() ? 5 * 60_000 : 30_000; + } +} + +/** Every event family the forge (1.27) can send a webhook. */ +export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request", + "pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"]; + +/** + * **One system webhook, for every event, at the receiver** (issue 729): created when none points there, corrected + * when one does but differs (events, active, content type), and its secret set again each start — the forge never + * hands a secret back, so a corrected secret cannot be told from a kept one. Answers what it did. + */ +export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<"created" | "corrected"> { + // POST /admin/hooks makes a system webhook, and its options take no flag for it (the forge's 1.27.3 API schema). The + // forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it. + const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*", + config: { url, content_type: "json", secret } }; + const listed = await admin.call("GET", "/admin/hooks?type=system&limit=50"); + if (listed.status !== 200) throw new Error(`the forge's system webhooks could not be listed: ${listed.status}`); + const hooks: any[] = Array.isArray(listed.body) ? listed.body : []; + const ours = hooks.filter((h) => h?.config?.url === url); + if (ours.length === 0) { + const made = await admin.call("POST", "/admin/hooks", want); + if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`); + return "created"; + } + for (const extra of ours.slice(1)) await admin.call("DELETE", `/admin/hooks/${extra.id}`); + const fixed = await admin.call("PATCH", `/admin/hooks/${ours[0].id}`, { active: true, events: HOOK_EVENTS, branch_filter: "*", + config: { url, content_type: "json", secret } }); + if (fixed.status !== 200) throw new Error(`the forge's system webhook could not be corrected: ${fixed.status}`); + return "corrected"; +} + +/** + * **Claim a head for announcing, once** (issue 729): true when `sha` is not already the head recorded for `key`, which + * it then is. The poll and the forge's event both claim before they announce, so whichever comes second finds it taken. + */ +export function claimHead(heads: Record, key: string, sha: string): boolean { + if (!sha || heads[key] === sha) return false; + heads[key] = sha; + return true; +} From d1debdcce8168af4de109392f0a81a0313886eb3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 22:38:38 +0200 Subject: [PATCH 03/11] Clear a forge token name held twice, and mint one process at a time (hq issue 740) The runtime starts the watcher and the tools as two processes over one kept token. Both minted at once on 2026-10-14 and left two tokens named mesh-tools; gitea refuses a delete by a name held twice, so every mint failed and every forge tool with it. A replace now deletes each token by the name by its id, from the account's whole list, and a mint holds a lock file beside the kept token and reuses what another process minted. --- modules/gitea/test/token.test.ts | 101 +++++++++++++++++++++-- modules/gitea/token.ts | 136 +++++++++++++++++++++++++++---- 2 files changed, 215 insertions(+), 22 deletions(-) diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index 750c8122..59cacb90 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -10,7 +10,7 @@ import { test, after } from "node:test"; import assert from "node:assert/strict"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -29,6 +29,12 @@ interface Forge { mints: number; lastScopes: string[] | null; tokens: Map; + /** Tokens beside `tokens` that carry a name already held there: what two mints at once leave behind. */ + twins: { id: number; name: string; sha1: string }[]; + /** How each DELETE named its token: "id" or "name". */ + deletedBy: string[]; + /** Plant a token by name, beside any other of the same name, as a raced mint would. */ + plant(name: string): void; scopesOf: Map; admins: Map; pullState: string; @@ -42,6 +48,10 @@ function fakeForge(): Promise { mints: 0, lastScopes: null as string[] | null, tokens: new Map(), // name -> value + twins: [] as { id: number; name: string; sha1: string }[], + deletedBy: [] as string[], + ids: new Map(), // value -> id, as gitea numbers its tokens + nextId: 0, scopesOf: new Map(), // value -> scopes, so a route can enforce them like gitea does admins: new Map([[ADMIN, PASSWORD]]), pullState: "open", @@ -82,12 +92,35 @@ function fakeForge(): Promise { const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`; forge.tokens.set(name, sha1); forge.scopesOf.set(sha1, scopes); - return json(res, 201, { id: forge.mints, name, sha1, scopes, token_last_eight: sha1.slice(-8) }); + forge.ids.set(sha1, ++forge.nextId); + return json(res, 201, { id: forge.ids.get(sha1), name, sha1, scopes, token_last_eight: sha1.slice(-8) }); + } + // gitea's list: every token of the account, a page at a time (at most three to a page here, so a + // client that reads only the first page misses some), the values never shown. + const listed = (): { id: number; name: string; sha1: string }[] => [ + ...[...forge.tokens].map(([name, sha1]) => ({ id: forge.ids.get(sha1) ?? 0, name, sha1 })), + ...forge.twins, + ]; + if (req.method === "GET" && !tokens[2]) { + const page = Math.max(1, Number(url.searchParams.get("page") ?? "1")); + const limit = Math.min(3, Math.max(1, Number(url.searchParams.get("limit") ?? "3"))); + const all = listed(); + res.setHeader("X-Total-Count", String(all.length)); + return json(res, 200, all.slice((page - 1) * limit, page * limit) + .map((t) => ({ id: t.id, name: t.name, sha1: "", token_last_eight: t.sha1.slice(-8), scopes: [] }))); } if (req.method === "DELETE" && tokens[2]) { - const name = decodeURIComponent(tokens[2]); - if (!forge.tokens.has(name)) return json(res, 404, { message: "token not found" }); - forge.tokens.delete(name); + // gitea: "identified by ID and if not available by name" — and a name held twice is refused. + const key = decodeURIComponent(tokens[2]); + const byId = /^\d+$/.test(key); + const matches = listed().filter((t) => (byId ? String(t.id) === key : t.name === key)); + if (matches.length === 0) return json(res, 404, { message: "token not found" }); + if (matches.length > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` }); + const t = matches[0]; + forge.deletedBy.push(byId ? "id" : "name"); + const twin = forge.twins.findIndex((w) => w.id === t.id); + if (twin >= 0) forge.twins.splice(twin, 1); + else forge.tokens.delete(t.name); return json(res, 204, null); } return json(res, 405, { message: "method not allowed" }); @@ -181,6 +214,14 @@ function fakeForge(): Promise { get pullTitle() { return forge.pullTitle; }, get branchDeleted() { return forge.branchDeleted; }, tokens: forge.tokens, + twins: forge.twins, + deletedBy: forge.deletedBy, + plant: (name: string) => { + const sha1 = `planted-${++forge.nextId}`; + forge.ids.set(sha1, forge.nextId); + if (forge.tokens.has(name)) forge.twins.push({ id: forge.nextId, name, sha1 }); + else forge.tokens.set(name, sha1); + }, scopesOf: forge.scopesOf, admins: forge.admins, close: () => new Promise((r) => server.close(() => r())), @@ -335,6 +376,56 @@ test("concurrent first calls share one mint", async () => { assert.equal(forge.mints, before + 1); }); +test("two tokens hold the name (two mints raced): both are deleted by their id, and one fresh token is minted", async () => { + const { env, file, logs } = await delivered(forge); + forge.tokens.clear(); + forge.twins.length = 0; + for (const other of ["laptop", "backup", "ci"]) forge.plant(other); // the account's other tokens: kept + forge.plant("mesh-tools"); + forge.plant("mesh-tools"); // gitea refuses to delete either by name from here on + forge.deletedBy.length = 0; + const before = forge.mints; + + const repos = await minted(env, logs).listRepos(); + + assert.equal(repos.length, 1); + assert.equal(forge.mints, before + 1); + assert.deepEqual(forge.deletedBy, ["id", "id"]); + assert.equal(forge.twins.length, 0); + assert.deepEqual([...forge.tokens.keys()].sort(), ["backup", "ci", "laptop", "mesh-tools"]); + assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n"); + assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n")); +}); + +test("two processes on one kept file, both without a token, mint once: the second waits for the first and reuses it", async () => { + const { env, logs } = await delivered(forge); + forge.tokens.clear(); + forge.twins.length = 0; + const before = forge.mints; + // Two sources over one state directory, as the tools bundle and the watcher are two processes. + const first = minted(env, logs); + const second = minted(env, logs); + + await Promise.all([first.listRepos(), second.listRepos()]); + + assert.equal(forge.mints, before + 1); + assert.equal(forge.twins.length, 0); + assert.ok(logs.some((l) => l.includes("another process minted it")), logs.join("\n")); +}); + +test("a lock left by a process that died mid-mint is taken over, not waited on for ever", async () => { + const { env, file, logs } = await delivered(forge); + forge.tokens.clear(); + forge.twins.length = 0; + await mkdir(join(env.MESH_GITEA_STATE_DIR!), { recursive: true }); + await writeFile(`${file}.lock`, `2147483646 ${Date.now()} gone\n`); // a pid no process holds + const before = forge.mints; + + assert.equal((await minted(env, logs).listRepos()).length, 1); + assert.equal(forge.mints, before + 1); + await assert.rejects(stat(`${file}.lock`), /ENOENT/); +}); + test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => { const { env, file, logs } = await delivered(forge); forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there diff --git a/modules/gitea/token.ts b/modules/gitea/token.ts index 3aac30b1..d34c303e 100644 --- a/modules/gitea/token.ts +++ b/modules/gitea/token.ts @@ -12,7 +12,15 @@ // hands a token's value out exactly once, so a token not kept is a token lost; // - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not // a fault: the forge's data was restored from a predecessor and the token the file names never -// existed there. +// existed there; +// - minted by one process at a time. The runtime starts each of the bundle's entrypoints as a process +// of its own (novox/hq ADR 0193): the watcher (index.js) and the tools (tools/index.js) are two +// processes over one kept file. Two that minted at once left the forge two tokens by one name, which +// gitea then refuses to delete by name — for ever, and every forge tool with it (the 2026-10-14 +// outage). So a mint holds a lock file beside the kept token, and reads the kept file again once it +// holds it: what another process minted meanwhile is reused, not minted over; +// - replaced by id: an old token by this name is found in the account's list and deleted by its id, +// every one of them, so a name held twice is cleared rather than refused. // // An explicitly configured token still wins, and is never minted over: if it is rejected, that is // reported, not repaired — somebody chose it. @@ -20,7 +28,8 @@ // The token is never logged. Lines say that one was minted, reused or renewed, and where it is // kept; never what it is. -import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs"; +import { randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; /** The name the token carries in the forge's own list — one per mesh runtime, found by name. */ @@ -39,7 +48,7 @@ export const TOKEN_NAME = "mesh-tools"; * `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the * operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token. * A token kept with a different list is minted again with this one at the next start (the scopes are kept - * beside it), and the forge drops the old one by name — so the scope a kept token had does not outlive this. + * beside it), and the old one is deleted from the forge by its id — so the scope a kept token had does not outlive this. * * A token kept from before a scope was added lacks it: the forge answers such a call with * `403 token does not have at least one of required scope(s)`, and the client treats that like a @@ -124,9 +133,9 @@ export class MintedToken implements TokenSource { */ static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken { const opts = MintedToken.optionsFromEnv(url, env); - // One source per kept file in a process. The watcher and the tools entrypoint both build a - // client in the same runtime; two sources over one file would each renew on a 401 and drop the - // other's token by name, forever. Shared, a renewal is one renewal. + // One source per kept file in a process: two sources in one process would each renew on a 401. + // This covers one process only. The watcher and the tools are two processes (the runtime starts + // each entrypoint as its own), and those are kept apart by the mint's lock file (mint, below). const shared = MintedToken.shared.get(opts.file); if (shared) return shared; const source = new MintedToken(opts); @@ -161,9 +170,9 @@ export class MintedToken implements TokenSource { this.log(`reusing the token kept at ${this.opts.file}`); return kept; } - if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them"); + if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept); } - return this.mint("no token kept — minting one"); + return this.mint("no token kept — minting one", null); } async renew(rejected: string): Promise { @@ -177,7 +186,7 @@ export class MintedToken implements TokenSource { return kept; } this.held = null; - return this.mint("the forge rejected the kept token — minting a fresh one"); + return this.mint("the forge rejected the kept token — minting a fresh one", rejected); } /** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */ @@ -185,17 +194,95 @@ export class MintedToken implements TokenSource { return status === 403 && /required scope/i.test(body); } - /** One mint at a time: concurrent first calls share it, rather than each minting its own. */ - private mint(why: string): Promise { + /** + * One mint at a time: concurrent first calls in this process share it, and a mint in another process + * on this machine is waited for (the lock file). Once the lock is held, the kept file is read again: + * a token there that is not `stale` (the one rejected, or kept with other scopes) and carries the + * scopes asked for was minted by the process that held the lock before — it is reused. + */ + private mint(why: string, stale: string | null): Promise { if (this.inflight === null) { this.log(why); - this.inflight = this.doMint().finally(() => { + this.inflight = this.locked(async () => { + const kept = this.read(); + if (kept !== null && kept !== stale && this.keptScopes() === this.scopesLine()) { + this.held = kept; + this.log(`another process minted it meanwhile — reusing the token kept at ${this.opts.file}`); + return kept; + } + return this.doMint(); + }).finally(() => { this.inflight = null; }); } return this.inflight; } + /** How long a mint may hold the lock before another process takes it as left behind, and how long one waits. */ + static readonly LOCK_STALE_MS = 120_000; + static readonly LOCK_WAIT_MS = 150_000; + + /** + * Run `work` holding the lock file beside the kept token: created exclusively, holding this process's + * pid, the time and a nonce; removed after, only when it is still ours. A lock whose process is gone, + * or older than LOCK_STALE_MS, was left by a process that died mid-mint and is taken over — never + * waited on for ever, which would be the same lock-out by another road. + */ + private async locked(work: () => Promise): Promise { + const lock = `${this.opts.file}.lock`; + mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 }); + const mine = `${process.pid} ${Date.now()} ${randomBytes(8).toString("hex")}\n`; + const deadline = Date.now() + MintedToken.LOCK_WAIT_MS; + for (;;) { + try { + const fd = openSync(lock, "wx", 0o600); + try { writeSync(fd, mine); } finally { closeSync(fd); } + break; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "EEXIST") { + throw new Error(`cannot take the mint lock at ${lock}: ${(err as Error).message}`); + } + } + if (this.leftBehind(lock)) { + this.log(`the mint lock at ${lock} was left by a process that is gone — taking it over`); + try { unlinkSync(lock); } catch { /* another waiter took it over first */ } + continue; + } + if (Date.now() > deadline) { + throw new Error(`another process has held the mint lock at ${lock} for over ${MintedToken.LOCK_WAIT_MS / 1000}s — not minting beside it`); + } + await new Promise((r) => setTimeout(r, 50 + Math.floor(Math.random() * 50))); + } + try { + return await work(); + } finally { + try { + if (readFileSync(lock, "utf8") === mine) unlinkSync(lock); + } catch { /* already gone */ } + } + } + + /** Whether the lock's holder is gone: its pid answers no signal, or it is older than LOCK_STALE_MS. */ + private leftBehind(lock: string): boolean { + let text: string; + let age: number; + try { + text = readFileSync(lock, "utf8"); + age = Date.now() - statSync(lock).mtimeMs; + } catch { + return false; // released meanwhile: the next attempt takes it + } + if (age > MintedToken.LOCK_STALE_MS) return true; + const pid = Number(text.split(" ")[0]); + if (!Number.isInteger(pid) || pid <= 0) return age > 1_000; // half-written, or not ours: give it a moment + try { + process.kill(pid, 0); + return false; + } catch (err) { + return (err as NodeJS.ErrnoException).code === "ESRCH"; + } + } + private read(): string | null { try { const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, ""); @@ -258,11 +345,26 @@ export class MintedToken implements TokenSource { if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status); if (res.status === 400 || res.status === 422) { // The forge still holds a token by this name whose value we no longer have — the kept file - // went while the forge's data stayed. It is ours to replace: drop it by name and mint again. - this.log(`the forge already holds a token named "${this.name}" — replacing it`); - const dropped = await call("DELETE", `/${encodeURIComponent(this.name)}`); - if (dropped.status !== 204 && dropped.status !== 404) { - throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${dropped.status} ${detail(dropped.body)}`); + // went while the forge's data stayed, or two processes once minted at once and left two. It is + // ours to replace: find every one by this name in the account's list and delete each by its id + // (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint. + const held: number[] = []; + for (let page = 1; page <= 100; page++) { + const listed = await call("GET", `?page=${page}&limit=50`); + if (listed.status !== 200 || !Array.isArray(listed.body)) { + throw new Error(`Gitea GET /users/${this.opts.admin}/tokens: ${listed.status} ${detail(listed.body)}`); + } + if (listed.body.length === 0) break; + for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id); + } + this.log(held.length === 1 + ? `the forge already holds a token named "${this.name}" — replacing it` + : `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`); + for (const id of held) { + const dropped = await call("DELETE", `/${id}`); + if (dropped.status !== 204 && dropped.status !== 404) { + throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`); + } } res = await call("POST", "", { name: this.name, scopes: this.scopes }); } From 3e16c055be8ec4744da084b459ff54aba401a78a Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 21:03:53 +0000 Subject: [PATCH 04/11] Let an agent's session run the TypeScript check outside the toolchain image (issue 742) Outside the image, assemble the same node_modules in the check's own workspace from mesh-tools' lock on the forge, so a module's TypeScript is checked locally before a push. Co-Authored-By: Claude Opus 5.5 --- merge-check-typescript.sh | 44 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/merge-check-typescript.sh b/merge-check-typescript.sh index 3b0ed856..b32bb08f 100644 --- a/merge-check-typescript.sh +++ b/merge-check-typescript.sh @@ -23,9 +23,51 @@ # # By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads # /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same. +# +# **In an agent's session, outside the toolchain image** (novox/hq issue 742), the same node_modules is +# assembled first, in a throwaway directory under the check's own workspace, the way the image's `deps` stage +# assembles it: the toolchain's own package.json and lock (mesh-tools' node-tools/, read from the forge this +# clone came from, so the list is never copied here) installed with `npm ci`, then the SDK the mesh last +# published (`@novox/mesh-sdk@latest`), the @novox scope from the forge's package registry, read without a +# credential. Node and npm are the dev-tools module's; nothing is installed on the machine, no npm +# configuration or cache of the account's is read or written. Only when neither TOOLCHAIN_MODULES nor +# /app/node_modules is there: the forge's check, in the image, fetches nothing, as before. The forge is +# MESH_FORGE (default: the origin remote's), the owner MESH_FORGE_OWNER (default: the origin's owner). set -eu +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + deps="${TOOLCHAIN_MODULES:-/app/node_modules}" +if [ -z "${TOOLCHAIN_MODULES:-}" ] && [ ! -f "$deps/typescript/bin/tsc" ]; then + origin=$(git remote get-url origin 2>/dev/null || true) + forge="${MESH_FORGE:-$(printf '%s\n' "$origin" | sed -n 's#^\(https\{0,1\}://[^/]*\)/.*#\1#p')}" + owner="${MESH_FORGE_OWNER:-$(printf '%s\n' "$origin" | sed -n 's#^https\{0,1\}://[^/]*/\([^/]*\)/.*#\1#p')}" + if [ -z "$forge" ] || [ -z "$owner" ]; then + echo "NOT CHECKED: no TypeScript toolchain at $deps, and no forge to assemble one from (origin is '$origin'; set MESH_FORGE and MESH_FORGE_OWNER, or TOOLCHAIN_MODULES)" + exit 1 + fi + echo "no TypeScript toolchain at $deps: assembling one from $forge/$owner/mesh-tools node-tools/ and $owner's package registry (novox/hq issue 742)" + tc="$work/toolchain" + mkdir -p "$tc" + printf '@%s:registry=%s/api/packages/%s/npm/\n' "$owner" "$forge" "$owner" >"$work/npmrc" + # Read with Node's own fetch (dev-tools declares Node, not curl); a status that is not 2xx fails. + if ! { node -e ' + const [base, dir] = process.argv.slice(1); + Promise.all(["package.json", "package-lock.json"].map(async (f) => { + const r = await fetch(base + f); + if (!r.ok) throw new Error(base + f + ": " + r.status); + require("fs").writeFileSync(dir + "/" + f, await r.text()); + })).catch((e) => { console.error(e.message); process.exit(1); });' \ + "$forge/$owner/mesh-tools/raw/branch/main/node-tools/" "$tc" && + (cd "$tc" && export npm_config_userconfig="$work/npmrc" npm_config_cache="$work/npm-cache" && + npm ci --ignore-scripts --no-audit --no-fund --loglevel=error && + npm install --ignore-scripts --no-audit --no-fund --no-save --loglevel=error "@$owner/mesh-sdk@latest"); }; then + echo "NOT CHECKED: the TypeScript toolchain could not be assembled from $forge (novox/hq issue 742)" + exit 1 + fi + deps="$tc/node_modules" +fi tsc="$deps/typescript/bin/tsc" if [ ! -f "$tsc" ]; then echo "NOT CHECKED: no TypeScript compiler at $tsc — this part runs in the TypeScript toolchain" @@ -39,8 +81,6 @@ touched=$({ printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p' } | sed '/^$/d' | sort -u) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT failed="" for m in $touched; do d="modules/$m" From 6f7f07635e0297256d48ad9e528c716d55eeaea3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 23:15:38 +0200 Subject: [PATCH 05/11] Cover a failed or stopped delivery once a later finished walk contains it (hq issue 747) A group member whose walk failed in an outage stayed failed after a later walk of its repository, containing its merge, was done; its group then stopped every later walk of the other repository, and the catalogue could deliver nothing. A failed or stopped delivery now reads covered when a done walk of its repository, made after its merge, whose commit contains it (the forge's word, as issue 362's containment), built every module it moves. A group counts it as delivered, and a member the group stopped for it no longer holds a walk back: the group stops a walk only for a member whose change is still undelivered. A stop covered under issue 480 now likewise makes its group read reached, not stopped. While the forge is asked, the walk is held back rather than stopped, for at most ten minutes from the first ask: a forge that keeps erroring or cannot say then lets the walk be asked to leave the stopped merge out, as before. The hold is said by waits, stalled and stalled walks. The pass reads kept walks each tick, so today's stuck deliveries clear after a restart. --- .../cmd/mesh-delivery/answers.go | 10 + .../cmd/mesh-delivery/effects.go | 5 +- .../mesh-delivery/cmd/mesh-delivery/holder.go | 243 ++++++++++++++++- .../cmd/mesh-delivery/issue747_test.go | 258 ++++++++++++++++++ .../mesh-delivery/cmd/mesh-delivery/model.go | 83 +++++- .../mesh-delivery/cmd/mesh-delivery/verbs.go | 4 + 6 files changed, 585 insertions(+), 18 deletions(-) create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go diff --git a/modules/mesh-delivery/cmd/mesh-delivery/answers.go b/modules/mesh-delivery/cmd/mesh-delivery/answers.go index 3323a5c5..e3c65cb3 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/answers.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/answers.go @@ -298,6 +298,9 @@ func (h *Holder) orphanLines(now time.Time) []StalledLine { Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+ "follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+ "starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)}) + if hold := h.coverHeld(id); hold != "" { + out[len(out)-1].Says += "; it is held back meanwhile: " + hold + } } return out } @@ -388,6 +391,9 @@ func (h *Holder) words() []func() { if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 { h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " + "asked to leave that merge out first" + if hold := h.coverHeld(walk); hold != "" { + h.waitsWhy[walk] = hold + } h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk, strings.Join(stopped, ", ")) continue @@ -479,6 +485,10 @@ func (h *Holder) WalksWaiting() []StalledLine { switch { case len(held) > 0: out = append(out, held...) + case h.coverHeld(id) != "": + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: coverAskBound.String(), H2: "none: it is asked to leave the merge out past the bound", + Says: "held back: " + h.coverHeld(id)}) case h.waitsWhy[id] != "": out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]}) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/effects.go b/modules/mesh-delivery/cmd/mesh-delivery/effects.go index e7db65f2..b7a2115f 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/effects.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/effects.go @@ -150,6 +150,9 @@ func statusOf(d *Delivery) (string, string) { case Delivered: return "success", "delivered" case Failed: + if d.coveredStop() { + return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 747) + } return "failure", clip("failed: "+why, 140) case Superseded: return "warning", clip("superseded: "+why, 140) @@ -178,7 +181,7 @@ func ViewBody(d *Delivery, g *Group) string { var b strings.Builder fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339)) if d.coveredStop() { - fmt.Fprintf(&b, "**Covered** after its stop: a later merge that contains it was delivered: %s\n\n", d.CoveredBy) + fmt.Fprintf(&b, "**Covered**: %s\n\n", d.coveredStopSaid()) } if d.HeldWhy != "" && d.State == Held { fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index da430dc2..383e2d67 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -67,6 +67,13 @@ type Holder struct { // leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and // to stop the walk whole. leaveRefused map[string]string + // coverAsked are the deliveries that failed or were stopped whose cover by a later finished walk waits for the + // forge's word on containment, worked out each pass (novox/hq issue 747): a walk carrying a merge stopped for one of + // them is held back meanwhile, never asked to leave it out before the answer. + coverAsked map[string]bool + // coverAskedSince is, by delivery, when the forge was first asked about its cover and has not answered: past + // coverAskBound the walk is no longer held back for it, and is asked to leave the merge out (review of issue 747). + coverAskedSince map[string]time.Time // tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407). tried owedTries // marks are the open pull requests read with a ready mark, by owner/repo#number (novox/hq issue 722, ready.go). @@ -1112,6 +1119,48 @@ func (h *Holder) plan() []func() { // A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what // that walk did not build, a later delivered merge may have; and when the cover test says none did, past the // grace, it is stopped saying so — never left waiting for a start that cannot come. + // A delivery that failed or was stopped is read for a later finished walk of its repository that delivered its + // change after all: a walk that failed in an outage and was resumed, or the walk of a later merge that contains it + // (novox/hq issue 747). Read each tick, from what the walks and the kept deliveries say, so one stuck before this + // was built clears by itself after a restart. + h.coverAsked = map[string]bool{} + if h.coverAskedSince == nil { + h.coverAskedSince = map[string]time.Time{} + } + pending := map[string]bool{} + for _, d := range h.deliveries { + if d.State != Failed && d.State != Stopped || d.coveredStop() { + continue + } + by, ask := h.walkCover(d) + if len(ask) > 0 && by == "" { + pending[d.ID] = true + since, asked := h.coverAskedSince[d.ID] + if !asked { + since = now + h.coverAskedSince[d.ID] = now + } + // Held back only within the bound: a forge that keeps erroring never holds a walk for good. + h.coverAsked[d.ID] = now.Sub(since) < coverAskBound + } + if by != "" { + d.DeliveredBy = by + h.tellCovered(d, now) + continue + } + for _, in := range ask { + if a := h.askContains(d, in, now); a != nil { + asks = append(asks, a) + } + } + } + + for id := range h.coverAskedSince { + if !pending[id] { + delete(h.coverAskedSince, id) + } + } + for _, d := range h.deliveries { ended := d.endedUnstarted() if d.coveredStop() && !d.CoverTold { @@ -1273,7 +1322,7 @@ func (h *Holder) laterOnItsWay(d *Delivery) string { // stays final, and the bus's transition event is not said again. func (h *Holder) tellCovered(d *Delivery, now time.Time) { d.CoverTold = true - line := oneLine(fmt.Sprintf("%s delivery %s: stopped, %s", now.UTC().Format(time.RFC3339), d.ID, d.coveredStopSaid())) + line := oneLine(fmt.Sprintf("%s delivery %s: %s, %s", now.UTC().Format(time.RFC3339), d.ID, d.State, d.coveredStopSaid())) d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now}) if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) { d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now}) @@ -1283,7 +1332,7 @@ func (h *Holder) tellCovered(d *Delivery, now time.Time) { state, desc := statusOf(d) d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now}) } - h.Logf("[mesh-delivery] %s: stopped, %s", d.ID, d.coveredStopSaid()) + h.Logf("[mesh-delivery] %s: %s, %s", d.ID, d.State, d.coveredStopSaid()) h.keep(d) } @@ -1294,6 +1343,104 @@ func strandedUncovered(d *Delivery) bool { return d.State == Stopped && d.Stranded != "" && d.CoveredBy == "" } +// walkCover says which later finished walk delivered the change of a delivery that failed or was stopped, or +// nothing, and which walk commits the forge must still be asked about (novox/hq issue 747). A walk covers it when it +// is of its repository and trunk, done, made after its merge, its commit containing its merge — the merge itself, or +// a later one the forge says contains it — and it built every module the delivery's plan moves, each it did not +// build being one the delivery's own walk said unchanged (issue 611). This is the containment "one walk answers every +// merge it contains" uses (issue 362, ADR 0276), read after the fact: a failed walk resumed to done answers its own +// merge, and a later merge's walk answers every merge it contains. A walk that only said a module unchanged for its +// own merges says nothing of this one's, so it covers no module that way. Walks are read from the controller's records +// this owner holds and from every delivery's kept walk, so a walk long ended still counts after a restart. +func (h *Holder) walkCover(d *Delivery) (string, []string) { + if d.State != Failed && d.State != Stopped || d.MergedAs == "" || d.Plan == nil || len(d.Plan.Moved) == 0 { + return "", nil + } + var ask []string + for _, c := range h.doneWalksOf(d.Repository) { + w := c.walk + if !sameBranch(w.Branch, d.Base) || !w.Created.IsZero() && w.Created.Before(d.MergedAt.Add(-time.Minute)) { + continue + } + var missing []string + for _, m := range d.Plan.Moved { + unchanged := false + if d.Walk != nil { + _, unchanged = d.Walk.Unchanged[m] + } + if !containsString(w.Moved, m) && !unchanged { + missing = append(missing, m) + } + } + if len(missing) > 0 { + continue + } + if !sameCommit(c.commit, d.MergedAs) { + yes, known := d.ContainedIn[c.commit] + if !known && !containsString(ask, c.commit) { + ask = append(ask, c.commit) + } + if !yes { + continue + } + } + return fmt.Sprintf("%s (%s at %s into %s, done, contains its merge %s)", w.ID, d.Repository, shortOf(c.commit, 8), + orNone(d.Base), shortOf(d.MergedAs, 8)), nil + } + return "", ask +} + +// doneWalk is a finished walk and the commit of its repository it carried. +type doneWalk struct { + walk *WalkSeen + commit string +} + +// doneWalksOf are the finished walks of a repository this owner knows, oldest first: the controller's records it +// holds, and each delivery's kept walk with the commit of its repository that walk carried for it. +func (h *Holder) doneWalksOf(repository string) []doneWalk { + byID := map[string]doneWalk{} + for id, s := range h.walks { + w := s.walk + if w.Release != nil || w.IsBatch() || w.State != walkDone || !strings.EqualFold(w.Repository, repository) { + continue + } + byID[id] = doneWalk{walk: w.Seen(), commit: w.Commit} + } + for _, o := range h.deliveries { + if o.Walk == nil || o.Walk.State != walkDone || !strings.EqualFold(o.Repository, repository) { + continue + } + if _, known := byID[o.Walk.ID]; known { + continue + } + commit := o.AnsweredBy + if commit == "" && o.MergedAs != "" && sameCommit(o.Walk.Commit, o.MergedAs) { + commit = o.MergedAs + } + if commit == "" { + continue + } + byID[o.Walk.ID] = doneWalk{walk: o.Walk, commit: commit} + } + out := make([]doneWalk, 0, len(byID)) + for _, c := range byID { + out = append(out, c) + } + sort.Slice(out, func(i, j int) bool { + if !out[i].walk.Created.Equal(out[j].walk.Created) { + return out[i].walk.Created.Before(out[j].walk.Created) + } + return out[i].walk.ID < out[j].walk.ID + }) + return out +} + +// stopLapsed is whether a member its group stopped for another is stopped for nothing any more (see stopLapsed). +func (h *Holder) stopLapsed(d *Delivery) bool { + return stopLapsed(d, func(id string) *Delivery { return h.deliveries[id] }) +} + // unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read // again after it: what holds it for a person, unless something already ends it. func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool { @@ -1736,7 +1883,8 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa // A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a // covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq // issues 442 and 445). - if d.State.Reached() || d.State == Superseded && d.CoveredBy != "" { + // A stop or failure a later walk delivered since counts as reached too (novox/hq issues 480 and 747). + if memberReached(d) { continue } if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) { @@ -1761,7 +1909,7 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa } if w.why != "" { for _, later := range ordered[i+1:] { - if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID, + if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID, d.ID, w.why)); a != nil { asks = append(asks, a) } @@ -1788,9 +1936,13 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa continue } return asks + case h.stopLapsed(d): + // Stopped for a member whose change reached the machines since: it stops nothing more. Its own change is + // delivered by a later walk of its repository that contains it, which then covers it (novox/hq issue 747). + continue case d.State == Failed || d.State == Stopped: for _, later := range ordered[i+1:] { - if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil { + if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil { asks = append(asks, a) } } @@ -2028,7 +2180,7 @@ func mayCarry(c, d *Delivery, module string) bool { // controller ends the walk and walks the other merges again without it, and without each later merge of its // repository, which contains it. Every member a pass stops on one walk is gathered first, and the walk is asked // once, by the first member's ask, after the pass: a member is never walked again because its stop came second. -func (h *Holder) stopInGroup(d *Delivery, why string) func() { +func (h *Holder) stopInGroup(d *Delivery, cause, why string) func() { if d == nil || d.State.Final() { return nil } @@ -2037,6 +2189,7 @@ func (h *Holder) stopInGroup(d *Delivery, why string) func() { h.refuse(err) return nil } + d.StoppedFor = cause // the member it was stopped for: once that one is delivered, the stop lapses (novox/hq issue 747) h.moved(d, []Transition{t}) // Its open walk; or, its walk stopped for another merge, the batch or walk that walks its merge again (review // of #221): a merge walked again and then stopped by its group is left out there too. @@ -2118,6 +2271,12 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) { if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" { continue } + // A stop whose change a later walk delivered since, or one its group made for a member delivered since, + // holds nothing back: the group stops a walk only for a member whose change is still undelivered (novox/hq + // issue 747). + if d.coveredStop() || h.stopLapsed(d) { + continue + } // By the merges it names alone: its own commit may be another's merge it no longer answers. if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool { return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) @@ -2135,6 +2294,51 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) { return out, why } +// coverAwaited is whether a stopped merge a walk carries, given as `@`, waits for the forge's +// word on whether a later finished walk delivered it or the member its group stopped it for (novox/hq issue 747). +// It says since when the forge was first asked, the earliest of them. Held only within coverAskBound of the first +// ask: past it the forge's silence or error lets the walk be asked to leave the merge out. Under the lock. +func (h *Holder) coverAwaited(merges []string) (time.Time, bool) { + var since time.Time + awaited := false + for _, d := range h.deliveries { + if d.State != Stopped || d.MergedAs == "" || !containsString(merges, d.Repository+"@"+d.MergedAs) { + continue + } + for _, id := range []string{d.ID, d.stoppedFor()} { + if id == "" || !h.coverAsked[id] { + continue + } + if at := h.coverAskedSince[id]; !awaited || at.Before(since) { + since = at + } + awaited = true + } + } + return since, awaited +} + +// coverHeld says why a walk is held back while the forge is asked whether a later finished walk delivered a stopped +// merge it carries (novox/hq issue 747), or nothing when it is not. Said by `waits`, `stalled` and `stalled walks`, so +// a forge that cannot answer is loud. Under the lock. +func (h *Holder) coverHeld(walk string) string { + without, _ := h.carriesStopped(walk) + if len(without) == 0 { + return "" + } + since, awaited := h.coverAwaited(without) + if !awaited { + return "" + } + return fmt.Sprintf("it carries %s, whose delivery was stopped, and the forge is asked since %s whether a later "+ + "finished walk delivered it; it is asked to leave the merge out if no answer comes by %s (novox/hq issue 747)", + strings.Join(without, ", "), since.Local().Format("15:04:05"), since.Add(coverAskBound).Local().Format("15:04:05")) +} + +// coverAskBound is how long a walk carrying a stopped merge is held back while the forge is asked whether a later +// finished walk delivered it (novox/hq issue 747); past it, the walk is asked to leave the merge out. +const coverAskBound = 10 * time.Minute + // lastWhyOf is the reason of a delivery's last transition. func lastWhyOf(d *Delivery) string { if len(d.Transitions) == 0 { @@ -2170,6 +2374,9 @@ func (h *Holder) leaveStopped() []func() { if without, _ := h.carriesStopped(id); len(without) > 0 { said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+ "leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", ")) + if hold := h.coverHeld(id); hold != "" { + said = "its walk " + id + " is held back: " + hold + } if r := h.leaveRefused[id]; r != "" { said += "; the controller refused: " + r } @@ -2193,6 +2400,9 @@ func (h *Holder) leaveStopped() []func() { delete(h.leaveRefused, id) continue } + if _, awaited := h.coverAwaited(without); awaited { + continue // held back, not stopped, until the forge says whether a later walk delivered it (issue 747) + } // Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should // the ask have gone nowhere. if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute { @@ -2314,18 +2524,28 @@ func againWords(a StopAnswer) string { func GroupState(g *Group, members []*Delivery) (string, string) { count := map[State]int{} reached, ended := 0, 0 - var uncovered []string + var uncovered, lapsed []string + byID := map[string]*Delivery{} + for _, d := range members { + byID[d.ID] = d + } for _, d := range members { - count[d.State]++ if d.State.Final() { ended++ } switch { - case d.State.Reached(), d.State == Superseded && d.CoveredBy != "": + case memberReached(d): + // Reached, whatever its final state: a failure or stop a later walk delivered since is no failure of the + // group (novox/hq issue 747). reached++ + continue + case stopLapsed(d, func(id string) *Delivery { return byID[id] }): + lapsed = append(lapsed, d.ID) + continue case d.State == Superseded: uncovered = append(uncovered, d.ID) } + count[d.State]++ } n := len(members) switch { @@ -2335,6 +2555,11 @@ func GroupState(g *Group, members []*Delivery) (string, string) { return "stopped", "a member was stopped" case n > 0 && reached == n: return "delivered", "every member is delivered" + case g.Closed && len(lapsed) > 0 && reached+len(lapsed) == n: + // Stopped for a member delivered since: its own change waits for a later walk of its repository, which then + // covers it (novox/hq issue 747). + return "delivering", fmt.Sprintf("%d of %d delivered; %s, stopped for a member delivered since, waits for a "+ + "later walk of its repository", reached, n, strings.Join(lapsed, ", ")) case g.Closed && n > 0 && ended == n: return "stopped", supersededWhy + strings.Join(uncovered, ", ") case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" || diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go new file mode 100644 index 00000000..7646ec14 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go @@ -0,0 +1,258 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336 +// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge, +// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still +// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge +// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of +// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is +// still undelivered. +// +// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the +// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge. + +const ( + g747 = "fix/a-module-moves-only-when-its-program-changes" + ctl747, cat747 = "novox/mesh-controller", "novox/mesh-catalog" + head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452" + head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6" + head301, merge301 = "d94764f00168471f", "2edef5a00ae05605" + merge307 = "0f76b6e2aaaaaaaa" +) + +var ( + at747 = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC) + id336, id339, id301 = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301) + unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"} +) + +// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df. +func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) } + +// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged. +func night747With(t *testing.T, contained bool, built []string) *world { + t.Helper() + return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built) +} + +// night747Over is night747With with the forge set as given: one that is not told the pair cannot say. +func night747Over(t *testing.T, set func(*fakeForge), built []string) *world { + t.Helper() + at := at747 + failedAt := at.Add(11 * time.Minute) + store := newMemStore() + moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747 + if built != nil { + moved339, unchanged339 = built, map[string]string{} + for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} { + if !containsString(built, m) { + unchanged339[m] = "its build source holds none of the files #339 changed" + } + } + } + m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747, + State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at, + AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, + Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"), + Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true, + Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2, + Unchanged: unchangedRouteProxy747}, + Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed, + Why: "its walk failed, carrying its own commit"}}} + m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726", + State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute), + AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"), + // Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so. + Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true, + Moved: moved339, Unchanged: unchanged339, + Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2}, + Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed, + Why: "its walk failed, carrying its own commit"}}} + m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747, + State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), + Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"}}} + for _, d := range []*Delivery{m336, m339, m301} { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Order: []string{id336, id301}, Said: "failed", + OrderOf: commitsOf([]*Delivery{m301, m336}), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute), + Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + // The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word. + ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}}) + ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute) + forge := newFakeForge() + set(forge) + w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w +} + +func groupState747(w *world) (string, string) { + g := w.h.groups[g747] + return GroupState(g, w.h.membersOf(g)) +} + +// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the +// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged. +// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose +// own walk was resumed to done, is covered by it too. +func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) { + w := night747(t, true) + d := w.delivery(id336) + if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") || + !strings.Contains(desc, "plan-339") { + t.Fatalf("#336's status reads %s %q", state, desc) + } + if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") { + t.Fatalf("the forge was told %q of #336", s) + } + if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") { + t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc) + } + if state, why := groupState747(w); state == "failed" || state == "stopped" { + t.Fatalf("the group reads %s: %s", state, why) + } + // Told once, and kept: a restart tells nothing again. + notes := len(w.forge.notesOn(merge336)) + again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute)) + again.settleAll() + if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" { + t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes) + } +} + +// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone, +// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the +// group waits for that walk to deliver the catalogue member's change. +func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) { + w := night747(t, true) + if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") { + t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 { + t.Fatalf("the catalogue's walk is held back for %v", stopped) + } + if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) { + t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why) + } + // Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is + // delivered. + w.forge.contains[merge301+" in "+merge307] = true + r := w.ctl.walk("plan-cat") + sent := w.now + r.Modules = map[string]*WalkModule{} + for _, m := range []string{"build-agent", "route-proxy", "gitea"} { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = walkDone + w.h.WalkMoved(w.ctl.put(r)) + w.later(time.Minute) + w.settleAll() + if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") { + t.Fatalf("the catalogue member reads %s %q", state, desc) + } + if state, why := groupState747(w); state != "delivered" { + t.Fatalf("the group reads %s: %s", state, why) + } +} + +// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change +// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries +// the stopped member's merge is asked to leave it out. +func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) { + w := night747(t, false) + d := w.delivery(id336) + if state, desc := statusOf(d); d.State != Failed || state != "failure" { + t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc) + } + if state, _ := groupState747(w); state != "failed" { + t.Fatalf("the group reads %s", state) + } + if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) { + t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts) + } +} + +// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk +// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only +// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue. +func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) { + w := night747With(t, true, []string{"mesh-controller"}) + if state, desc := statusOf(w.delivery(id336)); state != "failure" { + t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc) + } + if state, _ := groupState747(w); state != "failed" { + t.Fatalf("the group reads %s", state) + } + if len(w.ctl.withouts) == 0 { + t.Fatal("the catalogue's walk was not asked to leave the stopped merge out") + } +} + +// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on +// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the +// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk +// is asked to leave the stopped merge out, as with no later walk at all. +func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) { + for _, c := range []struct { + name string + set func(*fakeForge) + }{ + {"erroring", func(f *fakeForge) { f.containsDown = true }}, + {"not answering for the pair", func(*fakeForge) {}}, + } { + t.Run(c.name, func(t *testing.T) { + w := night747Over(t, c.set, nil) + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + if state, desc := statusOf(w.delivery(id336)); state != "failure" { + t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc) + } + said := "" + for _, l := range w.h.WalksWaiting() { + if l.Walk == "plan-cat" { + said += l.Says + } + } + w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it + w.settleAll() + for _, l := range w.h.Stalled() { + if l.Walk == "plan-cat" { + said += " | " + l.Says + } + } + if strings.Count(said, "the forge is asked since") != 2 { + t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said) + } + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + w.later(coverAskBound + time.Minute) + w.settleAll() + if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) { + t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts) + } + }) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index ec3e2e1d..2d2a0c67 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -83,6 +83,14 @@ type Delivery struct { // CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note // (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too. CoverTold bool `json:"cover_told,omitempty"` + // DeliveredBy says, for a delivery that failed or was stopped, the later finished walk of its repository whose + // commit contains its merge and which built every module it moves — each it did not build, its own walk said + // unchanged: its change reached the machines after all (novox/hq issue 747). Its state stays final, as a covered + // stop's does (issue 480), and it reads covered: its group counts it as delivered. + DeliveredBy string `json:"delivered_by,omitempty"` + // StoppedFor is, for a member its group stopped, the member whose end stopped it (novox/hq issue 747): once that + // member's change reached the machines, the stop has lapsed, and holds no walk of its repository back. + StoppedFor string `json:"stopped_for,omitempty"` // TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it, // which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9, // novox/hq issue 445). `show` shows both, and `stalled` says them past that moment. @@ -303,18 +311,77 @@ func (d *Delivery) FromForgesWord() bool { // shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose // pull request the forge may no longer hold — or holds under its number another one. -// coveredStop is whether a delivery stopped because its walk ended done before its word without building its modules -// (novox/hq issue 474) was covered since: a later merge into its trunk that contains it was delivered, carrying every -// module it moves to every machine running it (ADR 0270's cover test, kept as CoveredBy). Its state stays stopped — -// final, as a covered superseded member stays superseded (issue 442) — and it reads covered, by that merge (novox/hq -// issue 480). A stop for any other reason never reads covered. +// coveredStop is whether a delivery that ended stopped or failed was covered since, and reads covered. A stop +// because its walk ended done before its word without building its modules (novox/hq issue 474) is covered when a +// later merge into its trunk that contains it was delivered, carrying every module it moves to every machine running +// it (ADR 0270's cover test, kept as CoveredBy; issue 480). Any stop or failure is covered when a later finished walk +// of its repository whose commit contains its merge built its modules (DeliveredBy, novox/hq issue 747). Its state +// stays final, as a covered superseded member stays superseded (issue 442), and it reads covered, by that merge or +// walk. A person's stop with a cover found only while it was held keeps its error. func (d *Delivery) coveredStop() bool { - return d.State == Stopped && d.Stranded != "" && d.CoveredBy != "" + switch d.State { + case Stopped: + return d.Stranded != "" && d.CoveredBy != "" || d.DeliveredBy != "" + case Failed: + return d.DeliveredBy != "" + } + return false } -// coveredStopSaid is how a covered stop reads: on the forge's status, in its view and note, and in `show`. +// coveredStopSaid is how a covered stop or failure reads: on the forge's status, in its view and note, and in `show`. func (d *Delivery) coveredStopSaid() string { - return "covered after its stop: a later merge that contains it was delivered: " + d.CoveredBy + ended := "its stop" + if d.State == Failed { + ended = "its failure" + } + if d.Stranded != "" && d.CoveredBy != "" { + return "covered after " + ended + ": a later merge that contains it was delivered: " + d.CoveredBy + } + return "covered after " + ended + ": a later walk that contains it was delivered: " + d.DeliveredBy +} + +// memberReached is whether a group counts a member as delivered: its change reached the machines, or had nothing to +// reach them with — delivered, history or covered, a superseded member a later delivered walk carried (issue 442), +// or a stop or failure covered since (issues 480 and 747). +func memberReached(d *Delivery) bool { + return d.State.Reached() || d.State == Superseded && d.CoveredBy != "" || d.coveredStop() +} + +// stoppedFor is the member whose end stopped this one in its group: kept since novox/hq issue 747, and read from the +// stop's reason for a stop made before ("its group stopped: …"). +func (d *Delivery) stoppedFor() string { + if d.StoppedFor != "" || d.State != Stopped || d.Group == "" { + return d.StoppedFor + } + prefix := "its group " + d.Group + " stopped: " + for i := len(d.Transitions) - 1; i >= 0; i-- { + t := d.Transitions[i] + if t.Event != EvStop { + continue + } + if t.By != byOwner || !strings.HasPrefix(t.Why, prefix) { + return "" + } + member, _, _ := strings.Cut(strings.TrimPrefix(t.Why, prefix), " ") + return strings.TrimRight(member, ",;:") + } + return "" +} + +// stopLapsed is whether a member its group stopped for another member is stopped for nothing any more: that member's +// change reached the machines since — delivered, or its failure covered by a later walk (novox/hq issue 747). Its own +// change is still to deliver, by a later walk of its repository that contains it; its stop holds none back. find +// gives a delivery by its id. +func stopLapsed(d *Delivery, find func(string) *Delivery) bool { + if d.State != Stopped || d.coveredStop() { + return false + } + by := d.stoppedFor() + if by == "" || by == d.ID { + return false + } + o := find(by) + return o != nil && memberReached(o) } func (d *Delivery) shownOnItsPullRequest() bool { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go index b95aacd2..b2dfed42 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go @@ -72,6 +72,10 @@ func (h *Holder) waitsFor(d *Delivery) string { return "its walk: " + clip(d.Walk.Note, 120) } case Stopped: + if h.stopLapsed(d) { + return "a later walk of its repository that contains its merge: its group stopped it for " + d.stoppedFor() + + ", whose change was delivered since (novox/hq issue 747)" + } if strandedUncovered(d) { return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " + "listed by stalled until one does (novox/hq issue 480)" From 2833a049aa0f2408fb6708b0fdfe89b77bfb77fb Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 23:24:26 +0200 Subject: [PATCH 06/11] Make the forge's webhook a system one, and say whether its events are heard (hq issue 729) POST /admin/hooks on gitea 1.27.3 makes a default webhook unless config.is_system_webhook is "true": a default webhook fires for no existing repository, and the system listing does not name it, so every start made another and no event reached the receiver. Its edit never changes a secret or a kind, so the new system webhook is made first and read back, then every other hook at the receiver is deleted: system, default, and each repository's copy of a default one, which would send its events twice. gitea_events says where the receiver listens, the hook as ensured and what it replaced, the verified and refused deliveries and the polls' pace. When nothing verified for ten minutes, since the start or the last delivery, while the poll found two heads or merges no event said, it is logged loudly once per silent spell and answered unhealthy; so are deliveries refused since the last verified one. --- modules/gitea/index.ts | 34 ++-- modules/gitea/test/webhook.test.ts | 235 ++++++++++++++++++++++-- modules/gitea/webhook.ts | 277 +++++++++++++++++++++++++---- 3 files changed, 489 insertions(+), 57 deletions(-) diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 8efe7c73..b9c80da7 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -44,7 +44,7 @@ import { GiteaAdmin, GiteaClient, movedSince } from "./client.js"; import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked, type RepoCheckFacts, reposToLook } from "./pulls.js"; import { GateWriter, clonedAs, type BuildStarted } from "./gate.js"; -import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, keptSecret, receiver, type ForgeEvent } from "./webhook.js"; +import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js"; import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js"; // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is @@ -429,10 +429,16 @@ const gates = new GateWriter({ // **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own // process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to // reach, nothing is served, as the tools' bundle serves nothing then. -registerModuleTools("gitea", () => (gitea ? [commitStatusTool({ +/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what + * gitea_events says of them (webhook.ts). */ +const pace = new Pace(); +if (!gitea) pace.unheard("the forge is not watched: no way to a token", false); + +// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729). +registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({ setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status), gates, -}), mergeTool(gitea, { gates })] : [])); +}), mergeTool(gitea, { gates })] : [])]); // **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when // the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing. @@ -451,9 +457,6 @@ async function buildStarted(event: { body: unknown }): Promise { } } -/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are (webhook.ts). */ -const pace = new Pace(); - /** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */ async function actOn(client: GiteaClient, event: ForgeEvent): Promise { const b = event.body as { owner?: string; repo?: string; number?: number }; @@ -476,23 +479,32 @@ async function listenForForge(client: GiteaClient): Promise { const url = process.env.MESH_GITEA_HOOK_URL; if (!stateDir || !url) { console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge"); + pace.unheard("MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset", false); return false; } try { const secret = keptSecret(join(stateDir, "webhook.secret")); const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT); - const server = receiver({ secret, port, heard: () => pace.heard(), + const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0"; + const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery), + refused: (reason, from) => pace.refused(reason, from), publish: (event) => emit(event.name, event.body), act: (event) => actOn(client, event) }); await new Promise((resolve, reject) => { server.once("error", reject); - server.listen(port, process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0", () => resolve()); + server.listen(port, host, () => resolve()); }); - const did = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret); - console.log(`[gitea] listening for the forge's events on port ${port}; its system webhook ${did}, every event, signed`); + const hook = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret); + pace.listening(host, port, hook.url, hook); + console.log(`[gitea] listening for the forge's events on ${host}:${port}; its system webhook ${hook.id} ${hook.did} ` + + `at ${hook.url} (in place of ${hook.removed.system} system, ${hook.removed.default} default and ` + + `${hook.removed.repository} repository webhook(s)${hook.removed.repositories.length ? ` in ${hook.removed.repositories.join(", ")}` : ""}), ` + + `${hook.events.length} event families, signed`); return true; } catch (err) { - console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${err instanceof Error ? err.message : err}`); + const why = err instanceof Error ? err.message : String(err); + pace.unheard(why); + console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${why}`); return false; } } diff --git a/modules/gitea/test/webhook.test.ts b/modules/gitea/test/webhook.test.ts index 453ff2d0..1160de34 100644 --- a/modules/gitea/test/webhook.test.ts +++ b/modules/gitea/test/webhook.test.ts @@ -83,20 +83,235 @@ test("each forge event is named in the module's stream and trimmed", async () => assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other"); }); -test("the system webhook is made when none points at the receiver, and corrected when one does", async () => { - const { ensureSystemHook } = await import("../webhook.ts"); +// A forge's admin hooks as 1.27.3 keeps them: POST /admin/hooks makes a system webhook only when config.is_system_webhook +// is "true", and a default webhook otherwise; the listing names one kind at a time (type=system, the default, or default). +// Repositories hold their own webhooks (a default webhook's copies), listed a page at a time. +function fakeForge(start: { id: number; url: string; system: boolean }[] = [], + repoHooks: Record = {}, repoCount = 0) { + const hooks = start.map((h) => ({ ...h, events: ["push"], active: true })); + const repos = [...Object.keys(repoHooks)]; + for (let i = 0; repos.length < repoCount; i++) repos.push(`novox/r${i}`); + let next = 100; const calls: string[] = []; - let hooks: unknown[] = []; + const made: any[] = []; + const page = (items: any[], q: URLSearchParams) => { + const p = Number(q.get("page") ?? 1), l = Number(q.get("limit") ?? 50); + return items.slice((p - 1) * l, p * l); + }; const admin = { call: async (method: string, path: string, body?: any) => { calls.push(`${method} ${path}`); - if (method === "GET") return { status: 200, body: hooks }; - if (method === "POST") { assert.ok(body.events.includes("pull_request") && body.events.includes("push")); assert.equal(body.config.secret, secret); return { status: 201, body: {} }; } - return { status: 200, body: {} }; + const u = new URL(path, "http://f"); + const repoHook = u.pathname.match(/^\/repos\/([^/]+\/[^/]+)\/hooks(?:\/(\d+))?$/); + if (method === "GET" && u.pathname === "/repos/search") { + return { status: 200, body: { ok: true, data: page(repos.map((full_name) => ({ full_name })), u.searchParams) } }; + } + if (repoHook) { + const list = repoHooks[repoHook[1]] ?? []; + if (method === "GET") return { status: 200, body: page(list.map((h) => ({ id: h.id, config: { url: h.url } })), u.searchParams) }; + const i = list.findIndex((h) => h.id === Number(repoHook[2])); + if (i < 0) return { status: 404, body: {} }; + list.splice(i, 1); + return { status: 204, body: null }; + } + if (method === "GET") { + const type = u.searchParams.get("type") ?? "system"; + const listed = hooks.filter((h) => type === "all" || h.system === (type === "system")); + return { status: 200, body: listed.map((h) => ({ id: h.id, active: h.active, events: h.events, + config: { url: h.url, content_type: "json" } })) }; + } + if (method === "POST") { + made.push(body); + hooks.push({ id: next++, url: body.config.url, system: body.config.is_system_webhook === "true", events: body.events, + active: body.active }); + return { status: 201, body: {} }; + } + if (method === "DELETE") { + const id = Number(path.split("/").pop()); + const i = hooks.findIndex((h) => h.id === id); + if (i < 0) return { status: 404, body: {} }; + hooks.splice(i, 1); + return { status: 204, body: null }; + } + return { status: 405, body: {} }; } }; - assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "created"); - hooks = [{ id: 4, config: { url: "http://h:3047/forge-event" } }]; - assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "corrected"); - assert.ok(calls.includes("PATCH /admin/hooks/4")); + return { admin, hooks, calls, made, repoHooks }; +} + +test("the webhook is made a system webhook, which the forge sends every repository's events to", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const forge = fakeForge(); + const url = "http://h:3047/forge-event"; + const hook = await ensureSystemHook(forge.admin, url, secret); + assert.equal(forge.made.length, 1); + assert.equal(forge.made[0].config.is_system_webhook, "true", "without it the forge makes a default webhook, which fires for no repository"); + assert.equal(forge.made[0].config.secret, secret); + assert.ok(forge.made[0].events.includes("pull_request") && forge.made[0].events.includes("push")); + assert.deepEqual(forge.hooks.map((h) => h.system), [true]); + assert.equal(hook.kind, "system"); + assert.equal(hook.did, "created"); + assert.equal(hook.url, url); + assert.ok(hook.events.includes("pull_request")); +}); + +test("default webhooks left at the receiver by an earlier start are replaced by one system webhook", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const url = "http://h:3047/forge-event"; + // What the start at 23:05 left: a default webhook, invisible to the system listing, made again by every start. + const forge = fakeForge([{ id: 4, url, system: false }, { id: 5, url, system: false }, { id: 6, url: "http://other/x", system: true }]); + const hook = await ensureSystemHook(forge.admin, url, secret); + assert.deepEqual(hook.removed, { system: 0, default: 2, repository: 0, repositories: [] }); + assert.equal(hook.did, "replaced"); + assert.deepEqual(forge.hooks.map((h) => [h.url, h.system]), [["http://other/x", true], [url, true]], "another url is kept"); + // A second start replaces the system webhook, so the secret kept now is the one the forge signs with (its edit never + // changes a secret). + const again = await ensureSystemHook(forge.admin, url, secret); + assert.deepEqual(again.removed, { system: 1, default: 0, repository: 0, repositories: [] }); + assert.equal(forge.hooks.filter((h) => h.url === url).length, 1); + assert.ok(forge.calls.includes("DELETE /admin/hooks/4") && forge.calls.includes("DELETE /admin/hooks/5")); + // The new one is made before any old one is deleted, so no event falls in a gap. + const post = forge.calls.indexOf("POST /admin/hooks"); + assert.ok(post >= 0 && forge.calls.findIndex((c) => c.startsWith("DELETE")) > post); +}); + +test("a repository's copy of a default webhook at the receiver is deleted, on every page, and said", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const url = "http://h:3047/forge-event"; + // A repository made after the default webhook got its copy, which would send its events a second time; 120 + // repositories, so the copies sit beyond the first page. + const repoHooks: Record = {}; + for (let i = 0; i < 118; i++) repoHooks[`novox/r${i}`] = []; + repoHooks["novox/late"] = [{ id: 31, url }, { id: 32, url: "http://ci/hook" }]; + repoHooks["novox/later"] = [{ id: 33, url }]; + const forge = fakeForge([{ id: 4, url, system: false }], repoHooks); + const hook = await ensureSystemHook(forge.admin, url, secret); + assert.deepEqual(hook.removed, { system: 0, default: 1, repository: 2, repositories: ["novox/late", "novox/later"] }); + assert.deepEqual(forge.repoHooks["novox/late"].map((h) => h.id), [32], "a repository's other webhook is kept"); + assert.deepEqual(forge.repoHooks["novox/later"], []); + assert.ok(forge.calls.some((c) => /\/repos\/search\?.*page=3/.test(c)), "every page of repositories is read"); +}); + +test("a forge that makes no system webhook is said, never taken as heard", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const forge = fakeForge(); + const admin = { call: async (method: string, path: string, body?: any) => { + // A forge that ignores the flag: what 23:05 met, had the module read back. + if (method === "POST") body = { ...body, config: { ...body.config, is_system_webhook: undefined } }; + return forge.admin.call(method, path, body); + } }; + await assert.rejects(ensureSystemHook(admin, "http://h:3047/forge-event", secret), /names 0 new system webhooks/); +}); + +test("the receiver tells what it verified and what it refused", async () => { + const { receiver, HOOK_PATH } = await import("../webhook.ts"); + const heard: string[] = [], refused: string[] = []; + const server = receiver({ secret, port: 0, log: () => {}, publish: async () => {}, + heard: (kind, delivery) => heard.push(`${kind} ${delivery}`), refused: (reason) => refused.push(reason) }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`; + try { + await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-9" } }); + await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "x") } }); + await server.idle(); + assert.deepEqual(heard, ["pull_request d-9"]); + assert.deepEqual(refused, ["its signature does not verify"]); + } finally { server.close(); } +}); + +test("a forge that never delivers is said loudly once, and unhealthy, when the poll finds what no event said", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const pace = new Pace(() => now, (l) => said.push(l)); + const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true, + did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } }; + pace.listening("172.17.0.1", 3047, hook.url, hook); + now += 5 * 60_000; + pace.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(said.length, 0, "five minutes after the start is not yet silence"); + assert.equal((pace.report() as any).healthy, true); + pace.refused("it is unsigned", "172.17.0.3"); + assert.equal((pace.report() as any).healthy, false, "refused, and none verified"); + now += 6 * 60_000; + pace.miss("the merge of novox/hq#7"); + pace.miss("novox/hq#8 at bbbbbbbb"); + assert.equal(said.filter((l) => /NOT HEARD: no delivery ever verified/.test(l)).length, 1, "said once"); + let r = pace.report() as any; + assert.equal(r.healthy, false); + assert.equal(r.state, "unhealthy"); + assert.match(r.why, /never delivered/); + assert.equal(r.polls.found_unsaid, 3); + assert.equal(r.refused.count, 1); + assert.equal(r.refused.last.reason, "it is unsigned"); + assert.equal(r.hook.kind, "system"); + assert.equal(r.receiver.port, 3047); + // The first delivery clears it. + now += 1000; + pace.heard("push", "d-1"); + r = pace.report() as any; + assert.equal(r.healthy, true); + assert.equal(r.verified.count, 1); + assert.equal(r.verified.last.event, "push"); + assert.equal(r.polls.every_seconds, 300); + assert.match(said.join("\n"), /heard again/); +}); + +test("a forge whose events stop after one delivery is said loudly once per silent spell, and refusals are unhealthy", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const pace = new Pace(() => now, (l) => said.push(l)); + const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true, + did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } }; + pace.listening("172.17.0.1", 3047, hook.url, hook); + now += 60_000; + pace.heard("push", "d-1"); + // The hook is deleted by hand: nothing more comes, and the poll finds what no event said. + now += 11 * 60_000; + pace.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 0, "one find alone may be the poll beating an event"); + pace.miss("the merge of novox/hq#7"); + pace.miss("novox/hq#8 at bbbbbbbb"); + assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 1, "said once"); + assert.match(said.find((l) => /NOT HEARD/.test(l)) ?? "", /since the last at/); + assert.equal((pace.report() as any).healthy, false); + // A delivery ends the spell; the next spell is said again. + now += 1000; + pace.heard("push", "d-2"); + assert.equal((pace.report() as any).healthy, true); + now += 11 * 60_000; + pace.miss("novox/hq#9 at cccccccc"); + pace.miss("novox/hq#10 at dddddddd"); + assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 2, "a second silent spell is said again"); + // Every delivery refused after a verified one (a secret that no longer matches) is unhealthy at once. + now += 1000; + pace.heard("push", "d-3"); + now += 1000; + pace.refused("its signature does not verify", "172.17.0.3"); + pace.refused("its signature does not verify", "172.17.0.3"); + const r = pace.report() as any; + assert.equal(r.healthy, false); + assert.equal(r.refused.since_verified, 2); + assert.match(r.why, /2 deliveries refused since the last verified one.*does not verify/); + now += 1000; + pace.heard("push", "d-4"); + assert.equal((pace.report() as any).healthy, true, "a verified delivery clears it"); +}); + +test("without a receiver by configuration the polls alone watch, healthy; a receiver that failed is unhealthy", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const quiet = new Pace(() => now, (l) => said.push(l)); + quiet.unheard("MESH_GITEA_HOOK_URL unset", false); + now += 20 * 60_000; + quiet.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(said.length, 0, "no deliveries were expected"); + assert.equal((quiet.report() as any).healthy, true); + const failed = new Pace(() => now, () => {}); + failed.unheard("the forge's system webhook could not be made: 403"); + const r = failed.report() as any; + assert.equal(r.healthy, false); + assert.match(r.why, /403/); }); test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => { diff --git a/modules/gitea/webhook.ts b/modules/gitea/webhook.ts index 130a4e01..7cb63671 100644 --- a/modules/gitea/webhook.ts +++ b/modules/gitea/webhook.ts @@ -140,8 +140,11 @@ export interface ReceiverOptions { publish: (event: ForgeEvent) => Promise; /** Acts on it at once, beside publishing it (the announcer). */ act?: (event: ForgeEvent) => Promise; - /** Told of every verified delivery, so the polls know events are heard. */ - heard?: () => void; + /** Told of every verified delivery — its kind (X-Gitea-Event) and its delivery id — so the polls know events are + * heard, and the module can say when it last heard one. */ + heard?: (kind: string, delivery: string) => void; + /** Told of every refused delivery, with the reason and where it came from. */ + refused?: (reason: string, from: string) => void; log?: (line: string) => void; } @@ -158,6 +161,10 @@ export const REMEMBERED = 2000; */ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise } { const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`)); + const refuse = (from: string, reason: string): void => { + log(`refused a forge event from ${from}: ${reason}`); + opts.refused?.(reason, from); + }; const seen = new Set(); let queue: Promise = Promise.resolve(); const server = createServer((req, res) => { @@ -168,14 +175,14 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise MAX_BODY) { - log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`); + refuse(from, `its body is over ${MAX_BODY} bytes`); res.writeHead(413, { Connection: "close" }); res.end("too large\n"); req.destroy(); @@ -189,7 +196,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise MAX_BODY) { if (!over) { over = true; - log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`); + refuse(from, `its body is over ${MAX_BODY} bytes`); res.writeHead(413, { Connection: "close" }); res.end("too large\n"); req.destroy(); @@ -200,7 +207,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise REMEMBERED) seen.delete(seen.values().next().value as string); } - const event = eventOf(String(req.headers["x-gitea-event"] ?? "other"), payload); + const event = eventOf(kind, payload); res.statusCode = 200; res.end("queued\n"); queue = queue.then(async () => { @@ -242,27 +251,113 @@ export interface HookAdmin { call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>; } +/** The forge's one system webhook as ensured at start: read back from the forge after it was made. */ +export interface HookState { + /** Always "system": a hook the forge's listing of system webhooks names, so it fires for every repository. */ + kind: "system"; + id: number; + url: string; + events: string[]; + active: boolean; + content_type?: string; + /** Made where none was, or made in place of the ones there (system, default, or a repository's) at the receiver's url. */ + did: "created" | "replaced"; + /** The hooks at the receiver's url it took the place of, by kind: a repository's is a copy of a default webhook, which + * would have sent that repository's events a second time. */ + removed: { system: number; default: number; repository: number; repositories: string[] }; +} + +/** How long without a verified delivery — since the start, or since the last one — a forge is said silent once the poll + * finds what no event said. */ +export const SILENT_AFTER = 10 * 60_000; + /** - * **How fast the polls look** (issue 729): every five minutes only while the forge's events are heard — a verified - * delivery within the last ten minutes, and none missed since. Until the first delivery, after ten minutes without one, - * or once the poll found something no event said, every thirty seconds. + * **How fast the polls look, and whether the forge's events are heard** (issue 729): every five minutes only while the + * forge's events are heard — a verified delivery within the last ten minutes, and none missed since. Until the first + * delivery, after ten minutes without one, or once the poll found something no event said, every thirty seconds. + * + * It also keeps what `gitea_events` says: where the receiver listens, the hook as ensured, the verified and refused + * deliveries, and the polls' finds no event said. **A forge whose events stop is said** (issue 729): once the receiver + * listens and the hook is ensured, when nothing verified for ten minutes — since the start, or since the last delivery — + * and the poll has found at least two heads or merges no event said in that quiet (one could be the poll beating an + * event by a second), it is logged loudly once per silent spell, and the module's events are said unhealthy until a + * delivery verifies. Deliveries refused since the last verified one are said unhealthy too. */ export class Pace { private last = 0; private missed = 0; + private readonly started: number; private readonly now: () => number; private readonly log: (l: string) => void; + private where: { host: string; port: number; url: string } | null = null; + private hook: HookState | null = null; + private notHeard: string | null = null; + private notHeardFails = false; + private verifiedCount = 0; + private lastVerified: { at: number; event: string; delivery: string } | null = null; + private refusedCount = 0; + private refusedSpell = 0; + private lastRefused: { at: number; reason: string; from: string } | null = null; + private missCount = 0; + private lastMiss: { at: number; what: string } | null = null; + private silentSince = 0; + private quietMisses = 0; constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) { this.now = now; this.log = log; + this.started = now(); } - heard(): void { + /** The receiver listens, and the hook at it was ensured: from now on, deliveries are expected. */ + listening(host: string, port: number, url: string, hook: HookState): void { + this.where = { host, port, url }; + this.hook = hook; + this.notHeard = null; + } + /** No events are listened for, and why: by configuration (`fails` false), or because the receiver could not listen + * or the hook could not be ensured (`fails` true, said unhealthy). The polls alone watch. */ + unheard(why: string, fails = true): void { + this.notHeard = why; + this.notHeardFails = fails; + } + heard(event = "", delivery = ""): void { this.last = this.now(); + this.verifiedCount++; + this.lastVerified = { at: this.last, event, delivery }; + this.refusedSpell = 0; + this.quietMisses = 0; + if (this.silentSince) this.log(`the forge's events are heard again: ${event || "a delivery"} verified`); + this.silentSince = 0; + } + refused(reason: string, from: string): void { + this.refusedCount++; + this.refusedSpell++; + this.lastRefused = { at: this.now(), reason, from }; } /** The poll found a head or merge no forge event announced. */ miss(what: string): void { + const t = this.now(); if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`); - this.missed = this.now(); + this.missed = t; + this.missCount++; + this.lastMiss = { at: t, what }; + // Silent: nothing verified for SILENT_AFTER — never since the start, or not since the last delivery (a hook deleted by + // hand, every delivery refused) — while the poll finds what no event said. Said once per silent spell. + const quietSince = this.last || this.started; + this.quietMisses++; + if (this.expecting() && t - quietSince >= SILENT_AFTER && this.quietMisses >= 2 && !this.silentSince) { + this.silentSince = t; + const minutes = Math.round((t - quietSince) / 60_000); + const refusedSince = this.refusedSinceHeard(); + this.log(`THE FORGE'S EVENTS ARE NOT HEARD: ${this.last ? `no delivery verified since the last at ` + + `${new Date(this.last).toISOString()}` : "no delivery ever verified since the start"}, ${minutes} min, from the ` + + `system webhook at ${this.hook?.url}${refusedSince ? `; ${refusedSince} refused since, the last because ` + + `${this.lastRefused?.reason}` : ""}; the poll found ${what} — the polls alone watch the forge, every 30 s ` + + `(novox/hq issue 729)`); + } + } + /** Whether a delivery was refused after the last verified one (or with none verified). */ + private refusedSinceHeard(): number { + return this.lastRefused && this.lastRefused.at >= this.last ? this.refusedSpell : 0; } hearing(): boolean { const t = this.now(); @@ -271,36 +366,146 @@ export class Pace { every(): number { return this.hearing() ? 5 * 60_000 : 30_000; } + private expecting(): boolean { + return this.where !== null && this.hook !== null; + } + /** What `gitea_events` answers: healthy, unhealthy or waits (ADR 0283's shape), with every fact behind it. */ + report(): Record { + const t = this.now(); + const iso = (ms: number): string => new Date(ms).toISOString(); + let state: "healthy" | "unhealthy" | "waits" = "healthy"; + let why: string; + const refusedSince = this.refusedSinceHeard(); + if (this.silentSince) { + state = "unhealthy"; + why = `${this.last ? `no delivery verified since the last at ${iso(this.last)}` : `the forge has never delivered an ` + + `event since the start at ${iso(this.started)}`}, while the poll found ${this.missCount} head(s) or merge(s) no ` + + `event said, the last ${this.lastMiss?.what} at ${iso(this.lastMiss?.at ?? t)}`; + } else if (refusedSince) { + state = "unhealthy"; + why = `${refusedSince} deliveries refused ${this.last ? `since the last verified one at ${iso(this.last)}` : + "and none verified"}, the last at ${iso(this.lastRefused?.at ?? t)} because ${this.lastRefused?.reason}`; + } else if (!this.expecting()) { + why = this.notHeard ? `no events are listened for, so the polls alone watch the forge: ${this.notHeard}` : + "the receiver is not listening yet"; + if (this.notHeard && this.notHeardFails) state = "unhealthy"; + } else if (this.hearing()) { + why = `the forge's events are heard: ${this.verifiedCount} verified, the last at ${iso(this.last)}`; + } else if (this.last === 0) { + why = `no delivery heard yet; the forge is said silent if the poll finds what no event said ${SILENT_AFTER / 60_000} min after the start`; + } else { + why = `the last delivery was at ${iso(this.last)}; the polls look every 30 s until the next`; + } + return { + healthy: state !== "unhealthy", state, why, + started: iso(this.started), + receiver: this.where ? { listening: true, ...this.where } : { listening: false, why: this.notHeard ?? "not started" }, + hook: this.hook, + verified: { count: this.verifiedCount, last: this.lastVerified && { ...this.lastVerified, at: iso(this.lastVerified.at) } }, + refused: { count: this.refusedCount, since_verified: refusedSince, last: this.lastRefused && { ...this.lastRefused, at: iso(this.lastRefused.at) } }, + polls: { every_seconds: this.every() / 1000, hearing: this.hearing(), found_unsaid: this.missCount, + last_found: this.lastMiss && { ...this.lastMiss, at: iso(this.lastMiss.at) } }, + silent: this.silentSince ? { since: iso(this.silentSince) } : null, + }; + } } /** Every event family the forge (1.27) can send a webhook. */ export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request", "pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"]; -/** - * **One system webhook, for every event, at the receiver** (issue 729): created when none points there, corrected - * when one does but differs (events, active, content type), and its secret set again each start — the forge never - * hands a secret back, so a corrected secret cannot be told from a kept one. Answers what it did. - */ -export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<"created" | "corrected"> { - // POST /admin/hooks makes a system webhook, and its options take no flag for it (the forge's 1.27.3 API schema). The - // forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it. - const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*", - config: { url, content_type: "json", secret } }; - const listed = await admin.call("GET", "/admin/hooks?type=system&limit=50"); - if (listed.status !== 200) throw new Error(`the forge's system webhooks could not be listed: ${listed.status}`); - const hooks: any[] = Array.isArray(listed.body) ? listed.body : []; - const ours = hooks.filter((h) => h?.config?.url === url); - if (ours.length === 0) { - const made = await admin.call("POST", "/admin/hooks", want); - if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`); - return "created"; +async function hooksAt(admin: HookAdmin, type: "system" | "default", url: string): Promise { + const listed = await admin.call("GET", `/admin/hooks?type=${type}&limit=50`); + if (listed.status !== 200) throw new Error(`the forge's ${type} webhooks could not be listed: ${listed.status}`); + return (Array.isArray(listed.body) ? listed.body : []).filter((h) => h?.config?.url === url); +} + +/** Every page of a listing that answers an array (or `{ data: [] }`, the repository search), PAGE at a time. */ +const PAGE = 50; +async function everyPage(admin: HookAdmin, path: string, what: string): Promise { + const all: any[] = []; + for (let page = 1; ; page++) { + const r = await admin.call("GET", `${path}${path.includes("?") ? "&" : "?"}page=${page}&limit=${PAGE}`); + if (r.status !== 200) throw new Error(`${what} could not be listed: ${r.status}`); + const items: any[] = Array.isArray(r.body) ? r.body : Array.isArray(r.body?.data) ? r.body.data : []; + all.push(...items); + if (items.length < PAGE) return all; } - for (const extra of ours.slice(1)) await admin.call("DELETE", `/admin/hooks/${extra.id}`); - const fixed = await admin.call("PATCH", `/admin/hooks/${ours[0].id}`, { active: true, events: HOOK_EVENTS, branch_filter: "*", - config: { url, content_type: "json", secret } }); - if (fixed.status !== 200) throw new Error(`the forge's system webhook could not be corrected: ${fixed.status}`); - return "corrected"; +} + +/** Every repository's own webhooks at the receiver's url: copies of a default webhook made at it. */ +async function repoHooksAt(admin: HookAdmin, url: string): Promise<{ repo: string; id: number }[]> { + const found: { repo: string; id: number }[] = []; + for (const repo of await everyPage(admin, "/repos/search?sort=id", "the forge's repositories")) { + const full = String(repo?.full_name ?? ""); + if (!full.includes("/")) continue; + const hooks = await everyPage(admin, `/repos/${full}/hooks`, `${full}'s webhooks`); + for (const h of hooks) if (h?.config?.url === url) found.push({ repo: full, id: Number(h.id) }); + } + return found; +} + +/** + * **One system webhook, for every event, at the receiver** (issue 729), made each start and read back. + * + * The forge (1.27.3, routers/api/v1/utils/hook.go addHook) makes a *default* webhook from POST /admin/hooks unless + * `config.is_system_webhook` is "true": a default webhook fires for no repository and is only copied into repositories + * made after it, and the listing of system webhooks does not name it — so without the flag every start made another + * one and no event came. Its edit (editHook) never changes a webhook's secret nor its kind, so a hook already there is + * not corrected but replaced: the new system webhook is made first, with the secret kept now, so no event falls in a + * gap; then every other hook at the receiver's url is deleted — system, default, and each repository's copy of a + * default one, which would send that repository's events twice. The forge's listing of system webhooks must then name + * exactly one at the url, or this fails, saying so. + */ +export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise { + // The forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it. + const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*", + config: { url, content_type: "json", secret, is_system_webhook: "true" } }; + const before = new Set((await hooksAt(admin, "system", url)).map((h) => Number(h.id))); + const made = await admin.call("POST", "/admin/hooks", want); + if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`); + const fresh = (await hooksAt(admin, "system", url)).filter((h) => !before.has(Number(h.id))); + if (fresh.length !== 1) { + throw new Error(`the forge names ${fresh.length} new system webhooks at ${url} after one was made, not one: its events ` + + `are not known to be sent`); + } + const ours = fresh[0]; + const defaults = await hooksAt(admin, "default", url); + const gone = async (path: string): Promise => { + const r = await admin.call("DELETE", path); + if (r.status !== 204 && r.status !== 200 && r.status !== 404) throw new Error(`the forge's webhook ${path} could not be deleted: ${r.status}`); + }; + for (const id of before) await gone(`/admin/hooks/${id}`); + for (const h of defaults) await gone(`/admin/hooks/${h.id}`); + const copies = await repoHooksAt(admin, url); + for (const c of copies) await gone(`/repos/${c.repo}/hooks/${c.id}`); + const kept = await hooksAt(admin, "system", url); + if (kept.length !== 1 || Number(kept[0].id) !== Number(ours.id)) { + throw new Error(`the forge names ${kept.length} system webhooks at ${url} once the others were deleted, not the one ` + + `made (${ours.id})`); + } + const removed = { system: before.size, default: defaults.length, repository: copies.length, + repositories: [...new Set(copies.map((c) => c.repo))] }; + return { kind: "system", id: Number(ours.id), url, events: Array.isArray(ours.events) ? ours.events : [], + active: !!ours.active, content_type: ours.config?.content_type, + did: removed.system + removed.default + removed.repository > 0 ? "replaced" : "created", removed }; +} + +/** The `gitea_events` tool: what the module knows of the forge's events, from its Pace. */ +export function eventsTool(pace: Pace): { name: string; description: string; input: Record; + run: (args: Readonly>) => Promise } { + return { + name: "gitea_events", + description: "Whether the forge's events reach the mesh (novox/hq issue 729): where the receiver listens, the " + + "system webhook as ensured at the start (its id, url, events, kind, and what it replaced), how many deliveries " + + "verified and the last one's time and event, how many were refused and the last reason, how often the polls look " + + "and whether events are heard, and how many heads or merges the poll found that no event said. Unhealthy when the " + + "forge has delivered nothing that verified for ten minutes (since the start or since the last delivery) while the " + + "poll found what no event said, when deliveries were refused since the last verified one, or when the receiver " + + "could not listen or the webhook could not be made.", + input: {}, + run: async () => pace.report(), + }; } /** From 3a84044eceba6f28fb42743df89ad0e1cae239b9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 23:31:13 +0200 Subject: [PATCH 07/11] Add the kernel module: transparent huge pages set per machine (hq issue 746) With THP 'always' journald's mmapped journal files sit in 2 MiB folios, so each small journal write dirties and writes back 2 MiB; on the control node that saturated the root mirror holding the bus store and the store, and the controller lost its lease. The module sets the mode from a per-machine setting (default always, the kernel's own, so assigning it changes nothing until a machine sets madvise), verifies it took, rotates the journal once so new files are mapped under the new mode, gives back what it found when stopped, and serves kernel_huge_pages to measure file_thp per unit. --- modules/kernel/README.md | 50 ++++ modules/kernel/cmd/kernel-tools/hugepages.go | 241 ++++++++++++++++++ .../kernel/cmd/kernel-tools/hugepages_test.go | 113 ++++++++ modules/kernel/cmd/kernel-tools/machine.go | 65 +++++ modules/kernel/cmd/kernel-tools/main.go | 50 ++++ .../kernel/cmd/kernel-tools/manifest_test.go | 181 +++++++++++++ modules/kernel/cmd/kernel-tools/shape_test.go | 79 ++++++ modules/kernel/go.mod | 5 + modules/kernel/go.sum | 2 + modules/kernel/module.json | 69 +++++ 10 files changed, 855 insertions(+) create mode 100644 modules/kernel/README.md create mode 100644 modules/kernel/cmd/kernel-tools/hugepages.go create mode 100644 modules/kernel/cmd/kernel-tools/hugepages_test.go create mode 100644 modules/kernel/cmd/kernel-tools/machine.go create mode 100644 modules/kernel/cmd/kernel-tools/main.go create mode 100644 modules/kernel/cmd/kernel-tools/manifest_test.go create mode 100644 modules/kernel/cmd/kernel-tools/shape_test.go create mode 100644 modules/kernel/go.mod create mode 100644 modules/kernel/go.sum create mode 100644 modules/kernel/module.json diff --git a/modules/kernel/README.md b/modules/kernel/README.md new file mode 100644 index 00000000..faf4cc94 --- /dev/null +++ b/modules/kernel/README.md @@ -0,0 +1,50 @@ +# kernel + +The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one: +**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot +entries) is not built yet. + +## Why (novox/hq issue 746) + +With transparent huge pages `always`, the page cache of a file mapped by a program is collapsed into +2 MiB folios (khugepaged), and a small write into such a folio dirties all of it: writeback then writes +2 MiB. journald writes its journal files through mmap, and every journal reader (fail2ban's journal +jails among them) maps them too. Measured on the control node on 2026-10-14: journald's cgroup had +written 2.5 TB in 12 days while the journal grew by about 25-30 KB/s, fail2ban's cgroup 63 GB in 4 h +while its own files did not grow, both holding hundreds of MiB of file pages in huge folios; the bursts +saturated the mirrored spinning root disk that holds the bus store and the store, and the controller +lost its lease four times in 30 minutes. With `madvise`, only memory a program asks to have in huge pages +(`madvise(MADV_HUGEPAGE)`) gets them; a database or a JVM that wants them still asks. + +## What it owns + +| | what | notes | +|---|---|---| +| setting | `transparent-huge-pages` | `always`, `madvise` or `never`. Default `always`, what the distribution's kernel boots with, so assigning the module changes nothing until a machine sets another value | +| file | `${dir:state}/huge-pages` | the script, written from the setting: `apply` writes the mode to `/sys/kernel/mm/transparent_hugepage/enabled` and reads it back (a value that does not take fails the unit); `restore` gives back the mode found before the module first set it, kept in `${dir:state}/found` | +| file | `/etc/systemd/system/mesh-kernel-huge-pages.service` | oneshot, early in boot (`sysinit.target`, before the services that fill the page cache); stopping it runs `restore` | +| service | `mesh-kernel-huge-pages.service` | running and enabled, restarted when the script (so the setting) or the unit changes; its health is the unit's | + +The per-size controls (`hugepages-kB/enabled`) are left alone: on the machines read, 2048 kB says +`inherit` (it follows the mode) and every other size `never`. `kernel_huge_pages` shows each one. + +**A folio already in the page cache keeps its size.** A new mode applies to pages faulted or collapsed +from then on. The active journal files would keep their 2 MiB folios until archived, so `apply` rotates +the journal (`journalctl --rotate`) whenever it moves the mode away from `always`: new files are mapped +under the new mode. A failed rotation is said in the unit's journal and does not fail the setting. At +boot the kernel starts at `always`, so a machine set to `madvise` rotates its journal once per boot. + +## Tools + +| tool | | answers | +|---|---|---| +| `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) | + +## Tests + +`go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the +manifest's shape (the default, the order, the unit, restart and health); and the script as the mesh +writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once, +rotates the journal once and not for `always`, gives back what it found, refuses a value the kernel +does not take before writing anything, fails when the write does not take, and says when the kernel has +no transparent huge pages. diff --git a/modules/kernel/cmd/kernel-tools/hugepages.go b/modules/kernel/cmd/kernel-tools/hugepages.go new file mode 100644 index 00000000..697f4fd8 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/hugepages.go @@ -0,0 +1,241 @@ +package main + +import ( + "fmt" + "os" + "path" + "sort" + "strconv" + "strings" +) + +// THPDir is where the kernel says and takes its transparent huge pages. +const THPDir = "sys/kernel/mm/transparent_hugepage" + +// CgroupRoot is the unified cgroup hierarchy. +const CgroupRoot = "sys/fs/cgroup" + +// HugePagesAnswer is what kernel_huge_pages answers. +type HugePagesAnswer struct { + // Mode is the word the kernel brackets in enabled: always, madvise or never. + Mode string `json:"mode"` + Defrag string `json:"defrag,omitempty"` + Shmem string `json:"shmem_enabled,omitempty"` + PerSize []Size `json:"per_size,omitempty"` + // Meminfo is the machine's huge pages by kind, in MiB. + Meminfo map[string]float64 `json:"meminfo_mib,omitempty"` + Units []Unit `json:"units"` + Said []string `json:"said"` +} + +// Size is one per-size control (hugepages-kB/enabled). +type Size struct { + Size string `json:"size"` + Enabled string `json:"enabled"` + // Means is what inherit resolves to: the top-level mode. + Means string `json:"means"` +} + +// Unit is one cgroup holding file pages in huge folios. +type Unit struct { + Cgroup string `json:"cgroup"` + FileTHP int64 `json:"file_thp_bytes"` + FileDirty int64 `json:"file_dirty_bytes"` + FileWriteback int64 `json:"file_writeback_bytes"` + FileMapped int64 `json:"file_mapped_bytes"` + CollapsedByKhug int64 `json:"thp_collapse_alloc"` + Written int64 `json:"written_bytes"` +} + +// bracketed is the word a sysfs choice file brackets: "always [madvise] never" is madvise. +func bracketed(text string) string { + open := strings.IndexByte(text, '[') + end := strings.IndexByte(text, ']') + if open < 0 || end < open { + return "" + } + return text[open+1 : end] +} + +// HugePages reads the mode, the per-size controls, meminfo and the top units by file_thp. +func (m *Machine) HugePages(units int) (*HugePagesAnswer, error) { + enabled, err := m.read(THPDir + "/enabled") + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("this kernel has no /%s/enabled: it was built without transparent huge pages", THPDir) + } + return nil, fmt.Errorf("reading /%s/enabled: %w", THPDir, err) + } + a := &HugePagesAnswer{Mode: bracketed(enabled), Units: []Unit{}} + if s, err := m.read(THPDir + "/defrag"); err == nil { + a.Defrag = bracketed(s) + } + if s, err := m.read(THPDir + "/shmem_enabled"); err == nil { + a.Shmem = bracketed(s) + } + a.Said = append(a.Said, fmt.Sprintf("transparent huge pages: %s, defrag %s", a.Mode, orUnknown(a.Defrag))) + + sizes, _ := m.dirs(THPDir) + for _, d := range sizes { + if !strings.HasPrefix(d, "hugepages-") { + continue + } + s, err := m.read(THPDir + "/" + d + "/enabled") + if err != nil { + continue + } + sz := Size{Size: strings.TrimPrefix(d, "hugepages-"), Enabled: bracketed(s)} + sz.Means = sz.Enabled + if sz.Enabled == "inherit" { + sz.Means = a.Mode + } + a.PerSize = append(a.PerSize, sz) + } + sort.Slice(a.PerSize, func(i, j int) bool { return kib(a.PerSize[i].Size) < kib(a.PerSize[j].Size) }) + var on []string + for _, s := range a.PerSize { + if s.Means != "never" { + on = append(on, s.Size+" "+s.Means) + } + } + if len(a.PerSize) > 0 { + a.Said = append(a.Said, "sizes not never: "+orNone(on)) + } + + if mi, err := m.read("proc/meminfo"); err == nil { + a.Meminfo = meminfo(mi) + a.Said = append(a.Said, fmt.Sprintf("huge pages held: anonymous %.0f MiB, shared memory %.0f MiB, file %.0f MiB (mapped at 2 MiB %.0f MiB)", + a.Meminfo["AnonHugePages"], a.Meminfo["ShmemHugePages"], a.Meminfo["FileHugePages"], a.Meminfo["FilePmdMapped"])) + } + + if units > 0 { + all := m.cgroups() + sort.Slice(all, func(i, j int) bool { + if all[i].FileTHP != all[j].FileTHP { + return all[i].FileTHP > all[j].FileTHP + } + return all[i].Cgroup < all[j].Cgroup + }) + if len(all) > units { + all = all[:units] + } + a.Units = all + for _, u := range all { + a.Said = append(a.Said, fmt.Sprintf("%s: file in huge folios %s, dirty %s, writeback %s, written %s since it started", + u.Cgroup, mib(u.FileTHP), mib(u.FileDirty), mib(u.FileWriteback), mib(u.Written))) + } + if len(all) == 0 { + a.Said = append(a.Said, "no unit holds file pages in huge folios") + } + } + return a, nil +} + +// cgroups walks the leaf services and scopes (to three levels below the root) that hold file_thp. +func (m *Machine) cgroups() []Unit { + var out []Unit + var walk func(rel string, depth int) + walk = func(rel string, depth int) { + children, _ := m.dirs(path.Join(CgroupRoot, rel)) + for _, c := range children { + p := path.Join(rel, c) + if strings.HasSuffix(c, ".slice") { + if depth < 3 { + walk(p, depth+1) + } + continue + } + if !strings.HasSuffix(c, ".service") && !strings.HasSuffix(c, ".scope") { + continue + } + stat, err := m.read(path.Join(CgroupRoot, p, "memory.stat")) + if err != nil { + continue + } + kv := memoryStat(stat) + if kv["file_thp"] == 0 { + continue + } + u := Unit{Cgroup: p, FileTHP: kv["file_thp"], FileDirty: kv["file_dirty"], FileWriteback: kv["file_writeback"], + FileMapped: kv["file_mapped"], CollapsedByKhug: kv["thp_collapse_alloc"]} + if io, err := m.read(path.Join(CgroupRoot, p, "io.stat")); err == nil { + u.Written = mostWritten(io) + } + out = append(out, u) + } + } + walk("", 0) + return out +} + +// memoryStat reads memory.stat's "key value" lines. +func memoryStat(text string) map[string]int64 { + kv := map[string]int64{} + for _, l := range strings.Split(text, "\n") { + f := strings.Fields(l) + if len(f) != 2 { + continue + } + if n, err := strconv.ParseInt(f[1], 10, 64); err == nil { + kv[f[0]] = n + } + } + return kv +} + +// mostWritten is the most bytes any one device of io.stat saw written: a mirror's member disks and the +// mirror itself each count the same write, so the lines are never summed. +func mostWritten(text string) int64 { + var most int64 + for _, l := range strings.Split(text, "\n") { + for _, f := range strings.Fields(l) { + if v, ok := strings.CutPrefix(f, "wbytes="); ok { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > most { + most = n + } + } + } + } + return most +} + +// meminfo reads the huge page lines of /proc/meminfo, in MiB. +func meminfo(text string) map[string]float64 { + want := map[string]bool{"AnonHugePages": true, "ShmemHugePages": true, "ShmemPmdMapped": true, "FileHugePages": true, "FilePmdMapped": true} + out := map[string]float64{} + for _, l := range strings.Split(text, "\n") { + k, v, ok := strings.Cut(l, ":") + if !ok || !want[k] { + continue + } + f := strings.Fields(v) + if len(f) == 0 { + continue + } + if n, err := strconv.ParseFloat(f[0], 64); err == nil { + out[k] = n / 1024 + } + } + return out +} + +func kib(size string) int { + n, _ := strconv.Atoi(strings.TrimSuffix(size, "kB")) + return n +} + +func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) } + +func orUnknown(s string) string { + if s == "" { + return "unknown" + } + return s +} + +func orNone(list []string) string { + if len(list) == 0 { + return "none" + } + return strings.Join(list, ", ") +} diff --git a/modules/kernel/cmd/kernel-tools/hugepages_test.go b/modules/kernel/cmd/kernel-tools/hugepages_test.go new file mode 100644 index 00000000..2d6359c7 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/hugepages_test.go @@ -0,0 +1,113 @@ +package main + +// Over a stand-in tree holding novox's own files of 2026-10-14 23:25 (read through node-reads for +// novox/hq issue 746): the mode, the per-size controls, meminfo's huge page lines, and fail2ban's and +// journald's memory.stat and io.stat, cut to the lines the tool reads. + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func tree(t *testing.T, files map[string]string) *Machine { + t.Helper() + root := t.TempDir() + for p, content := range files { + full := filepath.Join(root, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return &Machine{Root: root} +} + +func novox(t *testing.T) *Machine { + thp := "sys/kernel/mm/transparent_hugepage/" + cg := "sys/fs/cgroup/system.slice/" + files := map[string]string{ + thp + "enabled": "[always] madvise never\n", + thp + "defrag": "always defer defer+madvise [madvise] never\n", + thp + "shmem_enabled": "always within_size [advise] never deny force\n", + thp + "hugepages-2048kB/enabled": "always [inherit] madvise never\n", + thp + "hugepages-64kB/enabled": "always inherit madvise [never]\n", + thp + "hugepages-1024kB/enabled": "always inherit madvise [never]\n", + "proc/meminfo": "MemTotal: 65748808 kB\nAnonHugePages: 3579904 kB\nShmemHugePages: 4096 kB\n" + + "ShmemPmdMapped: 4096 kB\nFileHugePages: 1693696 kB\nFilePmdMapped: 882688 kB\nHugepagesize: 2048 kB\n", + cg + "fail2ban.service/memory.stat": "anon 21815296\nfile 204623872\nfile_mapped 130736128\nfile_dirty 27262976\n" + + "file_writeback 4096\nanon_thp 0\nfile_thp 197132288\nthp_fault_alloc 0\nthp_collapse_alloc 550\n", + cg + "fail2ban.service/io.stat": "9:0 \n8:16 rbytes=3223552 wbytes=67723415552 rios=240 wios=281159 dbytes=0 dios=0\n" + + "8:0 rbytes=6152192 wbytes=67723415552 rios=832 wios=281159 dbytes=0 dios=0\n" + + "9:2 rbytes=9375744 wbytes=67723415552 rios=1072 wios=281159 dbytes=0 dios=0\n", + cg + "systemd-journald.service/memory.stat": "anon 3227648\nfile 1032015872\nfile_mapped 131928064\nfile_dirty 622592\n" + + "file_writeback 159744\nanon_thp 2097152\nfile_thp 379584512\nthp_fault_alloc 0\nthp_collapse_alloc 11741\n", + cg + "systemd-journald.service/io.stat": "9:0 \n8:16 rbytes=55209984 wbytes=2498768035840 rios=1317 wios=24719004 dbytes=0 dios=0\n" + + "8:0 rbytes=72953856 wbytes=2498768039936 rios=2194 wios=24719005 dbytes=0 dios=0\n" + + "9:2 rbytes=128163840 wbytes=2498724843520 rios=3511 wios=24707873 dbytes=0 dios=0\n", + cg + "sshd.service/memory.stat": "anon 1000\nfile_thp 0\n", + } + return tree(t, files) +} + +func TestNovoxBeforeTheChange(t *testing.T) { + a, err := novox(t).HugePages(10) + if err != nil { + t.Fatal(err) + } + if a.Mode != "always" || a.Defrag != "madvise" || a.Shmem != "advise" { + t.Fatalf("modes: %+v", a) + } + if len(a.PerSize) != 3 || a.PerSize[0].Size != "64kB" || a.PerSize[2].Size != "2048kB" || + a.PerSize[2].Enabled != "inherit" || a.PerSize[2].Means != "always" || a.PerSize[0].Means != "never" { + t.Fatalf("per size: %+v", a.PerSize) + } + if a.Meminfo["FileHugePages"] != 1654 || a.Meminfo["FilePmdMapped"] != 862 { + t.Fatalf("meminfo: %v", a.Meminfo) + } + if len(a.Units) != 2 || a.Units[0].Cgroup != "system.slice/systemd-journald.service" || + a.Units[1].Cgroup != "system.slice/fail2ban.service" { + t.Fatalf("units: %+v", a.Units) + } + // The mirror and its two members each count the write: the most any one saw, never their sum. + if a.Units[1].Written != 67723415552 || a.Units[1].FileTHP != 197132288 || a.Units[1].CollapsedByKhug != 550 { + t.Fatalf("fail2ban: %+v", a.Units[1]) + } + said := strings.Join(a.Said, "\n") + for _, want := range []string{"transparent huge pages: always, defrag madvise", "sizes not never: 2048kB always", + "file 1654 MiB (mapped at 2 MiB 862 MiB)", "system.slice/fail2ban.service: file in huge folios 188.0 MiB"} { + if !strings.Contains(said, want) { + t.Errorf("said lacks %q:\n%s", want, said) + } + } +} + +func TestUnitsAreBoundedAndNoneIsSaid(t *testing.T) { + a, err := novox(t).HugePages(1) + if err != nil || len(a.Units) != 1 { + t.Fatalf("%v %+v", err, a) + } + b, err := tree(t, map[string]string{"sys/kernel/mm/transparent_hugepage/enabled": "always [madvise] never\n"}).HugePages(10) + if err != nil || b.Mode != "madvise" || len(b.Units) != 0 || !strings.Contains(strings.Join(b.Said, "\n"), "no unit holds") { + t.Fatalf("%v %+v", err, b) + } +} + +func TestAKernelWithoutHugePagesIsAnError(t *testing.T) { + if _, err := tree(t, map[string]string{"proc/meminfo": ""}).HugePages(10); err == nil || + !strings.Contains(err.Error(), "built without transparent huge pages") { + t.Fatalf("%v", err) + } +} + +func TestTheArgumentIsBounded(t *testing.T) { + if _, err := whole(map[string]any{"units": -1.0}, "units", 10, 0, 100); err == nil { + t.Error("a negative count was accepted") + } + if n, _ := whole(map[string]any{"units": 1000.0}, "units", 10, 0, 100); n != 100 { + t.Errorf("not bounded: %d", n) + } +} diff --git a/modules/kernel/cmd/kernel-tools/machine.go b/modules/kernel/cmd/kernel-tools/machine.go new file mode 100644 index 00000000..b5f6c730 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/machine.go @@ -0,0 +1,65 @@ +package main + +// The machine as the tools see it: the files they read, under a root that tests replace with a stand-in +// tree. Everything read here is readable by the runtime's account; nothing is escalated. + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" +) + +// Machine reads this machine's kernel files under Root ("/" on the machine). +type Machine struct { + Root string +} + +// ThisMachine is the machine the runtime launched this bundle on. +func ThisMachine() *Machine { return &Machine{Root: "/"} } + +func (m *Machine) path(p string) string { return filepath.Join(m.Root, p) } + +func (m *Machine) read(p string) (string, error) { + b, err := os.ReadFile(m.path(p)) + return string(b), err +} + +func (m *Machine) dirs(p string) ([]string, error) { + entries, err := os.ReadDir(m.path(p)) + if err != nil { + return nil, err + } + var out []string + for _, e := range entries { + if e.IsDir() || e.Type()&fs.ModeSymlink != 0 { + out = append(out, e.Name()) + } + } + return out, nil +} + +// whole is a whole-number argument with a default, kept within bounds. +func whole(args map[string]any, key string, def, least, most int) (int, error) { + raw, present := args[key] + if !present || raw == nil { + return def, nil + } + f, ok := raw.(float64) + if !ok || f != float64(int(f)) { + return 0, fmt.Errorf("%s must be a whole number", key) + } + n := int(f) + if n < least { + return 0, fmt.Errorf("%s must be at least %d", key, least) + } + if n > most { + n = most + } + return n, nil +} + +// schema is a tool's input: its properties. +func schema(properties map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": properties} +} diff --git a/modules/kernel/cmd/kernel-tools/main.go b/modules/kernel/cmd/kernel-tools/main.go new file mode 100644 index 00000000..ec15c916 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/main.go @@ -0,0 +1,50 @@ +// The kernel module's tools bundle (novox/hq to-be 42 Phase 1 row 11, issue 746). +// +// Served by the node's runtime over MCP on stdio through the Go SDK (ADR 0188, ADR 0193). It only reads: +// the transparent huge pages the kernel runs with, and which units hold file pages in huge folios, which +// is what a small write to a mapped file costs a whole huge page of writeback for (issue 746). +package main + +import ( + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// binaryName is what the build names this bundle's executable: the manifest's `binary`. +const binaryName = "kernel-tools" + +func main() { + // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): kernel. + if err := stdio.Serve("", tools(ThisMachine())); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func tools(m *Machine) []stdio.Tool { + return []stdio.Tool{ + { + Name: "kernel_huge_pages", + Description: "Transparent huge pages on this machine: the mode in force (always, madvise or never) and defrag, " + + "each per-size control with what inherit resolves to, the machine's huge pages by kind (anonymous, " + + "shared memory, file, file mapped at 2 MiB) from /proc/meminfo, and the units holding the most file " + + "pages in huge folios (memory.stat file_thp) with their dirty and writeback bytes and the bytes their " + + "cgroup has written (io.stat, the most any one device saw). A unit with file_thp that writes far more " + + "than its files grow is paying a whole huge page of writeback per small write (novox/hq issue 746). " + + "Replaces cat /sys/kernel/mm/transparent_hugepage/enabled, grep Huge /proc/meminfo and reading " + + "memory.stat by hand. (r)", + Input: schema(map[string]any{ + "units": map[string]any{"type": "integer", "description": "how many units to name, the most file_thp first (default 10, 0 for none, at most 100)"}, + }), + Run: func(args map[string]any) (any, error) { + n, err := whole(args, "units", 10, 0, 100) + if err != nil { + return nil, err + } + return m.HugePages(n) + }, + }, + } +} diff --git a/modules/kernel/cmd/kernel-tools/manifest_test.go b/modules/kernel/cmd/kernel-tools/manifest_test.go new file mode 100644 index 00000000..fd1a0327 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/manifest_test.go @@ -0,0 +1,181 @@ +package main + +// The module's shape (novox/hq issue 746): the setting's default is what the kernel boots with, so +// assigning the module changes nothing until a machine sets another value; the script is written into the +// module's own state and run by a oneshot unit that is restarted whenever the script (and so the setting) +// or the unit changes; and the script itself sets, verifies, gives back and refuses as it says. + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestTheDefaultChangesNothingAndTheUnitFollowsTheSetting(t *testing.T) { + m := manifest(t) + s := m.Settings["transparent-huge-pages"] + if s["kind"] != "preference" || s["default"] != "always" || s["why"] == "" { + t.Fatalf("the setting: %v", s) + } + var ids []string + for _, r := range m.Resources { + ids = append(ids, r["id"].(string)) + } + if strings.Join(ids, " ") != "state huge-pages unit apply" { + t.Fatalf("order: %v", ids) + } + script := m.resource(t, "huge-pages") + // The script runs as root, so the setting it is written from is set at the terminal alone (issue 339). + if script["path"] != "${dir:state}/huge-pages" || script["mode"] != "0755" || script["trusted"] != true || + !strings.Contains(script["content"].(string), "want='${setting:transparent-huge-pages}'") { + t.Fatalf("the script: %v", script["path"]) + } + unit := m.resource(t, "unit")["content"].(string) + for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/huge-pages apply", + "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target", "DefaultDependencies=no"} { + if !strings.Contains(unit, line+"\n") { + t.Errorf("the unit lacks %q", line) + } + } + apply := m.resource(t, "apply") + if apply["unit"] != "mesh-kernel-huge-pages.service" || apply["state"] != "running" || apply["boot"] != "enabled" { + t.Fatalf("apply: %v", apply) + } + if on := apply["restart-on"].([]any); len(on) != 2 || on[0] != "huge-pages" || on[1] != "unit" { + t.Fatalf("restart-on: %v", on) + } + if h := apply["health"].(map[string]any); h["kind"] != "unit" { + t.Fatalf("health: %v", h) + } +} + +// bed is the script as the mesh writes it for one value of the setting, a stand-in transparent_hugepage +// directory, the module's state, and a PATH whose journalctl records that it was asked to rotate. +type bed struct { + script, sys, state, rotated, path string +} + +func newBed(t *testing.T, want, found string) bed { + t.Helper() + dir := t.TempDir() + b := bed{script: filepath.Join(dir, "huge-pages"), sys: filepath.Join(dir, "thp"), state: filepath.Join(dir, "state"), + rotated: filepath.Join(dir, "rotated")} + content := manifest(t).resource(t, "huge-pages")["content"].(string) + content = strings.ReplaceAll(content, "${setting:transparent-huge-pages}", want) + content = strings.ReplaceAll(content, "${dir:state}", b.state) + if strings.Contains(content, "${") { + t.Fatalf("a substitution the test does not make is left in the script") + } + bin := filepath.Join(dir, "bin") + for _, d := range []string{b.sys, b.state, bin} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + write(t, b.script, content, 0o755) + write(t, filepath.Join(b.sys, "enabled"), found, 0o644) + write(t, filepath.Join(bin, "journalctl"), "#!/bin/sh\necho \"$@\" >>"+b.rotated+"\n", 0o755) + b.path = bin + string(os.PathListSeparator) + os.Getenv("PATH") + return b +} + +func write(t *testing.T, p, content string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(p, []byte(content), mode); err != nil { + t.Fatal(err) + } +} + +func (b bed) run(t *testing.T, verb string) (string, error) { + t.Helper() + cmd := exec.Command("sh", b.script, verb, b.sys, b.state) + cmd.Env = append(os.Environ(), "PATH="+b.path) + out, err := cmd.CombinedOutput() + return string(out), err +} + +func (b bed) file(t *testing.T, p string) string { + t.Helper() + raw, err := os.ReadFile(p) + if err != nil { + return "" + } + return string(raw) +} + +func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t *testing.T) { + // novox's file on 2026-10-14, read through node-reads. + b := newBed(t, "madvise", "[always] madvise never\n") + out, err := b.run(t, "apply") + if err != nil || !strings.Contains(out, "always -> madvise") { + t.Fatalf("apply: %v %s", err, out) + } + if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" { + t.Fatalf("written: %q", got) + } + if got := b.file(t, filepath.Join(b.state, "found")); got != "always\n" { + t.Fatalf("found: %q", got) + } + if got := b.file(t, b.rotated); got != "--rotate\n" { + t.Fatalf("rotation: %q", got) + } + // Applied again (a restart with the same value): nothing written, nothing rotated, found kept. + if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise, as set") { + t.Fatalf("again: %v %s", err, out) + } + if got := b.file(t, b.rotated); got != "--rotate\n" { + t.Fatalf("rotated again: %q", got) + } + if out, err := b.run(t, "restore"); err != nil || !strings.Contains(out, "back to always") { + t.Fatalf("restore: %v %s", err, out) + } + if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "always\n" { + t.Fatalf("restored: %q", got) + } +} + +func TestAlwaysIsSetWithoutRotating(t *testing.T) { + b := newBed(t, "always", "always [madvise] never\n") + if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise -> always") { + t.Fatalf("apply: %v %s", err, out) + } + if got := b.file(t, b.rotated); got != "" { + t.Fatalf("rotated for always: %q", got) + } +} + +func TestAValueTheKernelDoesNotTakeIsRefusedBeforeAnythingIsWritten(t *testing.T) { + b := newBed(t, "sometimes", "[always] madvise never\n") + out, err := b.run(t, "apply") + if err == nil || !strings.Contains(out, "takes always, madvise or never") { + t.Fatalf("accepted: %v %s", err, out) + } + if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "[always] madvise never\n" { + t.Fatalf("written anyway: %q", got) + } +} + +func TestAWriteThatDoesNotTakeFailsTheUnit(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root writes a read-only file") + } + b := newBed(t, "madvise", "[always] madvise never\n") + if err := os.Chmod(filepath.Join(b.sys, "enabled"), 0o444); err != nil { + t.Fatal(err) + } + if out, err := b.run(t, "apply"); err == nil { + t.Fatalf("a refused write passed: %s", out) + } +} + +func TestAKernelWithoutHugePagesIsSaid(t *testing.T) { + b := newBed(t, "madvise", "") + if err := os.Remove(filepath.Join(b.sys, "enabled")); err != nil { + t.Fatal(err) + } + if out, err := b.run(t, "apply"); err == nil || !strings.Contains(out, "built without transparent huge pages") { + t.Fatalf("%v %s", err, out) + } +} diff --git a/modules/kernel/cmd/kernel-tools/shape_test.go b/modules/kernel/cmd/kernel-tools/shape_test.go new file mode 100644 index 00000000..c30224f3 --- /dev/null +++ b/modules/kernel/cmd/kernel-tools/shape_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "encoding/json" + "os" + "testing" +) + +type resource map[string]any + +type manifestShape struct { + Module string `json:"module"` + Capabilities []string `json:"capabilities"` + Settings map[string]map[string]any `json:"settings"` + Tools []string `json:"tools"` + Resources []resource `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` +} + +func manifest(t *testing.T) manifestShape { + t.Helper() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m manifestShape + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +func (m manifestShape) resource(t *testing.T, id string) resource { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + return r + } + } + t.Fatalf("no resource %s", id) + return nil +} + +// TestToolsAreTheManifests holds the served tools and the manifest's list to one another, and the +// bundle to the shape the builder compiles and the runtime loads. +func TestToolsAreTheManifests(t *testing.T) { + m := manifest(t) + names := map[string]bool{} + for _, tool := range tools(&Machine{Root: t.TempDir()}) { + if names[tool.Name] { + t.Errorf("%s is served twice", tool.Name) + } + names[tool.Name] = true + } + for _, want := range m.Tools { + if !names[want] { + t.Errorf("the manifest lists %s and the bundle does not serve it", want) + } + delete(names, want) + } + if len(names) != 0 { + t.Errorf("served and not listed: %v", names) + } + var tools map[string]any + for _, a := range m.Build.Artifacts { + if a["name"] == "tools" { + tools = a + } + } + if tools == nil || tools["kind"] != "bundle" || tools["language"] != "go" || tools["system"] != "arch" || + tools["from"] != "cmd/"+binaryName || tools["binary"] != binaryName { + t.Fatalf("the tools artifact: %v", tools) + } + if loads, _ := tools["loads"].([]any); len(loads) != 1 || loads[0] != binaryName { + t.Fatalf("loads: %v", tools["loads"]) + } +} diff --git a/modules/kernel/go.mod b/modules/kernel/go.mod new file mode 100644 index 00000000..748c098c --- /dev/null +++ b/modules/kernel/go.mod @@ -0,0 +1,5 @@ +module kernel + +go 1.22 + +require git.novox.be/novox/mesh-sdk/go v0.1.7 diff --git a/modules/kernel/go.sum b/modules/kernel/go.sum new file mode 100644 index 00000000..b474419e --- /dev/null +++ b/modules/kernel/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= +git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/kernel/module.json b/modules/kernel/module.json new file mode 100644 index 00000000..68645769 --- /dev/null +++ b/modules/kernel/module.json @@ -0,0 +1,69 @@ +{ + "module": "kernel", + "version": "1", + "capabilities": [ + "service-manager" + ], + "settings": { + "transparent-huge-pages": { + "kind": "preference", + "default": "always", + "why": "always is what the distribution's kernel boots with, so assigning the module changes nothing until a machine sets another value; madvise gives huge pages only to programs that ask for them, which stops a small write to a mapped file (the journal) dirtying and writing back a whole 2 MiB page on a machine whose disks are slow (novox/hq issue 746)" + } + }, + "tools": [ + "kernel_huge_pages" + ], + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": "." + }, + { + "id": "huge-pages", + "type": "file", + "path": "${dir:state}/huge-pages", + "mode": "0755", + "trusted": true, + "content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file.\n#\n# huge-pages apply|restore [ []]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\tfi\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [ []]\" >&2\n\texit 2\n\t;;\nesac\n" + }, + { + "id": "unit", + "type": "file", + "path": "/etc/systemd/system/mesh-kernel-huge-pages.service", + "mode": "0644", + "content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; stopping it gives back the mode the machine had before. It runs early in boot,\n# before the services that fill the page cache, and again whenever the setting changes.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n\n[Install]\nWantedBy=sysinit.target\n" + }, + { + "id": "apply", + "type": "service", + "unit": "mesh-kernel-huge-pages.service", + "state": "running", + "boot": "enabled", + "restart-on": [ + "huge-pages", + "unit" + ], + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/kernel-tools", + "binary": "kernel-tools", + "loads": [ + "kernel-tools" + ] + } + ] + } +} From ab1811d00357a95329870b0c3a7614168a12480b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 23:42:49 +0200 Subject: [PATCH 08/11] Reload, not restart, the huge pages unit when the setting changes (hq issue 746) A restart ran restore then apply, so every change passed back through the found mode and rotated the journal again. Reload applies the new value directly; restore also drops its state; no rotation while the machine boots; the unit is sandboxed. --- modules/kernel/README.md | 14 ++-- .../kernel/cmd/kernel-tools/manifest_test.go | 78 ++++++++++++++++--- modules/kernel/module.json | 8 +- 3 files changed, 81 insertions(+), 19 deletions(-) diff --git a/modules/kernel/README.md b/modules/kernel/README.md index faf4cc94..dddcebd4 100644 --- a/modules/kernel/README.md +++ b/modules/kernel/README.md @@ -22,8 +22,8 @@ lost its lease four times in 30 minutes. With `madvise`, only memory a program a |---|---|---| | setting | `transparent-huge-pages` | `always`, `madvise` or `never`. Default `always`, what the distribution's kernel boots with, so assigning the module changes nothing until a machine sets another value | | file | `${dir:state}/huge-pages` | the script, written from the setting: `apply` writes the mode to `/sys/kernel/mm/transparent_hugepage/enabled` and reads it back (a value that does not take fails the unit); `restore` gives back the mode found before the module first set it, kept in `${dir:state}/found` | -| file | `/etc/systemd/system/mesh-kernel-huge-pages.service` | oneshot, early in boot (`sysinit.target`, before the services that fill the page cache); stopping it runs `restore` | -| service | `mesh-kernel-huge-pages.service` | running and enabled, restarted when the script (so the setting) or the unit changes; its health is the unit's | +| file | `/etc/systemd/system/mesh-kernel-huge-pages.service` | oneshot, early in boot (`sysinit.target`, before the services that fill the page cache); reloading it runs `apply`, stopping it runs `restore`, which also removes `found`. Sandboxed: the file system read-only but for the module's state (`/sys` stays writable), no home, no network, no new privileges; the journal's rotation goes over its local socket, which a private network leaves reachable | +| service | `mesh-kernel-huge-pages.service` | running and enabled; **reloaded** when the script (so the setting) changes, so a new value is set directly and never passes back through the found one; restarted only when the unit changes; its health is the unit's | The per-size controls (`hugepages-kB/enabled`) are left alone: on the machines read, 2048 kB says `inherit` (it follows the mode) and every other size `never`. `kernel_huge_pages` shows each one. @@ -31,8 +31,9 @@ The per-size controls (`hugepages-kB/enabled`) are left alone: on the machine **A folio already in the page cache keeps its size.** A new mode applies to pages faulted or collapsed from then on. The active journal files would keep their 2 MiB folios until archived, so `apply` rotates the journal (`journalctl --rotate`) whenever it moves the mode away from `always`: new files are mapped -under the new mode. A failed rotation is said in the unit's journal and does not fail the setting. At -boot the kernel starts at `always`, so a machine set to `madvise` rotates its journal once per boot. +under the new mode. A failed rotation is said in the unit's journal and does not fail the setting. No +rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing` +or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit. ## Tools @@ -43,8 +44,9 @@ boot the kernel starts at `always`, so a machine set to `madvise` rotates its jo ## Tests `go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the -manifest's shape (the default, the order, the unit, restart and health); and the script as the mesh +manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once, -rotates the journal once and not for `always`, gives back what it found, refuses a value the kernel +rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a +reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel does not take before writing anything, fails when the write does not take, and says when the kernel has no transparent huge pages. diff --git a/modules/kernel/cmd/kernel-tools/manifest_test.go b/modules/kernel/cmd/kernel-tools/manifest_test.go index fd1a0327..bcdc42e3 100644 --- a/modules/kernel/cmd/kernel-tools/manifest_test.go +++ b/modules/kernel/cmd/kernel-tools/manifest_test.go @@ -2,8 +2,9 @@ package main // The module's shape (novox/hq issue 746): the setting's default is what the kernel boots with, so // assigning the module changes nothing until a machine sets another value; the script is written into the -// module's own state and run by a oneshot unit that is restarted whenever the script (and so the setting) -// or the unit changes; and the script itself sets, verifies, gives back and refuses as it says. +// module's own state and run by a sandboxed oneshot unit that is reloaded (applied again) when the script, +// and so the setting, changes and restarted only when the unit does; and the script itself sets, verifies, +// gives back and refuses as it says. import ( "os" @@ -34,7 +35,9 @@ func TestTheDefaultChangesNothingAndTheUnitFollowsTheSetting(t *testing.T) { } unit := m.resource(t, "unit")["content"].(string) for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/huge-pages apply", - "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target", "DefaultDependencies=no"} { + "ExecReload=${dir:state}/huge-pages apply", "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target", + "DefaultDependencies=no", "ProtectSystem=strict", "ReadWritePaths=${dir:state}", "ProtectHome=yes", "PrivateNetwork=yes", + "NoNewPrivileges=yes"} { if !strings.Contains(unit, line+"\n") { t.Errorf("the unit lacks %q", line) } @@ -43,9 +46,14 @@ func TestTheDefaultChangesNothingAndTheUnitFollowsTheSetting(t *testing.T) { if apply["unit"] != "mesh-kernel-huge-pages.service" || apply["state"] != "running" || apply["boot"] != "enabled" { t.Fatalf("apply: %v", apply) } - if on := apply["restart-on"].([]any); len(on) != 2 || on[0] != "huge-pages" || on[1] != "unit" { + // A new value reloads (apply again); only a new unit restarts, whose stop would pass back through the + // mode the machine was found with. + if on := apply["restart-on"].([]any); len(on) != 1 || on[0] != "unit" { t.Fatalf("restart-on: %v", on) } + if on := apply["reload-on"].([]any); len(on) != 1 || on[0] != "huge-pages" { + t.Fatalf("reload-on: %v", on) + } if h := apply["health"].(map[string]any); h["kind"] != "unit" { t.Fatalf("health: %v", h) } @@ -54,12 +62,17 @@ func TestTheDefaultChangesNothingAndTheUnitFollowsTheSetting(t *testing.T) { // bed is the script as the mesh writes it for one value of the setting, a stand-in transparent_hugepage // directory, the module's state, and a PATH whose journalctl records that it was asked to rotate. type bed struct { - script, sys, state, rotated, path string + script, sys, state, rotated, path, bin string } func newBed(t *testing.T, want, found string) bed { + return newBedIn(t, t.TempDir(), want, found) +} + +// newBedIn writes the script for want into dir; a second bed in the same dir shares the machine and the +// state, as a reload with a new value does. +func newBedIn(t *testing.T, dir, want, found string) bed { t.Helper() - dir := t.TempDir() b := bed{script: filepath.Join(dir, "huge-pages"), sys: filepath.Join(dir, "thp"), state: filepath.Join(dir, "state"), rotated: filepath.Join(dir, "rotated")} content := manifest(t).resource(t, "huge-pages")["content"].(string) @@ -75,12 +88,22 @@ func newBed(t *testing.T, want, found string) bed { } } write(t, b.script, content, 0o755) - write(t, filepath.Join(b.sys, "enabled"), found, 0o644) + if found != "" { + write(t, filepath.Join(b.sys, "enabled"), found, 0o644) + } write(t, filepath.Join(bin, "journalctl"), "#!/bin/sh\necho \"$@\" >>"+b.rotated+"\n", 0o755) + b.bin = bin + b.booted(t, "running") b.path = bin + string(os.PathListSeparator) + os.Getenv("PATH") return b } +// booted makes the service manager say state for is-system-running. +func (b bed) booted(t *testing.T, state string) { + t.Helper() + write(t, filepath.Join(b.bin, "systemctl"), "#!/bin/sh\n[ \"$1\" = is-system-running ] && echo "+state+"\n", 0o755) +} + func write(t *testing.T, p, content string, mode os.FileMode) { t.Helper() if err := os.WriteFile(p, []byte(content), mode); err != nil { @@ -134,6 +157,44 @@ func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "always\n" { t.Fatalf("restored: %q", got) } + // Nothing of the module is kept once it is stopped. + if _, err := os.Stat(filepath.Join(b.state, "found")); !os.IsNotExist(err) { + t.Fatalf("found kept after restore: %v", err) + } +} + +func TestAReloadWithANewValueGoesStraightThereAndNeverRewritesFound(t *testing.T) { + dir := t.TempDir() + b := newBedIn(t, dir, "madvise", "[always] madvise never\n") + if out, err := b.run(t, "apply"); err != nil { + t.Fatalf("apply: %v %s", err, out) + } + // The setting changes to never: the mesh rewrites the script and reloads the unit, which applies again. + n := newBedIn(t, dir, "never", "") + out, err := n.run(t, "apply") + if err != nil || !strings.Contains(out, "madvise -> never") { + t.Fatalf("reload: %v %s", err, out) + } + if got := n.file(t, filepath.Join(n.state, "found")); got != "always\n" { + t.Fatalf("found rewritten by a reload: %q", got) + } +} + +func TestNoRotationWhileTheMachineIsStarting(t *testing.T) { + for _, state := range []string{"initializing", "starting"} { + b := newBed(t, "madvise", "[always] madvise never\n") + b.booted(t, state) + out, err := b.run(t, "apply") + if err != nil || !strings.Contains(out, "still starting") { + t.Fatalf("%s: %v %s", state, err, out) + } + if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" { + t.Fatalf("%s: not set: %q", state, got) + } + if got := b.file(t, b.rotated); got != "" { + t.Fatalf("%s: rotated at boot: %q", state, got) + } + } } func TestAlwaysIsSetWithoutRotating(t *testing.T) { @@ -172,9 +233,6 @@ func TestAWriteThatDoesNotTakeFailsTheUnit(t *testing.T) { func TestAKernelWithoutHugePagesIsSaid(t *testing.T) { b := newBed(t, "madvise", "") - if err := os.Remove(filepath.Join(b.sys, "enabled")); err != nil { - t.Fatal(err) - } if out, err := b.run(t, "apply"); err == nil || !strings.Contains(out, "built without transparent huge pages") { t.Fatalf("%v %s", err, out) } diff --git a/modules/kernel/module.json b/modules/kernel/module.json index 68645769..15077979 100644 --- a/modules/kernel/module.json +++ b/modules/kernel/module.json @@ -27,14 +27,14 @@ "path": "${dir:state}/huge-pages", "mode": "0755", "trusted": true, - "content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file.\n#\n# huge-pages apply|restore [ []]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\tfi\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [ []]\" >&2\n\texit 2\n\t;;\nesac\n" + "content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode\n# moves straight to the new value and never passes back through the one the machine was found with.\n#\n# huge-pages apply|restore [ []]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\t# At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation\n\t# while the service manager is still bringing the machine up.\n\tbooting=$(systemctl is-system-running 2>/dev/null || true)\n\tcase \"$booting\" in\n\tinitializing | starting) echo \"the machine is still starting: nothing to rotate yet\" ;;\n\t*)\n\t\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\t\tfi\n\t\t;;\n\tesac\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t# Unassigned or shut down: nothing of the module is kept; the next start records what it finds again.\n\trm -f \"$state/found\"\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [ []]\" >&2\n\texit 2\n\t;;\nesac\n" }, { "id": "unit", "type": "file", "path": "/etc/systemd/system/mesh-kernel-huge-pages.service", "mode": "0644", - "content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; stopping it gives back the mode the machine had before. It runs early in boot,\n# before the services that fill the page cache, and again whenever the setting changes.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n\n[Install]\nWantedBy=sysinit.target\n" + "content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the\n# mode the machine had before. It runs early in boot, before the services that fill the page cache.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecReload=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket.\nProtectSystem=strict\nReadWritePaths=${dir:state}\nProtectHome=yes\nPrivateNetwork=yes\nNoNewPrivileges=yes\n\n[Install]\nWantedBy=sysinit.target\n" }, { "id": "apply", @@ -43,9 +43,11 @@ "state": "running", "boot": "enabled", "restart-on": [ - "huge-pages", "unit" ], + "reload-on": [ + "huge-pages" + ], "health": { "kind": "unit" } From 48f0662fdea295ac81609eb3bda01a8655e535aa Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 22:28:25 +0000 Subject: [PATCH 09/11] Make the forge's tools act as an account of their own, not the administrator (issue 469, ADR 0365 clause 2) The tools' token reached everything the administrator may do, so their boundary rested on their own guards. The gitea module now makes a plain account, mesh-tools, with the admin login, gives it write on code, issues and pull requests of every novox repository through one team, and mints the tools' token for it; the admin's old token goes only after the new one is kept. The Go forge bundle sends that token, never the admin login. The forge shows protection rules only to a repository administrator, so every check reads a branch's protection from the branch. mesh-admin stays a ready author, so marks made before the move keep counting. Co-Authored-By: Claude Opus 5.5 --- modules/gitea/README.md | 13 +- modules/gitea/client.ts | 63 +++- modules/gitea/cmd/forge/account_test.go | 120 +++++++ modules/gitea/cmd/forge/commit.go | 29 +- modules/gitea/cmd/forge/forge.go | 9 +- modules/gitea/cmd/forge/forge_test.go | 8 +- modules/gitea/index.ts | 18 +- modules/gitea/internal/forgehttp/forgehttp.go | 87 ++++- modules/gitea/merge.ts | 9 +- modules/gitea/module.json | 9 +- modules/gitea/pulls.ts | 20 +- modules/gitea/test/announcing.test.ts | 21 +- modules/gitea/test/guard.test.ts | 20 +- modules/gitea/test/token.test.ts | 21 +- modules/gitea/test/tools-account.test.ts | 332 ++++++++++++++++++ modules/gitea/token.ts | 146 ++++++-- modules/gitea/tools-account.ts | 52 +++ modules/gitea/tools/index.ts | 10 +- 18 files changed, 887 insertions(+), 100 deletions(-) create mode 100644 modules/gitea/cmd/forge/account_test.go create mode 100644 modules/gitea/test/tools-account.test.ts create mode 100644 modules/gitea/tools-account.ts diff --git a/modules/gitea/README.md b/modules/gitea/README.md index cb5edb23..c769a732 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -52,7 +52,10 @@ base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an ag | `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path | **Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266): -never the repository's default branch, never a branch a protection rule names. A rule covers a branch as the +never the repository's default branch, never a branch a protection rule names. The tools' account may not read the +rules themselves (the forge keeps them for a repository's administrator, novox/hq ADR 0365): for it an existing branch +is judged by the protection the forge says it applies there, and a branch not made yet by the forge itself, which +refuses the commit that would make a branch a rule guards. A rule covers a branch as the forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob (`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the @@ -66,7 +69,7 @@ the commit the branch (or, for a new branch, its base) was read at, never at its forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than the file. Neither endpoint takes a commit to build on, only a branch (Gitea 1.27), so the parent answered is the one the forge says it made the commit on; when the branch moved between the read and the commit, the answer -says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the forge's admin account, so its sha is not the +says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the tools' account, so its sha is not the local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a branch not at the commit the forge named is a failure, never a success. @@ -75,8 +78,10 @@ is read once and searched in memory; a binary file is passed over, a file over 4 `skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`. -It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), and one -tool call takes at most 50 seconds. +It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), as the +tools' own account (novox/hq ADR 0365 §2, issue 469): the token the TypeScript bundle mints for `mesh-tools` and +keeps (`MESH_GITEA_TOKEN_FILE`), sent only when its scopes line names that account, never the admin login. One tool +call takes at most 50 seconds. `go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on a new branch from its base; an existing branch committed on; the default branch, a protected branch and unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 389f3b62..00524c4c 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -8,6 +8,7 @@ import { ConfiguredToken, MintedToken, type TokenSource } from "./token.js"; import { protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js"; import { Refused, checkSentPath, pathSegment, pathSegments } from "./guard.js"; import { isCommit } from "./checks.js"; +import { TOOLS_ACCOUNT } from "./tools-account.js"; /** A repository's path on the forge, its owner and name each one segment (guard.ts pathSegment): uniform for every * call, whether the names came from a caller or from the forge's own answer. */ @@ -124,10 +125,12 @@ export interface ReadyMark { /** **The accounts whose "ready at" comment counts** (novox/hq issue 722): the trusted setting ready-authors, which * the mesh writes to the file MESH_GITEA_READY_AUTHORS_FILE names as a line `authors= …`. A file - * that cannot be read leaves the mesh's own forge account alone, said; never an empty list read as everyone. */ + * that cannot be read leaves the mesh's own forge accounts alone, said; never an empty list read as everyone: the + * account the tools act as (novox/hq ADR 0365) and the admin account, whose marks were made before it. */ export function readyAuthors(file = process.env.MESH_GITEA_READY_AUTHORS_FILE, own = process.env.MESH_GITEA_ADMIN_USER, - read: (f: string) => string = (f) => readFileSync(f, "utf8")): string[] { - const fallback = own ? [own] : []; + read: (f: string) => string = (f) => readFileSync(f, "utf8"), + tools = process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT): string[] { + const fallback = [...new Set([tools, own].filter((a): a is string => !!a))]; if (!file) return fallback; try { const line = read(file).split("\n").find((l) => l.startsWith("authors=")); @@ -444,13 +447,15 @@ export class GiteaClient { * fault in the announcer cannot hide itself. Reads only. */ async openPulls(readyAuthors: string[] = []): Promise { const out: OpenPull[] = []; - const rules = new Map(); + const rules = new Map(); for (const found of await this.searchOpenPulls()) { const full = `${found.owner}/${found.name}`; const pull = await this.getPullRequest(found.owner, found.name, found.number); if (pull.state !== "open" || !pull.head_sha) continue; - if (!rules.has(full)) rules.set(full, await this.branchProtections(found.owner, found.name).catch(() => undefined)); - const rule = rules.get(full)?.find((p) => (p.rule_name ?? p.branch_name) === (pull.base || "main")); + // The base's protection as the forge applies it to that branch (appliedProtection): readable without administration. + const base = `${full}:${pull.base || "main"}`; + if (!rules.has(base)) rules.set(base, await this.appliedProtection(found.owner, found.name, pull.base || "main").catch(() => undefined)); + const rule = rules.get(base); const checks = await this.headChecks(found.owner, found.name, pull.head_sha); const labels = found.labels ?? []; // The ready mark (novox/hq issue 722): its timeline is read, whole, only for a pull request that carries the label. @@ -673,16 +678,44 @@ export class GiteaClient { // ---- Branch protection ---- - /** A repository's branch protection rules. */ + /** A repository's branch protection rules. The forge answers them to a repository's administrator alone, which the + * tools' account is not (novox/hq ADR 0365 §2): for it this is refused (403), and what a branch's protection asks + * is read with appliedProtection. */ async branchProtections(owner: string, repo: string): Promise { return (await this.request(`${repoPath(owner, repo)}/branch_protections`)) ?? []; } - /** The rule named for a branch, null when it has none. */ + /** The rule named for a branch, null when it has none — from the rules' list, so a repository administrator's. */ async branchProtection(owner: string, repo: string, rule: string): Promise { return (await this.branchProtections(owner, repo)).find((p) => (p.rule_name ?? p.branch_name) === rule) ?? null; } + /** + * The protection that covers an existing branch, as the forge applies it, or null when none does. Read from the + * branch itself, which anyone who reads the code may read — the rules' own list is a repository administrator's + * (novox/hq ADR 0365 §2) — and as the forge matches it, a glob rule included (its name is the rule's). The + * branch's `user_can_push` stands for `enable_push`: whether this account may push there. A branch that does not + * exist is null: nothing covers it yet, and the forge judges a push that makes it. + */ + async appliedProtection(owner: string, repo: string, branch: string): Promise { + let b: any; + try { + b = await this.request(`${repoPath(owner, repo)}/branches/${pathSegments("branch", branch)}`); + } catch (err) { + if (/: 404 /.test(String((err as Error)?.message))) return null; + throw err; + } + if (!b?.protected) return null; + return { + rule_name: String(b.effective_branch_protection_name || branch), + branch_name: branch, + enable_push: Boolean(b.user_can_push), + enable_status_check: Boolean(b.enable_status_check), + status_check_contexts: Array.isArray(b.status_check_contexts) ? b.status_check_contexts.map(String) : [], + required_approvals: Number(b.required_approvals ?? 0), + }; + } + /** Make a branch require these commit statuses to merge (novox/hq ADR 0237): the rule named for the * branch is edited — its required statuses replaced by these, everything else it says kept unless * asked — or created, refusing direct pushes unless asked otherwise. Answers the rule as it is now. */ @@ -948,12 +981,16 @@ export class GiteaAdmin { // unit the team does not have — so code read must be said here: without it gitea answers a // member's clone of a private repository with "not found", which is how the builder's first // credentialed clone failed against a team that named only packages. - const units = { + return this.ensureTeamWith(org, team, { permission: "read", units_map: { "repo.code": "read", "repo.packages": packageWrite ? "write" : "read" }, includes_all_repositories: true, can_create_org_repo: false, - }; + }); + } + + /** Ensure a team exists with exactly these units, and return its id (ensureTeam; the tools' team, tools-account.ts). */ + async ensureTeamWith(org: string, team: string, units: Record): Promise { const found = await this.findTeam(org, team); if (found !== null) { const patch = await this.request(`/teams/${found}`, { @@ -988,7 +1025,7 @@ export class GiteaAdmin { * The edit path is taken only when the user actually exists. A 422 from the create is also what * a plain validation failure returns, and reading it as "already there" made the follow-up edit * 404 — burying the create's own message, which is the one that says what is actually wrong. */ - async ensureUser(username: string, password: string, email: string): Promise { + async ensureUser(username: string, password: string, email: string, plain = false): Promise { const res = await this.request("/admin/users", { method: "POST", body: JSON.stringify({ username, email, password, must_change_password: false }), @@ -1002,7 +1039,9 @@ export class GiteaAdmin { // login_name is required by the admin edit endpoint; for a local user it is the username. // active and prohibit_login: a deactivated or login-prohibited user is refused like a wrong // password, so the provisioner's check reports it lost; applying again must undo both. - body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false }), + // plain: never a site administrator — one made so by hand is made a plain account again (novox/hq ADR 0365). + body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false, + ...(plain ? { admin: false } : {}) }), }); if (patch.status === 200) return; GiteaAdmin.fail(`/admin/users/${username}`, patch); diff --git a/modules/gitea/cmd/forge/account_test.go b/modules/gitea/cmd/forge/account_test.go new file mode 100644 index 00000000..d37c6594 --- /dev/null +++ b/modules/gitea/cmd/forge/account_test.go @@ -0,0 +1,120 @@ +package main + +// The commit and the reads act as the tools' own account (novox/hq ADR 0365 §2, issue 469): the token the module +// keeps for `mesh-tools`, never the admin login. That account is no repository's administrator, so the forge +// refuses it the protection rules (403, as gitea's reqAdmin does); the commit then judges an existing branch by the +// protection the forge says it applies, and leaves a branch not made yet to the forge's own judgement. + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "gitea/internal/forgehttp" +) + +const toolsToken = "tools-token-never-shown" + +// asTools serves the fake forge to a bundle holding the tools' token: only that token is taken, the rules are +// refused as to a non-administrator, and the branches named protected say so as the forge does. +func asTools(t *testing.T, f *fakeForge, protected map[string]string, scopes string) (*Forge, *[]string) { + dir := t.TempDir() + file := filepath.Join(dir, "token") + if err := os.WriteFile(file, []byte(toolsToken+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file+".scopes", []byte(scopes+"\n"), 0o600); err != nil { + t.Fatal(err) + } + var seen []string + const repo = "/api/v1/repos/acme/app" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Method+" "+r.URL.Path) + if _, _, basic := r.BasicAuth(); basic || r.Header.Get("Authorization") != "token "+toolsToken { + http.Error(w, "unauthorized", 401) + return + } + if r.URL.Path == repo+"/branch_protections" { + http.Error(w, `{"message":"token does not have the administration of this repository"}`, 403) + return + } + if name := strings.TrimPrefix(r.URL.Path, repo+"/branches/"); r.Method == "GET" && name != r.URL.Path { + if rule, ok := protected[name]; ok { + _, _ = w.Write([]byte(`{"name":"` + name + `","commit":{"id":"c0"},"protected":true,"effective_branch_protection_name":"` + rule + `"}`)) + return + } + } + r.SetBasicAuth("mesh-admin", f.password) // the fake's own door, past the token checked above + f.handle(w, r) + })) + t.Cleanup(srv.Close) + return &Forge{Client: forgehttp.Client{URL: srv.URL, TokenFile: file, Account: "mesh-tools"}}, &seen +} + +func TestTheToolsAccountCommitsOnANewBranchWithItsTokenThoughTheRulesAreNotItsToRead(t *testing.T) { + f := newFake(t) + forge, _ := asTools(t, f, nil, "read:user,write:issue,write:repository for mesh-tools") + got, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365", + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err != nil { + t.Fatal(err) + } + if len(f.writes) != 1 || !got.Created { + t.Fatalf("%d writes, %+v", len(f.writes), got) + } +} + +func TestTheToolsAccountIsRefusedAProtectedBranchTheForgeSaysItGuardsAndTheDefaultBranch(t *testing.T) { + for branch, rule := range map[string]string{"release/1.0": "release/*", "fix/1-old": "fix/1-old", "main": ""} { + f := newFake(t) + protected := map[string]string{} + if rule != "" { + protected[branch] = rule + } + forge, _ := asTools(t, f, protected, "read:user,write:issue,write:repository for mesh-tools") + _, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": branch, + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err == nil || !strings.Contains(err.Error(), "refused") || !strings.Contains(err.Error(), "ADR 0266") { + t.Errorf("%s: %v", branch, err) + } + if rule != "" && !strings.Contains(err.Error(), rule) { + t.Errorf("%s: the rule %s is not named: %v", branch, rule, err) + } + if len(f.writes)+len(f.patches) != 0 { + t.Errorf("%s: something was written", branch) + } + } +} + +func TestATokenKeptForTheAdministratorIsNeverSent(t *testing.T) { + f := newFake(t) + // What the module kept before ADR 0365: the admin's token, its scopes line naming no account. + forge, seen := asTools(t, f, nil, "read:user,write:issue,write:repository") + _, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365", + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err == nil || !strings.Contains(err.Error(), "not mesh-tools's") { + t.Fatalf("%v", err) + } + if len(*seen) != 0 { + t.Fatalf("the forge was asked %v", *seen) + } +} + +func TestTheBundleHandedATokenFileNeedsNoAdminLogin(t *testing.T) { + t.Setenv("MESH_GITEA_URL", "http://127.0.0.1:3000") + t.Setenv("MESH_GITEA_TOKEN_FILE", "/state/token") + t.Setenv("MESH_GITEA_TOOLS_USER", "") + t.Setenv("MESH_GITEA_ADMIN_USER", "") + t.Setenv("MESH_GITEA_ADMIN_PASSWORD_FILE", "") + f, err := ForgeFromEnv() + if err != nil { + t.Fatal(err) + } + if f.TokenFile != "/state/token" || f.Account != "mesh-tools" || f.User != "" || f.PasswordFile != "" { + t.Fatalf("%+v", f.Client) + } +} diff --git a/modules/gitea/cmd/forge/commit.go b/modules/gitea/cmd/forge/commit.go index 6976da57..d3c2af52 100644 --- a/modules/gitea/cmd/forge/commit.go +++ b/modules/gitea/cmd/forge/commit.go @@ -9,6 +9,8 @@ import ( "net/url" "strings" "unicode/utf8" + + "gitea/internal/forgehttp" ) // gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull @@ -211,7 +213,32 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { // Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed). var rules []Protection if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil { - return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err) + var status *forgehttp.StatusError + if !errors.As(err, &status) || status.Status != http.StatusForbidden { + return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err) + } + // The rules are a repository administrator's to read, and the tools' account is none (novox/hq ADR 0365 + // §2). What it may read is the protection the forge applies to an existing branch: one protected is + // refused here. A branch not made yet is covered by no rule this account can see, and the forge itself + // judges the commit that makes it against every rule, refusing one a rule guards. + if err := targetRefused(a.Branch, r.DefaultBranch, nil); err != nil { + return nil, err + } + b, err := f.branch(ctx, repo, a.Branch) + if err != nil { + return nil, fmt.Errorf("whether a protection rule guards %s of %s/%s could not be read, so nothing was sent: %w", a.Branch, a.Owner, a.Repo, err) + } + if b != nil && b.Protected { + rule := b.Rule + if rule == "" { + rule = a.Branch + } + rules = []Protection{{RuleName: rule}} + if !ruleCovers(rule, a.Branch) { + return nil, refuse("a commit on %s is refused: the protection rule %s guards it (novox/hq ADR 0266). Put it on a "+ + "branch of its own. Nothing was sent", a.Branch, rule) + } + } } if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil { return nil, err diff --git a/modules/gitea/cmd/forge/forge.go b/modules/gitea/cmd/forge/forge.go index 3637116d..a28a915b 100644 --- a/modules/gitea/cmd/forge/forge.go +++ b/modules/gitea/cmd/forge/forge.go @@ -11,8 +11,8 @@ import ( "gitea/internal/forgehttp" ) -// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account -// (internal/forgehttp, shared with the npm-registry bundle beside it). +// The forge as this bundle reaches it: its own HTTP interface on the machine, as the tools' own account (novox/hq +// ADR 0365 §2; internal/forgehttp, shared with the npm-registry bundle beside it). // Forge is the forge's HTTP interface. type Forge struct { @@ -91,12 +91,15 @@ func (p Protection) name() string { return p.BranchName } -// Branch is one branch with the commit it points at. +// Branch is one branch with the commit it points at, and the protection the forge applies to it: what an account +// that may not read the rules themselves (the tools' account, novox/hq ADR 0365) knows of them. type Branch struct { Name string `json:"name"` Commit struct { ID string `json:"id"` } `json:"commit"` + Protected bool `json:"protected"` + Rule string `json:"effective_branch_protection_name"` } // branch is the named branch, nil when the forge has none of that name. diff --git a/modules/gitea/cmd/forge/forge_test.go b/modules/gitea/cmd/forge/forge_test.go index 7d9464df..1d315cc3 100644 --- a/modules/gitea/cmd/forge/forge_test.go +++ b/modules/gitea/cmd/forge/forge_test.go @@ -619,11 +619,17 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) { if len(a.Loads) != 1 || a.Loads[0] != a.Binary { t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary) } - for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} { + // The tools' own account, never the admin login (novox/hq ADR 0365 §2). + for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE"} { if a.Env[k] == "" { t.Errorf("the bundle is not given %s", k) } } + for _, k := range []string{"MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} { + if a.Env[k] != "" { + t.Errorf("the bundle is given %s: it acts as the tools' account, not the administrator", k) + } + } } } if !found { diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index b9c80da7..c730bc2a 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -42,7 +42,7 @@ import { commitStatusTool } from "./delivery.js"; import { mergeTool } from "./merge.js"; import { GiteaAdmin, GiteaClient, movedSince } from "./client.js"; import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked, - type RepoCheckFacts, reposToLook } from "./pulls.js"; + repoCheckFacts, reposToLook } from "./pulls.js"; import { GateWriter, clonedAs, type BuildStarted } from "./gate.js"; import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js"; import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js"; @@ -400,22 +400,6 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`); } -// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the -// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection -// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads -// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is -// required would let an untested repository merge. -async function repoCheckFacts(client: GiteaClient, c: Checked, base: string | undefined): Promise { - if (c["repo-check"]?.verdict !== "warning") return {}; - const defined = await client.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined); - if (defined !== false) return { defined }; - const rules = await client.branchProtections(c.owner, c.repo).catch(() => undefined); - if (!rules) return { defined }; - const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === (base || "main")); - const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT); - return { defined, required }; -} - // **This module is the one writer of a head's mesh/merge-gate** (novox/hq issue 446): the announcement's pending words, // where the check is when a build machine takes it, and the verdict — each head's writes in one chain, the verdict last // (gate.ts). A second writer elsewhere raced the verdict and could leave a pass reading pending. diff --git a/modules/gitea/internal/forgehttp/forgehttp.go b/modules/gitea/internal/forgehttp/forgehttp.go index 9b76e553..3b3e8c89 100644 --- a/modules/gitea/internal/forgehttp/forgehttp.go +++ b/modules/gitea/internal/forgehttp/forgehttp.go @@ -1,7 +1,17 @@ // Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge, -// cmd/npm-registry): the forge on its own machine, as its admin account, whose password the vault delivered -// into a file this module owns (novox/hq ADR 0086), by basic authentication. The password is read when a call -// is made and never logged, answered or put in an error. One copy for both bundles (novox/hq issue 539). +// cmd/npm-registry), on the forge's own machine. One copy for both bundles (novox/hq issue 539). +// +// Two ways in, by what the manifest hands the bundle: +// +// - **as the tools' account** (novox/hq ADR 0365 §2), when MESH_GITEA_TOKEN_FILE names the token the module +// keeps for it (token.ts mints it, for `mesh-tools`, and keeps it at 0600 with its scopes line beside it). A +// token whose scopes line does not say it is the tools' account's — one kept from before, the administrator's — +// is never sent: the call fails, saying so, until the module has minted the account's own. This bundle mints +// nothing; a token the forge rejects is renewed by the module's next call, and said here. +// - **as the admin account**, whose password the vault delivered into a file this module owns (novox/hq ADR +// 0086), by basic authentication, when no token file is named. +// +// A credential is read when a call is made and never logged, answered or put in an error. package forgehttp import ( @@ -25,12 +35,31 @@ type Client struct { URL string // e.g. http://127.0.0.1:3000 User string PasswordFile string - HTTP *http.Client // when nil, a client with Timeout - Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero + // TokenFile, when set, is the tools' account's token (MESH_GITEA_TOKEN_FILE), sent instead of the admin login; + // Account the account its scopes line must name (MESH_GITEA_TOOLS_USER, default mesh-tools). + TokenFile string + Account string + HTTP *http.Client // when nil, a client with Timeout + Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero } -// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. +// ToolsAccount is the account the forge's tools act as when the manifest names none (token.ts, tools-account.ts). +const ToolsAccount = "mesh-tools" + +// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. A bundle +// handed MESH_GITEA_TOKEN_FILE acts as the tools' account and needs no admin login. func FromEnv() (Client, []string) { + if file := os.Getenv("MESH_GITEA_TOKEN_FILE"); file != "" { + c := Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), TokenFile: file, + Account: os.Getenv("MESH_GITEA_TOOLS_USER")} + if c.Account == "" { + c.Account = ToolsAccount + } + if c.URL == "" { + return c, []string{"MESH_GITEA_URL"} + } + return c, nil + } c := Client{ URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), User: os.Getenv("MESH_GITEA_ADMIN_USER"), @@ -80,14 +109,45 @@ func (c *Client) password() (string, error) { return strings.TrimSpace(string(raw)), nil } +// token is the tools' account's token, refused unless its scopes line names that account. +func (c *Client) token() (string, error) { + raw, err := os.ReadFile(c.TokenFile) + if err != nil { + return "", fmt.Errorf("the tools' token is not kept yet (the module mints it for %s at its first call): %w", c.Account, err) + } + line, err := os.ReadFile(c.TokenFile + ".scopes") + if err != nil || !strings.HasSuffix(strings.TrimSpace(string(line)), " for "+c.Account) { + return "", fmt.Errorf("the token kept at %s is not %s's (novox/hq ADR 0365): the module mints that account's own at its next call; nothing was sent", c.TokenFile, c.Account) + } + token := strings.TrimSpace(string(raw)) + if token == "" { + return "", fmt.Errorf("the token kept at %s is empty; nothing was sent", c.TokenFile) + } + return token, nil +} + +// authorize puts the bundle's credential on a request: the tools' token, or the admin login. +func (c *Client) authorize(req *http.Request) error { + if c.TokenFile != "" { + token, err := c.token() + if err != nil { + return err + } + req.Header.Set("Authorization", "token "+token) + return nil + } + password, err := c.password() + if err != nil { + return err + } + req.SetBasicAuth(c.User, password) + return nil +} + // Do sends one request, with payload as its JSON body when not nil, and answers the body, read up to limit; // 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it // exactly: a cut body is refused, never read as if it were whole. func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) { - password, err := c.password() - if err != nil { - return nil, err - } var body io.Reader if payload != nil { raw, err := json.Marshal(payload) @@ -100,7 +160,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit if err != nil { return nil, err } - req.SetBasicAuth(c.User, password) + if err := c.authorize(req); err != nil { + return nil, err + } req.Header.Set("Accept", "application/json") if payload != nil { req.Header.Set("Content-Type", "application/json") @@ -118,6 +180,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20) } switch { + case res.StatusCode == http.StatusUnauthorized && c.TokenFile != "": + return nil, fmt.Errorf("the forge rejected the tools' token for %s (401): the module mints it afresh at its next call; nothing was done: %w", + method+" "+path, &StatusError{method, path, res.StatusCode, "unauthorized"}) case res.StatusCode == http.StatusNotFound: return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound) case res.StatusCode >= 300: diff --git a/modules/gitea/merge.ts b/modules/gitea/merge.ts index 05ee77a2..9c8e2570 100644 --- a/modules/gitea/merge.ts +++ b/modules/gitea/merge.ts @@ -54,9 +54,10 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition const method = args.method ? String(args.method) : "merge"; if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`); const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch); - // **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here rather than left - // to the forge, whose administrator — the account this tool's token is minted for — may merge past a - // rule that lets it. A rule that cannot be read is no rule passed. + // **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here as well as by the + // forge, which holds the tools' own account to it (novox/hq ADR 0365 §2). The protection is the one the forge + // applies to the base, read from the branch: the rules' own list is a repository administrator's. A rule that + // cannot be read is no rule passed. // **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d, // force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the // head was and nothing told it which head the review had approved. The caller names it; a head that moved @@ -80,7 +81,7 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` + `${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`); } - const rule = await gitea.branchProtection(owner, repo, pull.base); + const rule = await gitea.appliedProtection(owner, repo, pull.base); // Only into a branch whose protection requires the checks: a pull request whose base was moved to a // branch nothing guards would merge past every check (novox/hq ADR 0266). if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) { diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 210c2437..987cd1d6 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -159,8 +159,8 @@ "settings": { "ready-authors": { "kind": "preference", - "default": "mesh-admin jschoubben", - "why": "the forge accounts whose comment \"ready at \" marks a pull request ready for mesh-delivery to merge, separated by spaces: the mesh's own forge account and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word" + "default": "mesh-tools mesh-admin jschoubben", + "why": "the forge accounts whose comment \"ready at \" marks a pull request ready for mesh-delivery to merge, separated by spaces: the account the forge's tools act as (novox/hq ADR 0365), the admin account for marks made before the tools acted as their own, and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word" } }, "resources": [ @@ -308,6 +308,7 @@ "MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json", "MESH_GITEA_ADMIN_USER": "mesh-admin", "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", + "MESH_GITEA_TOOLS_USER": "mesh-tools", "MESH_GITEA_STATE_DIR": "${dir:runtime-state}", "MESH_RECEIVES": "${dir:grants}/npm.json", "MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors", @@ -345,8 +346,8 @@ ], "env": { "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", - "MESH_GITEA_ADMIN_USER": "mesh-admin", - "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret" + "MESH_GITEA_TOOLS_USER": "mesh-tools", + "MESH_GITEA_TOKEN_FILE": "${dir:runtime-state}/token" } } ] diff --git a/modules/gitea/pulls.ts b/modules/gitea/pulls.ts index a51156a1..2c3a549d 100644 --- a/modules/gitea/pulls.ts +++ b/modules/gitea/pulls.ts @@ -9,7 +9,7 @@ // // Pure functions here, so they are tested without a forge; index.ts does the asking and the setting. -import type { CommitStatus, GiteaPull, GiteaRepo } from "./client.js"; +import type { CommitStatus, GiteaClient, GiteaPull, GiteaRepo } from "./client.js"; import { movedSince } from "./client.js"; /** The status context a merge check's gate is kept under: one per commit, the newest replacing the last. */ @@ -176,6 +176,24 @@ export interface RepoCheckFacts { required?: boolean; } +// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the +// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection +// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads +// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is +// required would let an untested repository merge. The base's protection is the one the forge applies, read +// from the branch: the rules' own list is a repository administrator's, which the tools' account is not +// (novox/hq ADR 0365). +export async function repoCheckFacts( + forge: Pick, c: Checked, base: string | undefined): Promise { + if (c["repo-check"]?.verdict !== "warning") return {}; + const defined = await forge.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined); + if (defined !== false) return { defined }; + const rule = await forge.appliedProtection(c.owner, c.repo, base || "main").catch(() => undefined); + if (rule === undefined) return { defined }; + const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT); + return { defined, required }; +} + /** The description of a required repository check that is not defined. */ export const UNDEFINED_REQUIRED = "fail: mesh/repo-check is required here and this head defines no merge-check.sh — " + "a person adds one, or lifts the requirement from the branch's protection"; diff --git a/modules/gitea/test/announcing.test.ts b/modules/gitea/test/announcing.test.ts index 01ce7a05..a7a08c58 100644 --- a/modules/gitea/test/announcing.test.ts +++ b/modules/gitea/test/announcing.test.ts @@ -47,11 +47,18 @@ test("every open pull request is read with what its base requires and what its h if (url.startsWith("/api/v1/repos/novox/mesh-sdk/pulls/14")) { return send({ number: 14, state: "open", base: { ref: "main" }, head: { ref: "y", sha: "bbb2222" }, html_url: "s" }); } - if (url.startsWith("/api/v1/repos/novox/hq/branch_protections")) { - return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/repo-check"] }]); + // The base's protection, read from the branch as the tools' account may (novox/hq ADR 0365); the rules' list is refused it. + if (url.includes("/branch_protections")) { + res.statusCode = 403; + return res.end(JSON.stringify({ message: "the rules are a repository administrator's" })); } - if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branch_protections")) { - return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] }]); + if (url.startsWith("/api/v1/repos/novox/hq/branches/main")) { + return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true, + status_check_contexts: ["mesh/repo-check"] }); + } + if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branches/main")) { + return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true, + status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] }); } if (url.includes("/commits/aaa1111/status")) return send({ statuses: [] }); if (url.includes("/commits/bbb2222/status")) return send({ statuses: [{ context: "mesh/merge-gate", status: "pending", @@ -251,6 +258,8 @@ test("a long thread hides nothing: every page of the timeline is read", async () test("the accounts that may mark are the setting's, and only the mesh's own when it cannot be read", async () => { const { readyAuthors } = await import("../client.ts"); assert.deepEqual(readyAuthors("f", "mesh-admin", () => "# x\nauthors=mesh-admin jschoubben\n"), ["mesh-admin", "jschoubben"]); - assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }), ["mesh-admin"]); - assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n"), ["mesh-admin"]); + // The tools' account and the admin's, whose marks were made before ADR 0365 (issue 469). + assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }, "mesh-tools"), ["mesh-tools", "mesh-admin"]); + assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n", "mesh-tools"), ["mesh-tools", "mesh-admin"]); + assert.deepEqual(readyAuthors(undefined, "mesh-admin", undefined, "mesh-tools"), ["mesh-tools", "mesh-admin"]); }); diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index f14b50ec..567ad023 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -30,6 +30,9 @@ const state = { // What the tools said on the bus. emitted: [] as { type: string; body: any }[], rules: [] as any[], + // Whether the token is a repository administrator's. The tools' account is not (novox/hq ADR 0365): the forge + // refuses it the rules' list, as gitea's reqAdmin does, and it reads a branch's protection from the branch. + admin: false, lastCommit: "2026-01-01T00:00:00Z", // The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's // record of the pull request keeps saying `head` either way, as it would in the race issue 381 is about; and the @@ -59,6 +62,7 @@ beforeEach(() => { state.trunk = TRUNK; state.emitted = []; state.rules = [protectedMain()]; + state.admin = false; state.head = REVIEWED; state.landed = ""; state.merged = false; @@ -87,8 +91,18 @@ const server = createServer(async (req, res) => { if (method !== "GET") state.sent.push({ method, path, body }); const repo = "/repos/novox/mesh-catalog"; if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" }); + if (path.startsWith(`${repo}/branch_protections`) && !state.admin) { + return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" }); + } if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules); - if (method === "GET" && path === `${repo}/branches/main`) return json(res, 200, { name: "main", commit: { id: state.trunk } }); + const branch = method === "GET" && path.startsWith(`${repo}/branches/`) ? decodeURIComponent(path.slice(`${repo}/branches/`.length)) : null; + if (branch !== null) { + // As the forge says it to any reader of the code: the rule that covers the branch, as the forge matches it. + const rule = (state.rules as any[]).find((r) => ruleCovers(String(r.rule_name ?? r.branch_name), branch)); + return json(res, 200, { name: branch, commit: { id: branch === "main" ? state.trunk : "c0ffee" }, protected: !!rule, + ...(rule ? { effective_branch_protection_name: rule.rule_name ?? rule.branch_name, enable_status_check: !!rule.enable_status_check, + status_check_contexts: rule.status_check_contexts ?? [], required_approvals: rule.required_approvals ?? 0, user_can_push: false } : {}) }); + } if (method === "GET" && path === `${repo}/pulls/7/files`) { return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []); } @@ -199,6 +213,7 @@ test("a protection rule covers a branch as the forge decides it: classes, altern }); test("gitea_api still writes what an author writes", async () => { + state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365) await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/issues/3/comments", method: "POST", body: { body: "hi" } }); await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/pulls/7/reviews", method: "POST", body: { event: "COMMENT" } }); await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/branch_protections" }); @@ -206,6 +221,7 @@ test("gitea_api still writes what an author writes", async () => { }); test("gitea_branch_protection_set only strengthens", async () => { + state.admin = true; // a repository administrator's token: the tools' account is refused the rules by the forge const set = (args: Record) => tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "main", ...args }); await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate, mesh\/repo-check/); @@ -396,6 +412,7 @@ test("gitea_merge_pull_request reads back a fast-forward, and says it cannot rea }); test("gitea_delete_repo deletes only the tools' own user's repository, unguarded, untouched for 30 days, named in full", async () => { + state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365) const del = (owner: string, name: string, confirm: unknown) => tool("gitea_delete_repo").run({ owner, name, confirm }); await assert.rejects(del("novox", "mesh-catalog", true), /confirm must be the repository's full name/); await assert.rejects(del("novox", "mesh-catalog", "novox/mesh-catalog"), /not a repository of the tools' own user/); @@ -427,6 +444,7 @@ test("gitea_merge_pull_request merges only into a branch whose protection requir }); test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => { + state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365) state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true, status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true }); const set = (args: Record) => diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index 59cacb90..d72202f2 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -45,6 +45,7 @@ interface Forge { function fakeForge(): Promise { const forge = { + mintedFor: "", mints: 0, lastScopes: null as string[] | null, tokens: new Map(), // name -> value @@ -83,7 +84,14 @@ function fakeForge(): Promise { const tokens = url.pathname.match(/^\/api\/v1\/users\/([^/]+)\/tokens(?:\/([^/]+))?$/); if (tokens) { const user = basic(req); - if (user === null || user !== decodeURIComponent(tokens[1])) return json(res, 401, { message: "auth required" }); + // gitea's reqSelfOrAdmin: the account itself, or the administrator for another (the tools' account, ADR 0365). + if (user === null || (user !== decodeURIComponent(tokens[1]) && user !== ADMIN)) return json(res, 401, { message: "auth required" }); + // This fake keeps one account's tokens: the tools' account's when the module mints for it (ADR 0365), and the + // admin's own list is then empty — nothing of it is retired. + if (decodeURIComponent(tokens[1]) === ADMIN && forge.mintedFor === "mesh-tools") { + return req.method === "GET" ? json(res, 200, []) : json(res, 404, { message: "token not found" }); + } + if (req.method === "POST") forge.mintedFor = decodeURIComponent(tokens[1]); if (req.method === "POST") { const { name, scopes } = await body(req); if (forge.tokens.has(name)) return json(res, 400, { message: "token name has already been used" }); @@ -186,6 +194,17 @@ function fakeForge(): Promise { { full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" }, ]); } + // What setting up the tools' account asks with the admin login (tools-account.ts, ADR 0365): answered plainly. + if (basic(req) === ADMIN) { + const p = url.pathname; + if (req.method === "GET" && p === "/api/v1/user") return json(res, 200, { login: ADMIN, is_admin: true }); + if (req.method === "GET" && p === "/api/v1/users/mesh-tools") return json(res, 200, { login: "mesh-tools", is_admin: false }); + if (req.method === "POST" && p === "/api/v1/admin/users") return json(res, 422, { message: "user already exists" }); + if (req.method === "PATCH" && p === "/api/v1/admin/users/mesh-tools") return json(res, 200, { login: "mesh-tools" }); + if (req.method === "GET" && p === "/api/v1/orgs/novox/teams") return json(res, 200, [{ id: 7, name: "mesh-tools" }]); + if (req.method === "PATCH" && p === "/api/v1/teams/7") return json(res, 200, { id: 7 }); + if (req.method === "PUT" && p === "/api/v1/teams/7/members/mesh-tools") return json(res, 204, null); + } const adminUser = url.pathname.match(/^\/api\/v1\/admin\/users\/([^/]+)$/); if (adminUser && req.method === "PATCH") { const h = req.headers.authorization ?? ""; diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts new file mode 100644 index 00000000..efc6dcfa --- /dev/null +++ b/modules/gitea/test/tools-account.test.ts @@ -0,0 +1,332 @@ +// The forge's tools act as an account of their own, not the site administrator (novox/hq ADR 0365 §2, issue 469): +// the token the tools and the watcher use is minted for `mesh-tools`, a plain account made by the module with the +// administrator's login and given write on the mesh's repositories through one team; the administrator's own +// token by that name goes; a token kept from before, the administrator's, is minted again for the account. +// +// The forge is a fake that keeps accounts, their tokens and the organisation's teams apart, and answers as gitea +// does: an administrator may mint for another account (reqSelfOrAdmin), anyone else only for itself; a protection +// edit needs the repository's administration, which neither a plain account nor its write team holds. + +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { mkdtemp, readFile, writeFile, mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { MintedToken } from "../dist/token.js"; +import { GiteaClient } from "../dist/client.js"; +import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS } from "../dist/tools-account.js"; +import { repoCheckFacts } from "../dist/pulls.js"; + +const ADMIN = "mesh-admin"; +const PASSWORD = "the-vault-minted-this"; + +interface Account { admin: boolean; password: string; active: boolean } +interface Token { id: number; name: string; sha1: string; user: string; scopes: string[] } +interface Team { id: number; name: string; units: any; members: Set } + +function fakeForge() { + const users = new Map([[ADMIN, { admin: true, password: PASSWORD, active: true }]]); + const tokens: Token[] = []; + const teams: Team[] = []; + const calls: string[] = []; + // What the forge does wrong when a test says so: a token mint or delete it fails, a team another process made + // between this one's look and its create (gitea answers the create 422). + const faults = { mint: false, deleteToken: false, teamRace: false }; + let next = 0; + const json = (res: ServerResponse, status: number, body: unknown): void => { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(body === null ? "" : JSON.stringify(body)); + }; + const read = (req: IncomingMessage): Promise => + new Promise((resolve) => { + let t = ""; + req.on("data", (c) => (t += c)); + req.on("end", () => resolve(t ? JSON.parse(t) : null)); + }); + const who = (req: IncomingMessage): string | null => { + const h = req.headers.authorization ?? ""; + if (h.startsWith("Basic ")) { + const [u, p] = Buffer.from(h.slice(6), "base64").toString().split(":"); + return users.get(u)?.password === p ? u : null; + } + if (h.startsWith("token ")) return tokens.find((t) => t.sha1 === h.slice(6))?.user ?? null; + return null; + }; + const isAdmin = (u: string | null) => u !== null && users.get(u)?.admin === true; + + const server = createServer(async (req, res) => { + const url = new URL(req.url ?? "/", "http://fake"); + const p = url.pathname.replace(/^\/api\/v1/, ""); + const me = who(req); + calls.push(`${req.method} ${p} as ${me}`); + if (me === null) return json(res, 401, { message: "unauthorized" }); + + let m = p.match(/^\/users\/([^/]+)\/tokens(?:\/(\d+))?$/); + if (m) { + const owner = decodeURIComponent(m[1]); + if (!(req.headers.authorization ?? "").startsWith("Basic ")) return json(res, 401, { message: "basic auth required" }); + if (owner !== me && !isAdmin(me)) return json(res, 403, { message: "reqSelfOrAdmin" }); + if (!users.has(owner)) return json(res, 404, { message: "user does not exist" }); + if (req.method === "POST") { + if (faults.mint) return json(res, 500, { message: "the forge is not well" }); + const { name, scopes } = await read(req); + if (tokens.some((t) => t.user === owner && t.name === name)) return json(res, 400, { message: "token name has already been used" }); + const t = { id: ++next, name, sha1: `sha-${owner}-${next}`, user: owner, scopes }; + tokens.push(t); + return json(res, 201, { id: t.id, name, sha1: t.sha1, scopes }); + } + if (req.method === "GET") { + const page = Number(url.searchParams.get("page") ?? "1"); + const mine = tokens.filter((t) => t.user === owner); + return json(res, 200, page === 1 ? mine.map((t) => ({ id: t.id, name: t.name, sha1: "" })) : []); + } + if (req.method === "DELETE" && m[2]) { + if (faults.deleteToken && owner === ADMIN) return json(res, 500, { message: "the forge is not well" }); + const i = tokens.findIndex((t) => t.user === owner && t.id === Number(m![2])); + if (i < 0) return json(res, 404, { message: "not found" }); + tokens.splice(i, 1); + return json(res, 204, null); + } + } + if (req.method === "GET" && p === "/user") return json(res, 200, { login: me, is_admin: isAdmin(me) }); + m = p.match(/^\/users\/([^/]+)$/); + if (m && req.method === "GET") { + const u = users.get(decodeURIComponent(m[1])); + if (!u) return json(res, 404, { message: "user does not exist" }); + return json(res, 200, { login: m[1], is_admin: u.admin, active: u.active, prohibit_login: false }); + } + if (p.startsWith("/admin/") && !isAdmin(me)) return json(res, 403, { message: "must be a site administrator" }); + if (req.method === "POST" && p === "/admin/users") { + const b = await read(req); + if (users.has(b.username)) return json(res, 422, { message: "user already exists" }); + if (!b.password || b.password.length < 8) return json(res, 422, { message: "password is too short" }); + users.set(b.username, { admin: b.admin === true, password: b.password, active: true }); + return json(res, 201, { login: b.username, is_admin: b.admin === true }); + } + m = p.match(/^\/admin\/users\/([^/]+)$/); + if (m && req.method === "PATCH") { + const u = users.get(decodeURIComponent(m[1])); + if (!u) return json(res, 404, { message: "user does not exist" }); + const b = await read(req); + if (b.admin !== undefined) u.admin = b.admin; + if (b.active !== undefined) u.active = b.active; + return json(res, 200, { login: m[1], is_admin: u.admin }); + } + if (p === "/orgs/novox/teams") { + if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" }); + if (req.method === "GET") return json(res, 200, teams.map((t) => ({ id: t.id, name: t.name }))); + const b = await read(req); + const t = { id: ++next, name: b.name, units: b, members: new Set() }; + teams.push(t); + if (faults.teamRace) return json(res, 422, { message: "team already exists" }); + return json(res, 201, { id: t.id, name: t.name }); + } + m = p.match(/^\/teams\/(\d+)(?:\/members\/([^/]+))?$/); + if (m) { + if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" }); + const t = teams.find((x) => x.id === Number(m![1])); + if (!t) return json(res, 404, { message: "team does not exist" }); + if (req.method === "PATCH") { t.units = await read(req); return json(res, 200, { id: t.id }); } + if (req.method === "PUT" && m[2]) { t.members.add(decodeURIComponent(m[2])); return json(res, 204, null); } + } + // Repository administration: a site administrator, or a team member whose team administers the repository. + m = p.match(/^\/repos\/novox\/([^/]+)\/branches\/(.+)$/); + if (m && req.method === "GET") { + if (m[2] === "main") return json(res, 200, { name: "main", protected: true, effective_branch_protection_name: "main", + enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"], user_can_push: false }); + if (m[2] === "feat/unguarded") return json(res, 200, { name: m[2], protected: false }); + return json(res, 404, { message: "branch does not exist" }); + } + // gitea's reqAdmin on the whole group: reading the rules is administration too. + m = p.match(/^\/repos\/novox\/([^/]+)\/branch_protections(?:\/.*)?$/); + if (m) { + const admins = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.permission === "admin"); + if (!admins) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" }); + return json(res, 201, { rule_name: "main" }); + } + m = p.match(/^\/repos\/novox\/([^/]+)\/statuses\/([0-9a-f]+)$/); + if (m && req.method === "POST") { + const writes = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.units_map?.["repo.code"] === "write"); + if (!writes) return json(res, 403, { message: "write on code required" }); + return json(res, 201, { context: (await read(req)).context, creator: { login: me } }); + } + if (req.method === "GET" && p === "/repos/search") return json(res, 200, { data: [] }); + return json(res, 404, { message: "no such route in the fake" }); + }); + return new Promise<{ url: string; users: typeof users; tokens: typeof tokens; teams: typeof teams; calls: string[]; + faults: typeof faults; close(): Promise }>((resolve) => { + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as { port: number }; + resolve({ url: `http://127.0.0.1:${port}`, users, tokens, teams, calls, faults, + close: () => new Promise((r) => server.close(() => r())) }); + }); + }); +} + +const forge = await fakeForge(); +after(() => forge.close()); + +async function environment(): Promise { + const dir = await mkdtemp(join(tmpdir(), "gitea-tools-account-")); + const passwordFile = join(dir, "admin.secret"); + await writeFile(passwordFile, PASSWORD + "\n", { mode: 0o600 }); + return { + MESH_GITEA_URL: forge.url, + MESH_GITEA_ADMIN_USER: ADMIN, + MESH_GITEA_ADMIN_PASSWORD_FILE: passwordFile, + MESH_GITEA_STATE_DIR: join(dir, "state"), + }; +} + +/** A source as a fresh process builds it from the manifest's environment (fromEnv shares one per file). */ +function fresh(env: NodeJS.ProcessEnv): GiteaClient { + return GiteaClient.fromEnv(env); +} + +test("the tools' token is minted for an account of their own, which is no administrator and writes through its team", async () => { + const env = await environment(); + await fresh(env).listRepos(); + + const account = forge.users.get(TOOLS_ACCOUNT); + assert.ok(account, "the module made the tools' account"); + assert.equal(account!.admin, false, "the tools' account is no administrator"); + const held = forge.tokens.filter((t) => t.name === "mesh-tools"); + assert.deepEqual(held.map((t) => t.user), [TOOLS_ACCOUNT], "the one token is the tools' account's, none is the administrator's"); + assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8"), held[0].sha1 + "\n"); + + const team = forge.teams.find((t) => t.name === "mesh-tools"); + assert.ok(team, "the team is made in the mesh's organisation"); + assert.ok(team!.members.has(TOOLS_ACCOUNT)); + assert.equal(team!.units.permission, "write"); + assert.equal(team!.units.includes_all_repositories, true); + assert.equal(team!.units.can_create_org_repo, false, "a repository's maker would administer it"); + assert.deepEqual(team!.units.units_map, TOOLS_TEAM_UNITS.units_map); + assert.ok(!Object.values(team!.units.units_map).includes("admin")); +}); + +test("with the tools' token, the forge writes a status and refuses a protection edit — the forge itself, not a guard", async () => { + const env = await environment(); + const client = fresh(env); + await client.setCommitStatus("novox", "hq", "abc123def4567890abc123def4567890abc12345", { state: "success", context: "mesh/merge-gate", description: "ok" } as any); + await assert.rejects( + client.setBranchProtection("novox", "hq", "main", { statusChecks: ["mesh/merge-gate"] }), + /403/, + ); +}); + +test("what a base's protection asks is read from the branch, which the tools' account may read, not from the rules' list it may not", async () => { + const env = await environment(); + const client = fresh(env); + await assert.rejects(client.branchProtections("novox", "hq"), /403/); + const rule = await client.appliedProtection("novox", "hq", "main"); + assert.deepEqual(rule?.status_check_contexts, ["mesh/merge-gate", "mesh/repo-check"]); + assert.equal(rule?.enable_status_check, true); + assert.equal(rule?.rule_name, "main"); + assert.equal(await client.appliedProtection("novox", "hq", "feat/unguarded"), null); + assert.equal(await client.appliedProtection("novox", "hq", "no/such-branch"), null); +}); + +test("the administrator's token from before goes: a kept admin token is minted again for the tools' account, and the forge drops the old one", async () => { + const env = await environment(); + // What the module kept before ADR 0365: a token of the administrator's own, by the same name, its scopes beside it. + const old = { id: 900, name: "mesh-tools", sha1: "sha-admin-old", user: ADMIN, scopes: ["write:repository", "write:issue", "read:user"] }; + forge.tokens.push(old); + await mkdir(env.MESH_GITEA_STATE_DIR, { recursive: true }); + await writeFile(join(env.MESH_GITEA_STATE_DIR, "token"), old.sha1 + "\n", { mode: 0o600 }); + await writeFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "read:user,write:issue,write:repository\n", { mode: 0o600 }); + + await fresh(env).listRepos(); + + assert.ok(!forge.tokens.some((t) => t.user === ADMIN), "the administrator holds no tools' token any more"); + const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim(); + assert.notEqual(kept, old.sha1); + assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT); + assert.match(await readFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "utf8"), / for mesh-tools\n$/); +}); + +test("a tools' account someone made an administrator is made a plain account again at the next mint", async () => { + const env = await environment(); + forge.users.get(TOOLS_ACCOUNT)!.admin = true; + await fresh(env).listRepos(); + assert.equal(forge.users.get(TOOLS_ACCOUNT)!.admin, false); +}); + +test("the tools never act as the administrator: MESH_GITEA_TOOLS_USER naming it is refused", async () => { + const env = await environment(); + assert.throws(() => MintedToken.fromEnv(forge.url, { ...env, MESH_GITEA_TOOLS_USER: ADMIN }), /names the admin account/); +}); + +test("the manifest hands the tools their account, and the merge-ready authors name it rather than the administrator", async () => { + const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8")); + const code = manifest.build.artifacts.find((a: any) => a.name === "code"); + assert.equal(code.env.MESH_GITEA_TOOLS_USER, TOOLS_ACCOUNT); + const authors = String(manifest.settings["ready-authors"].default).split(/\s+/); + assert.ok(authors.includes(TOOLS_ACCOUNT)); + // Kept for now (review of issue 469): a pull request the admin account marked ready before the tools acted as + // their own keeps counting as ready after the delivery. + assert.ok(authors.includes(ADMIN), "marks made by the admin account before ADR 0365 keep counting"); + // The Go forge bundle reads the token the tools' account holds, never the administrator's password. + const forgeBundle = manifest.build.artifacts.find((a: any) => a.name === "forge"); + assert.ok(forgeBundle.env.MESH_GITEA_TOKEN_FILE); + assert.equal(forgeBundle.env.MESH_GITEA_ADMIN_PASSWORD_FILE, undefined); +}); + +// Review of issue 469: what reads the rules' list is moved to the branch, and nothing leaves the tools without a token. + +test("a repository check said as a warning on a head without merge-check.sh reads the base's protection from the branch", async () => { + const client = fresh(await environment()); + const forgeOf = { holdsFile: async () => false, appliedProtection: client.appliedProtection.bind(client) }; + const c = { owner: "novox", repo: "hq", commit: "abc123def4567890abc123def4567890abc12345", "repo-check": { verdict: "warning" } } as any; + // main requires mesh/repo-check: required, so a head that defines no check fails it for a person to see. + assert.deepEqual(await repoCheckFacts(forgeOf, c, "main"), { defined: false, required: true }); + // A base nothing guards: not required, so the head's repository check passes rather than fails. + assert.deepEqual(await repoCheckFacts(forgeOf, c, "feat/unguarded"), { defined: false, required: false }); +}); + +test("a mint that fails leaves the administrator's old token in place: the tools are never left without one", async () => { + const env = await environment(); + const old = { id: 901, name: "mesh-tools", sha1: "sha-admin-kept", user: ADMIN, scopes: ["read:user"] }; + forge.tokens.push(old); + forge.faults.mint = true; + try { + await assert.rejects(fresh(env).listRepos(), /500/); + } finally { + forge.faults.mint = false; + } + assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's token was deleted before a new one was kept"); + forge.tokens.splice(forge.tokens.indexOf(old), 1); +}); + +test("an old token that cannot be deleted after the mint is said, the new token kept, and deleted at the next start", async () => { + const env = await environment(); + const old = { id: 902, name: "mesh-tools", sha1: "sha-admin-stuck", user: ADMIN, scopes: ["read:user"] }; + forge.tokens.push(old); + forge.faults.deleteToken = true; + try { + await fresh(env).listRepos(); + } finally { + forge.faults.deleteToken = false; + } + const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim(); + assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT, "the account's token is kept and works"); + assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1)); + // The next process reuses the kept token and retires the old one it could not before. + const next = new MintedToken({ url: forge.url, admin: ADMIN, passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!, + file: join(env.MESH_GITEA_STATE_DIR, "token"), account: TOOLS_ACCOUNT, log: () => {} }); + assert.equal(await next.current(), kept); + assert.ok(!forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's old token was deleted at the next start"); +}); + +test("a team another process made between the look and the create is found, not a failure", async () => { + forge.teams.splice(0); + forge.faults.teamRace = true; + try { + await fresh(await environment()).listRepos(); + } finally { + forge.faults.teamRace = false; + } + const team = forge.teams.find((t) => t.name === "mesh-tools"); + assert.ok(team?.members.has(TOOLS_ACCOUNT)); +}); diff --git a/modules/gitea/token.ts b/modules/gitea/token.ts index d34c303e..8d7f7afd 100644 --- a/modules/gitea/token.ts +++ b/modules/gitea/token.ts @@ -22,15 +22,21 @@ // - replaced by id: an old token by this name is found in the account's list and deleted by its id, // every one of them, so a name held twice is cleared rather than refused. // +// - minted for the tools' account, not the administrator (novox/hq ADR 0365 §2, issue 469): the administrator's +// login mints it for `mesh-tools` (tools-account.ts), after making that account and its team, and deletes any +// token by this name the administrator itself still holds — the token the tools used before. A token kept +// for another account than the one asked (the scopes line says whose) is minted again at the next start. +// // An explicitly configured token still wins, and is never minted over: if it is rejected, that is // reported, not repaired — somebody chose it. // // The token is never logged. Lines say that one was minted, reused or renewed, and where it is // kept; never what it is. -import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs"; +import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs"; import { randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; +import { setUpToolsAccount, TOOLS_ACCOUNT, type AdminCall } from "./tools-account.js"; /** The name the token carries in the forge's own list — one per mesh runtime, found by name. */ export const TOKEN_NAME = "mesh-tools"; @@ -104,6 +110,14 @@ export interface MintedTokenOptions { readonly file: string; readonly name?: string; readonly scopes?: readonly string[]; + /** + * The account the token is minted for: the tools' own (novox/hq ADR 0365 §2). The admin login mints it, after + * `setUp` made the account, and the admin's own tokens by this name are deleted. Left out, the token is the + * admin's own, as it was before. + */ + readonly account?: string; + /** Run with the admin's password before each mint for `account`: makes the account and its team (tools-account.ts). */ + readonly setUp?: (password: string) => Promise; readonly log?: (line: string) => void; readonly fetch?: typeof fetch; } @@ -115,12 +129,14 @@ export class MintedToken implements TokenSource { private inflight: Promise | null = null; private readonly name: string; private readonly scopes: readonly string[]; + private readonly account: string; private readonly log: (line: string) => void; private readonly fetchImpl: typeof fetch; constructor(private readonly opts: MintedTokenOptions) { this.name = opts.name ?? TOKEN_NAME; this.scopes = opts.scopes ?? TOKEN_SCOPES; + this.account = opts.account ?? opts.admin; this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`)); this.fetchImpl = opts.fetch ?? fetch; } @@ -157,7 +173,17 @@ export class MintedToken implements TokenSource { if (missing.length) { throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`); } - return { url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token") }; + // The tools' own account, never the administrator (novox/hq ADR 0365 §2): MESH_GITEA_TOOLS_USER names it. + const account = env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT; + if (account === admin) throw new Error(`MESH_GITEA_TOOLS_USER names the admin account "${admin}"; the tools act as an account of their own (novox/hq ADR 0365)`); + return { + url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token"), account, + // GiteaAdmin is loaded when first used: client.ts imports this file. + setUp: async (password) => { + const { GiteaAdmin } = await import("./client.js"); + await setUpToolsAccount(new GiteaAdmin(url, admin!, password), account, () => randomBytes(24).toString("base64url")); + }, + }; } async current(): Promise { @@ -168,6 +194,8 @@ export class MintedToken implements TokenSource { if (kept !== null && this.keptScopes() === this.scopesLine()) { this.held = kept; this.log(`reusing the token kept at ${this.opts.file}`); + // A retirement of the admin's old token that failed after the mint is tried again, once a process. + if (existsSync(this.retireFile)) await this.locked(() => this.retireAdmin()).catch((err) => this.log(`${err}`)); return kept; } if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept); @@ -293,9 +321,10 @@ export class MintedToken implements TokenSource { } } - /** The scopes as kept beside the token: sorted, one line. */ + /** The scopes as kept beside the token: sorted, one line, and whose it is when it is not the admin's. */ private scopesLine(): string { - return [...this.scopes].sort().join(","); + const scopes = [...this.scopes].sort().join(","); + return this.account === this.opts.admin ? scopes : `${scopes} for ${this.account}`; } /** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */ @@ -318,17 +347,20 @@ export class MintedToken implements TokenSource { renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`); } - private async doMint(): Promise { - let password: string; + /** One call to the forge's API with the admin login, the password read now. */ + private password(): string { try { - password = readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, ""); + return readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, ""); } catch (err) { throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`); } + } + + private adminCall(password = this.password()): AdminCall { const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64"); - const tokens = `${this.opts.url.replace(/\/+$/, "")}/api/v1/users/${encodeURIComponent(this.opts.admin)}/tokens`; - const call = async (method: string, path = "", body?: unknown): Promise<{ status: number; body: any }> => { - const res = await this.fetchImpl(tokens + path, { + const api = `${this.opts.url.replace(/\/+$/, "")}/api/v1`; + const admin: AdminCall = async (method, path, body) => { + const res = await this.fetchImpl(api + path, { method, headers: { "Content-Type": "application/json", Authorization: authorization }, ...(body === undefined ? {} : { body: JSON.stringify(body) }), @@ -340,6 +372,44 @@ export class MintedToken implements TokenSource { } return { status: res.status, body: parsed }; }; + return admin; + } + + /** Where a retirement of the admin's old token that failed is remembered, to be tried again (retireAdmin). */ + private get retireFile(): string { + return `${this.opts.file}.retire-admin`; + } + + /** + * The admin's own token by this name goes: what the tools used before must not outlive the move (novox/hq ADR + * 0365 §2). Run after the account's token is minted and kept, so a failure here never leaves the tools without + * a token: it is said, remembered beside the kept token, and tried again when a process next reads that token. + */ + private async retireAdmin(admin: AdminCall = this.adminCall()): Promise { + if (this.account === this.opts.admin) return; + try { + const retired = await this.dropNamed(admin, `/users/${encodeURIComponent(this.opts.admin)}/tokens`, this.opts.admin); + if (retired > 0) this.log(`deleted ${retired} token(s) named "${this.name}" the admin account "${this.opts.admin}" held: the tools act as "${this.account}" now`); + rmSync(this.retireFile, { force: true }); + } catch (err) { + writeFileSync(this.retireFile, `${new Date().toISOString()}\n`, { mode: 0o600 }); + this.log(`FAILED to delete the token(s) named "${this.name}" the admin account "${this.opts.admin}" holds — tried again at the next start: ${(err as Error).message}`); + } + } + + private async doMint(): Promise { + const password = this.password(); + const admin = this.adminCall(password); + const tokensOf = (account: string) => `/users/${encodeURIComponent(account)}/tokens`; + const call = (method: string, path = "", body?: unknown) => admin(method, tokensOf(this.account) + path, body); + + if (this.account !== this.opts.admin) { + // The tools' account and its team first (novox/hq ADR 0365 §2): a failure here fails the mint, tried again + // at the next call. + const me = await admin("GET", "/user"); + if (me.status === 401 || me.status === 403) throw new AdminRefused(this.opts.admin, me.status); + await this.opts.setUp?.(password); + } let res = await call("POST", "", { name: this.name, scopes: this.scopes }); if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status); @@ -348,37 +418,51 @@ export class MintedToken implements TokenSource { // went while the forge's data stayed, or two processes once minted at once and left two. It is // ours to replace: find every one by this name in the account's list and delete each by its id // (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint. - const held: number[] = []; - for (let page = 1; page <= 100; page++) { - const listed = await call("GET", `?page=${page}&limit=50`); - if (listed.status !== 200 || !Array.isArray(listed.body)) { - throw new Error(`Gitea GET /users/${this.opts.admin}/tokens: ${listed.status} ${detail(listed.body)}`); - } - if (listed.body.length === 0) break; - for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id); - } - this.log(held.length === 1 - ? `the forge already holds a token named "${this.name}" — replacing it` - : `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`); - for (const id of held) { - const dropped = await call("DELETE", `/${id}`); - if (dropped.status !== 204 && dropped.status !== 404) { - throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`); - } - } + await this.dropNamed(admin, tokensOf(this.account), this.account, true); res = await call("POST", "", { name: this.name, scopes: this.scopes }); } if (res.status !== 201 && res.status !== 200) { - throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} ${detail(res.body)}`); + throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} ${detail(res.body)}`); } const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null; - if (!token) throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} but no token in the reply`); + if (!token) throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} but no token in the reply`); this.keep(token); this.held = token; - this.log(`minted a token for "${this.opts.admin}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`); + this.log(`minted a token for "${this.account}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`); + // Only now, with the account's own token kept, does the admin's old one go. + await this.retireAdmin(admin); return token; } + + /** + * Delete every token by this name an account holds, each by its id (gitea refuses a delete by a name held twice: + * "multiple matches for token name"); answers how many. `replacing` says the line for a mint that found its name + * taken. + */ + private async dropNamed(admin: AdminCall, tokens: string, account: string, replacing = false): Promise { + const held: number[] = []; + for (let page = 1; page <= 100; page++) { + const listed = await admin("GET", `${tokens}?page=${page}&limit=50`); + if (listed.status !== 200 || !Array.isArray(listed.body)) { + throw new Error(`Gitea GET /users/${account}/tokens: ${listed.status} ${detail(listed.body)}`); + } + if (listed.body.length === 0) break; + for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id); + } + if (replacing) { + this.log(held.length === 1 + ? `the forge already holds a token named "${this.name}" — replacing it` + : `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`); + } + for (const id of held) { + const dropped = await admin("DELETE", `${tokens}/${id}`); + if (dropped.status !== 204 && dropped.status !== 404) { + throw new Error(`Gitea DELETE /users/${account}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`); + } + } + return held.length; + } } function detail(body: unknown): string { diff --git a/modules/gitea/tools-account.ts b/modules/gitea/tools-account.ts new file mode 100644 index 00000000..03df8e32 --- /dev/null +++ b/modules/gitea/tools-account.ts @@ -0,0 +1,52 @@ +// The forge's tools act as an account of their own, not as the site administrator (novox/hq ADR 0365 §2, +// issue 469). +// +// The administrator's token reached everything the forge holds, so the tools' boundary rested on their own guards +// (guard.ts): a protection edit, say, the forge itself would have taken. Now the tools' token is minted for a plain +// account, `mesh-tools`, which is no administrator and writes to the mesh's repositories through one team of the +// mesh's organisation: +// +// - **the account**: made by the module with the administrator's login (that login sets the forge up and does +// nothing else), never an administrator — one found made an administrator is made a plain account again. Its +// password is random, sent once and kept nowhere: the account is reached only by the tokens the administrator +// mints for it (gitea lets an administrator mint for another account: `reqSelfOrAdmin`). +// - **the team**: `mesh-tools` in the organisation `novox`, on every repository of it, writing code (pushes to a +// branch, commit statuses, merges), issues (notes, comments, labels) and pull requests. No administration of a +// repository, so the forge refuses the account a protection edit, a repository's settings and its deletion; +// no repository made in the organisation, since its maker would be that repository's administrator. +// +// Every step is idempotent (GiteaAdmin's, client.ts), and run with the administrator's login before each mint of the tools' token, so a team +// or a membership lost behind the module's back is made again at the next mint. + +/** The account the tools act as. */ +export const TOOLS_ACCOUNT = "mesh-tools"; +/** The organisation whose repositories are the mesh's, and the team through which the tools' account writes there. */ +export const TOOLS_ORG = "novox"; +export const TOOLS_TEAM = "mesh-tools"; + +/** What the team grants: write on code, issues and pull requests of every repository of the organisation, and no more. */ +export const TOOLS_TEAM_UNITS = { + permission: "write", + units_map: { "repo.code": "write", "repo.issues": "write", "repo.pulls": "write" }, + includes_all_repositories: true, + can_create_org_repo: false, +} as const; + +/** One call to the forge's API with the administrator's login: a path under /api/v1, the status and body answered. */ +export type AdminCall = (method: string, path: string, body?: unknown) => Promise<{ status: number; body: any }>; + +/** What setting the account up needs of the forge's admin API: GiteaAdmin's own (client.ts), shared with the + * package provisioner, so a create race (422) and a user made by hand are handled once. */ +export interface ToolsForge { + ensureUser(username: string, password: string, email: string, plain?: boolean): Promise; + ensureTeamWith(org: string, team: string, units: Record): Promise; + addUserToTeam(teamId: number, username: string): Promise; +} + +/** Make the tools' account, plain, a member of its team on every repository of the organisation. Its password is + * set afresh each time and kept nowhere: the account is reached only by the tokens the administrator mints. */ +export async function setUpToolsAccount(forge: ToolsForge, account = TOOLS_ACCOUNT, password: () => string): Promise { + await forge.ensureUser(account, password(), `${account}@noreply.localhost`, true); + const team = await forge.ensureTeamWith(TOOLS_ORG, TOOLS_TEAM, { name: TOOLS_TEAM, ...TOOLS_TEAM_UNITS }); + await forge.addUserToTeam(team, account); +} diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index b8c46151..70d4d445 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -504,7 +504,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { base = base || "main"; let rule: BranchProtection | null = null, unreadable: string | undefined; try { - rule = await gitea.branchProtection(owner, repo, base); + rule = await gitea.appliedProtection(owner, repo, base); } catch (err) { unreadable = err instanceof Error ? err.message : String(err); } @@ -589,8 +589,12 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { const { owner, repo, branch, newBranch } = judged.contents; const found = await gitea.api<{ default_branch?: string }>(`/repos/${checkName("owner", owner)}/${checkName("name", repo)}`); const defaultBranch = String(found?.default_branch ?? "main"); - const refused = judgeContentsWrite(newBranch ?? branch ?? defaultBranch, defaultBranch, - await gitea.branchProtections(owner, repo)); + // The protection the forge applies to the branch written, read from the branch: the rules' own list is a + // repository administrator's, which the tools' account is not (novox/hq ADR 0365). A branch not made yet is + // left to the forge, which refuses a write that would make a branch a rule guards. + const target = newBranch ?? branch ?? defaultBranch; + const applied = target === defaultBranch ? null : await gitea.appliedProtection(owner, repo, target); + const refused = judgeContentsWrite(target, defaultBranch, applied ? [applied] : []); if (refused) throw new Refused(refused); } // The path judged is the path sent: normalised, with only the allow-listed query keys (guard.ts). From 373f339750779fc6224a3401b3746a916216ee92 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 15 Oct 2026 00:25:58 +0200 Subject: [PATCH 10/11] Give a walk the word of a stopped delivery that waits for it (hq issue 753) A delivery its group stopped for a member delivered since waits for a later walk of its repository that contains its merge (issue 747). When that walk carried no living delivery, nothing gave it its word and it waited forever. The delivery now takes its turn on the walk that answers its merge, in its group's order. A walk owed such a word stays under the orphans' timer, so past the grace without it `stalled` lists it with why (held, or the controller's refusal), and a walk no living delivery holds names the ended deliveries it carries. --- .../cmd/mesh-delivery/answers.go | 202 +++++++++++- .../cmd/mesh-delivery/fakes_test.go | 5 + .../mesh-delivery/cmd/mesh-delivery/holder.go | 4 + .../cmd/mesh-delivery/issue753_test.go | 292 ++++++++++++++++++ 4 files changed, 501 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go diff --git a/modules/mesh-delivery/cmd/mesh-delivery/answers.go b/modules/mesh-delivery/cmd/mesh-delivery/answers.go index e3c65cb3..d62685cd 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/answers.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/answers.go @@ -144,7 +144,9 @@ func (h *Holder) accounted(w Walk) bool { return false } -// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. +// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose +// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so +// past the grace without its word it is listed by `stalled`, saying why it was not given. func (h *Holder) heldWalk(w Walk) bool { for _, d := range h.deliveries { if d.State.Final() { @@ -292,12 +294,25 @@ func (h *Holder) orphanLines(now time.Time) []StalledLine { continue } w := s.walk + if lapsed, behind := h.lapsedOf(w); len(lapsed) > 0 || len(behind) > 0 { + // Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why. + says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+ + "merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8)) + if why := h.lapsedSaid(w); why != "" { + says += ": " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID, + For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), + H2: "none: its word is given at its turn — the operator's past the bound", Says: says}) + continue + } out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID, For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's", Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+ "follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+ "starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)}) + out[len(out)-1].Says += h.endedSaid(w) if hold := h.coverHeld(id); hold != "" { out[len(out)-1].Says += "; it is held back meanwhile: " + hold } @@ -347,6 +362,7 @@ func (h *Holder) forgetWalks(now time.Time) { } case !w.Open() && now.Sub(s.at) > keptWalkFor: delete(h.walks, id) + delete(h.deliverErr, id) } } } @@ -359,6 +375,168 @@ type turnWant struct { whys []string } +// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live +// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none. +func (h *Holder) endedSaid(w Walk) string { + var ended []string + for _, d := range h.deliveries { + if !d.State.Final() { + continue + } + _, answers := walkAnswers(w, d) + if answers || d.Walk != nil && d.Walk.ID == w.ID { + ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State)) + } + } + if len(ended) == 0 { + return "" + } + sort.Strings(ended) + return "; every delivery it carries ended: " + strings.Join(ended, ", ") +} + +// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that +// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for +// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it, +// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so +// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for +// itself, or by a person, gets nothing from it. Under the lock. +// +// In its group's order (review of issue 753): a member gives its word only once every member ordered before it reached +// the machines, or is itself a stop that lapsed whose merge this same walk answers — the walk then delivers both, in +// the group's order among its tiers, as members sharing one walk do. A member behind one still undelivered gives +// nothing; lapsedBehind names that one. +func (h *Holder) lapsedOn(w Walk) []string { + out, _ := h.lapsedOf(w) + return out +} + +// lapsedOf is lapsedOn, and, by delivery, the member before it in its group that holds back the word of a stop that +// lapsed whose merge the walk answers. Under the lock. +func (h *Holder) lapsedOf(w Walk) ([]string, map[string]string) { + if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits { + return nil, nil + } + var out []string + behind := map[string]string{} + for _, d := range h.deliveries { + if !h.lapsedAnswered(w, d) { + continue + } + if before := h.lapsedBehind(w, d); before != "" { + behind[d.ID] = before + continue + } + out = append(out, d.ID) + } + sort.Strings(out) + return out, behind +} + +// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out. +func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool { + if d.MergedAs == "" || !h.stopLapsed(d) { + return false + } + if _, ok := walkAnswers(w, d); !ok { + return false + } + _, left := w.leftOut(d.Repository, d.MergedAs) + return !left +} + +// lapsedBehind is the member ordered before a delivery in its group that has not reached the machines and that this +// walk does not deliver with it, with its state; empty for none. +func (h *Holder) lapsedBehind(w Walk, d *Delivery) string { + g := h.groups[d.Group] + if d.Group == "" || g == nil { + return "" + } + members := h.membersOf(g) + byID := map[string]*Delivery{} + for _, m := range members { + byID[m.ID] = m + } + for _, m := range h.inOrder(g, members) { + if m.ID == d.ID { + return "" + } + o := byID[m.ID] + if o == nil || memberReached(o) || h.lapsedAnswered(w, o) { + continue + } + return fmt.Sprintf("%s (%s)", o.ID, o.State) + } + return "" +} + +// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet +// delivered before them, what the last tick held it for, or the controller's refusal of the word. Under the lock. +func (h *Holder) lapsedSaid(w Walk) string { + var parts []string + _, behind := h.lapsedOf(w) + ids := make([]string, 0, len(behind)) + for id := range behind { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + parts = append(parts, fmt.Sprintf("%s waits for %s, ordered before it in its group", id, behind[id])) + } + if why := h.waitsWhy[w.ID]; why != "" { + parts = append(parts, "not let go yet: "+why) + } + if err := h.deliverErr[w.ID]; err != "" { + parts = append(parts, "the controller refused its word: "+err) + } + lapsed, _ := h.lapsedOf(w) + for _, id := range lapsed { + if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) { + parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it", + id, d.LetGo.Local().Format("15:04:05"))) + } + } + return strings.Join(parts, "; ") +} + +// lapsedOwed is whether a walk's word is owed by stops that lapsed: given now, or once a member before one is delivered. +func (h *Holder) lapsedOwed(w Walk) bool { + lapsed, behind := h.lapsedOf(w) + return len(lapsed) > 0 || len(behind) > 0 +} + +// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop +// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once, +// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words). +// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock. +func (h *Holder) lapsedTurns() { + for id, s := range h.walks { + for _, did := range h.lapsedOn(s.walk) { + d := h.deliveries[did] + if strings.Contains(d.LetGoWhy, lapsedWord(id)) { + continue + } + if h.turning == nil { + h.turning = map[string]*turnWant{} + } + want := h.turning[id] + if want == nil { + want = &turnWant{} + h.turning[id] = want + } + if !containsString(want.ids, d.ID) { + want.ids = append(want.ids, d.ID) + want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+ + "was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+ + "walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository)) + } + } + } +} + +// lapsedWord is how the word a lapsed stop gives a walk names that walk. +func lapsedWord(walk string) string { return "the walk " + walk + " answers" } + // words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per // walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5). // Under the lock. @@ -409,9 +587,14 @@ func (h *Holder) words() []func() { defer h.mu.Unlock() delete(h.asking, walk) if err != nil { + if h.deliverErr == nil { + h.deliverErr = map[string]string{} + } + h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753) h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err) return } + delete(h.deliverErr, walk) for _, id := range ids { d := h.deliveries[id] if d == nil { @@ -492,6 +675,21 @@ func (h *Holder) WalksWaiting() []StalledLine { case h.waitsWhy[id] != "": out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]}) + case !live && h.lapsedOwed(w): + // Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never + // unheld, while the controller has not taken it, with what holds it. + lapsed, _ := h.lapsedOf(w) + says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " + + "delivered since (novox/hq issue 753)" + if len(lapsed) > 0 { + says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " + + "whose merge it answers (novox/hq issue 753)" + } + if why := h.lapsedSaid(w); why != "" { + says += "; " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: "0s", H2: "none: it goes at its turn", Says: says}) case !live: says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " + "`mesh-delivery.show` for the deliveries of " + w.Repository @@ -502,7 +700,7 @@ func (h *Holder) WalksWaiting() []StalledLine { "), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word" } out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited, - Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says}) + Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)}) } } sort.Slice(out, func(i, j int) bool { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go index d6572f79..ded86f62 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go @@ -106,6 +106,8 @@ type fakeController struct { moves map[string][]string plans int down bool + // refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753). + refuseDeliver bool // onCheck runs as a check is asked, before it is answered: what the forge held at that moment. onCheck func() // planned is what Plan answers instead of "builds app", when set. @@ -182,6 +184,9 @@ func (c *fakeController) Deliver(walk, why string) error { if c.down { return errors.New("down") } + if c.refuseDeliver { + return errors.New("the controller refuses the word") + } w, known := c.walks[walk] if !known || w.Delivery == nil || w.Delivery.Go != nil { return fmt.Errorf("%s waits for nobody", walk) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index 383e2d67..a256007a 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -60,6 +60,8 @@ type Holder struct { waitsWhy map[string]string // asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile. asking map[string]bool + // deliverErr is, by walk, the controller's last refusal of its word, until it is given (review of issue 753). + deliverErr map[string]string // leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299). leaving map[string][]string // leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped). @@ -1244,6 +1246,8 @@ func (h *Holder) plan() []func() { } h.turn(d, "its turn: a delivery in no group") } + // A walk that answers the merge of a delivery whose stop lapsed: that delivery's turn (novox/hq issue 753). + h.lapsedTurns() // The word, once per walk, when every live delivery it carries is at its turn. asks = append(asks, h.words()...) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go new file mode 100644 index 00000000..8bb50c99 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go @@ -0,0 +1,292 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at +// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its +// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery +// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later +// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's +// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word +// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries. +// +// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart. + +const ( + merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307" + plan753 = "plan-1792012015539062661" +) + +var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307) + +// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304 +// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting. +func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) } + +// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301. +func night753With(t *testing.T, stop Transition, more ...*Delivery) *world { + t.Helper() + return night753Over(t, stop, nil, nil, more...) +} + +// night753Over is night753With with more merges named by the walk and the controller set as given before the first +// tick. +func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world { + t.Helper() + at := at747 + store := newMemStore() + m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747, + State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at, + AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"), + Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true, + Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}} + m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747, + State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}} + if stop.By == byOwner { + m301.StoppedFor = id336 + } + m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events", + State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304, + MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"), + // Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747). + DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)", + CoverTold: true, + Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main", + TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)}, + Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop, + Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}} + m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check", + State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307, + MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")} + for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + inGroup, order := []*Delivery{m301, m336}, []string{id336} + for _, d := range more { + if d.Group == g747 { + inGroup, order = append(inGroup, d), append(order, d.ID) + } + } + order = append(order, id301) + var ids []string + for _, d := range inGroup { + ids = append(ids, d.ID) + } + g := &Group{ID: g747, Members: ids, Closed: true, Order: order, Said: "delivering", + OrderOf: commitsOf(inGroup), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute), + Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}} + ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307}, + {Repository: cat747, Commit: merge307}}}}) + if len(merges) > 0 { + r := ctlr.walk(plan753) + r.Delivery.Merges = append(merges, r.Delivery.Merges...) + ctlr.put(r) + } + if set != nil { + set(ctlr) + } + ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt + w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w +} + +// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since. +func groupStop753() Transition { + return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"} +} + +func saidOf(lines []StalledLine, walk string) (string, State) { + said, state := "", State("") + for _, l := range lines { + if l.Walk == walk { + said, state = said+l.Says, l.State + } + } + return said, state +} + +// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for +// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this +// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld. +func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) { + w := night753(t, groupStop753()) + if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 { + t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers) + } + d := w.delivery(id301) + if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") { + t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy) + } + if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld { + t.Fatalf("the walk reads unheld: %q", said) + } + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + // Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747). + w.forge.contains[merge301+" in "+merge307] = true + r := w.ctl.walk(plan753) + sent := w.now + for _, m := range []string{"build-agent", "gitea", "route-proxy"} { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = walkDone + w.h.WalkMoved(w.ctl.put(r)) + w.later(time.Minute) + w.settleAll() + if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) { + t.Fatalf("#301 reads %s %q", state, desc) + } +} + +// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among +// the walk's, never a way past a live delivery it carries. +func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) { + at := at747 + held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308, + Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour), + MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main", + Waits: true, Created: at.Add(46 * time.Minute)}} + w := night753With(t, groupStop753(), held) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers) + } + if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held { + t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said) + } +} + +// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out, +// never let go for it. +func TestAPersonsStopGivesNoWord(t *testing.T) { + w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen", + Why: "stopped by hand"}) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers) + } + if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 { + t.Fatal("the walk carrying a person's stop was neither left without it nor stopped") + } +} + +// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it +// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never +// only that nothing holds it. +func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) { + w := night753(t, groupStop753()) + // The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed + // gives it its word, and no live one holds it. + r := w.ctl.walk(plan753) + r.Delivery.Go = nil + r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}} + w.h.WalkMoved(w.ctl.put(r)) + said, state := saidOf(w.h.WalksWaiting(), plan753) + if state != Unheld || !strings.Contains(said, id307+" (delivered)") { + t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307) + } + w.settleAll() + w.later(matchGrace + time.Minute) + w.settleAll() + said, _ = saidOf(w.h.Stalled(), plan753) + if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") { + t.Fatalf("`stalled` reads %q", said) + } +} + +// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk. +func m299() *Delivery { + at := at747 + return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299, + Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute), + Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"), + Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"}}} +} + +// **A stopped member gives its word only in its group's order** (review of issue 753): #299, ordered before #301 and +// stopped for #336 too, is not delivered and this walk does not carry its merge, so #301 gives no word — it would land +// before #299 — and `stalled walks` names #299. When the walk names #299's merge too, it delivers both, in order, and +// is let go. +func TestAStoppedMemberGivesItsWordOnlyInItsGroupsOrder(t *testing.T) { + w := night753With(t, groupStop753(), m299()) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go for #301 before #299, ordered before it: %v", w.ctl.delivers) + } + said, state := saidOf(w.h.WalksWaiting(), plan753) + if state == Unheld || !strings.Contains(said, m299().ID+" (stopped)") { + t.Fatalf("`stalled walks` reads %s %q; want #299 named", state, said) + } + w = night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: "e299aaaaaaaaaaaa", Carried: merge307}}, nil, + m299()) + if !containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk carrying both members was not let go: %v", w.ctl.delivers) + } +} + +// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past +// the grace `stalled` lists the walk, with the refusal. +func TestARefusedWordIsListedByStalled(t *testing.T) { + w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true }) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go: %v", w.ctl.delivers) + } + w.later(matchGrace + time.Minute) + w.settleAll() + said, state := saidOf(w.h.Stalled(), plan753) + if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") { + t.Fatalf("`stalled` reads %s %q", state, said) + } + // Once the controller takes it, the walk is let go. + w.ctl.refuseDeliver = false + w.settleAll() + if !containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers) + } +} + +// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person, +// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the +// grace `stalled` lists the walk, saying why. +func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) { + at := at747 + m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305, + Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), + MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready, + To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}} + w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}}, + func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers) + } + w.later(matchGrace + time.Minute) + w.settleAll() + said, _ := saidOf(w.h.Stalled(), plan753) + if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") || + !strings.Contains(said, "whose delivery was stopped") { + t.Fatalf("`stalled` reads %q", said) + } +} From 2e32f47e4596bb0d4741ddfe412bfff8b09e5232 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Thu, 15 Oct 2026 00:18:39 +0000 Subject: [PATCH 11/11] Read branch protection as the forge's administrator, so builds register again The forge answers a repository's protection rules to its administrator alone, and the tools' account no longer is one (ADR 0365), so every delivery building a module failed at the controller's build-source check with a 403. gitea_branch_protection_get now reads the rules with the administrator account (ADR 0375); it only reads, and the protection set and every write stay refused to the tools' account. Issue 772. Co-Authored-By: Claude --- modules/gitea/client.ts | 9 ++++ modules/gitea/test/protection-read.test.ts | 59 ++++++++++++++++++++++ modules/gitea/tools/index.ts | 17 ++++--- 3 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 modules/gitea/test/protection-read.test.ts diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 00524c4c..0782397a 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -958,6 +958,15 @@ export class GiteaAdmin { throw new Error(`Gitea admin ${path}: ${res.status} ${detail}`); } + /** A repository's branch protection rules, read with the administrator account — the forge answers them to a + * repository's administrator alone, which the tools' account is not (novox/hq ADR 0365 §2, ADR 0375). Only a read. */ + async branchProtections(owner: string, repo: string): Promise { + const path = `${repoPath(owner, repo)}/branch_protections`; + const res = await this.request(path); + if (res.status !== 200) GiteaAdmin.fail(path, res); + return Array.isArray(res.body) ? res.body : []; + } + /** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent * create won the race) are all success. */ async ensureOrg(name: string): Promise { diff --git a/modules/gitea/test/protection-read.test.ts b/modules/gitea/test/protection-read.test.ts new file mode 100644 index 00000000..d2928ab9 --- /dev/null +++ b/modules/gitea/test/protection-read.test.ts @@ -0,0 +1,59 @@ +// The controller's build-source check asks gitea_branch_protection_get for a repository's rules, which the forge +// answers to a repository's administrator alone. The tools' account is not one (novox/hq ADR 0365 §2), so the read +// goes with the forge's administrator account, and only the read (ADR 0375, issue 772): a protection edit through the +// tools still gets the forge's refusal. + +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { createServer } from "node:http"; + +import { GiteaAdmin, GiteaClient } from "../dist/client.js"; +import { ConfiguredToken } from "../dist/token.js"; +import { getGiteaTools } from "../dist/tools/index.js"; + +const adminBasic = "Basic " + Buffer.from("mesh-admin:secret").toString("base64"); +const rules = [ + { rule_name: "main", enable_push: false, enable_status_check: true, status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true }, + { branch_name: "release", enable_push: false }, +]; +const seen: string[] = []; + +const server = createServer((req, res) => { + const path = (req.url ?? "").replace(/^\/api\/v1/, ""); + const as = req.headers.authorization === adminBasic ? "admin" : "tools"; + seen.push(`${req.method} ${path} as ${as}`); + const send = (status: number, body: unknown) => { res.writeHead(status, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); }; + if (path.includes("/branch_protections")) { + if (as !== "admin") return send(403, { message: "user should be an owner or a collaborator with admin write of a repository" }); + if (req.method === "GET") return send(200, rules); + } + send(404, { message: "not found" }); +}); +await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); +after(() => new Promise((r) => server.close(() => r()))); +const { port } = server.address() as { port: number }; +const url = `http://127.0.0.1:${port}`; +const tools = getGiteaTools(new GiteaClient(url, new ConfiguredToken("tools-token")), () => new GiteaAdmin(url, "mesh-admin", "secret")); +const tool = (name: string) => tools.find((t) => t.name === name)!; + +test("gitea_branch_protection_get answers every rule, read with the administrator account the tools' token cannot read with", async () => { + const out: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller" }); + assert.deepEqual(out.rules.map((r: any) => r.rule), ["main", "release"]); +}); + +test("gitea_branch_protection_get answers the rule for a branch, null when it has none", async () => { + const main: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "main" }); + assert.equal(main.branch, "main"); + assert.equal(main.rule.rule, "main"); + assert.equal(main.rule.admin_may_override, false); + const none: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "dev" }); + assert.equal(none.rule, null); +}); + +test("gitea_branch_protection_set still gets the forge's refusal, as the tools' account", async () => { + await assert.rejects( + tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-controller", branch: "main", status_checks: "mesh/merge-gate" }), + /403/, + ); + assert.ok(!seen.some((s) => /^(PATCH|POST|PUT|DELETE) .*branch_protections.* as admin$/.test(s)), seen.join("\n")); +}); diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index 70d4d445..27a8c610 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -10,7 +10,7 @@ import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; import { emit } from "@novox/mesh-sdk/events"; -import { GiteaClient, readyAuthors, type BranchProtection } from "../client.js"; +import { GiteaAdmin, GiteaClient, readyAuthors, type BranchProtection } from "../client.js"; import { appendNote, run, viewBody, viewComment } from "../delivery.js"; import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js"; import { @@ -59,13 +59,13 @@ function namesChecked(tools: ToolDefinition[]): ToolDefinition[] { })); } -export function getGiteaTools(gitea: GiteaClient): ToolDefinition[] { - return namesChecked(allGiteaTools(gitea)); +export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] { + return namesChecked(allGiteaTools(gitea, admin)); } // gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one // writer (novox/hq issue 456, ADR 0323). -function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { +function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] { return [ // ---- Repositories ---- { @@ -401,8 +401,11 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { }, run: async (args) => { const owner = String(args.owner), repo = String(args.repo); - if (!args.branch) return { rules: (await gitea.branchProtections(owner, repo)).map(summarised) }; - const rule = await gitea.branchProtection(owner, repo, String(args.branch)); + // The one read made as the forge's administrator (novox/hq ADR 0375, issue 772): the rules' list is the + // repository administrator's, which the tools' account is not. No other tool uses this account. + const rules = await (admin ? admin() : gitea).branchProtections(owner, repo); + if (!args.branch) return { rules: rules.map(summarised) }; + const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === String(args.branch)) ?? null; return { branch: String(args.branch), rule: rule ? summarised(rule) : null }; }, }, @@ -615,7 +618,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { // way, gitea contributes none rather than failing the whole runtime, and says why. registerModuleTools("gitea", (env) => { try { - return getGiteaTools(GiteaClient.fromEnv(env)); + return getGiteaTools(GiteaClient.fromEnv(env), () => GiteaAdmin.fromEnv(env)); } catch (err) { console.log(`[gitea] no tools — ${err instanceof Error ? err.message : String(err)}`); return [];