diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 8efe7c73..b9c80da7 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -44,7 +44,7 @@ import { GiteaAdmin, GiteaClient, movedSince } from "./client.js"; import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked, type RepoCheckFacts, reposToLook } from "./pulls.js"; import { GateWriter, clonedAs, type BuildStarted } from "./gate.js"; -import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, keptSecret, receiver, type ForgeEvent } from "./webhook.js"; +import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js"; import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js"; // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is @@ -429,10 +429,16 @@ const gates = new GateWriter({ // **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own // process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to // reach, nothing is served, as the tools' bundle serves nothing then. -registerModuleTools("gitea", () => (gitea ? [commitStatusTool({ +/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what + * gitea_events says of them (webhook.ts). */ +const pace = new Pace(); +if (!gitea) pace.unheard("the forge is not watched: no way to a token", false); + +// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729). +registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({ setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status), gates, -}), mergeTool(gitea, { gates })] : [])); +}), mergeTool(gitea, { gates })] : [])]); // **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when // the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing. @@ -451,9 +457,6 @@ async function buildStarted(event: { body: unknown }): Promise { } } -/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are (webhook.ts). */ -const pace = new Pace(); - /** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */ async function actOn(client: GiteaClient, event: ForgeEvent): Promise { const b = event.body as { owner?: string; repo?: string; number?: number }; @@ -476,23 +479,32 @@ async function listenForForge(client: GiteaClient): Promise { const url = process.env.MESH_GITEA_HOOK_URL; if (!stateDir || !url) { console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge"); + pace.unheard("MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset", false); return false; } try { const secret = keptSecret(join(stateDir, "webhook.secret")); const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT); - const server = receiver({ secret, port, heard: () => pace.heard(), + const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0"; + const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery), + refused: (reason, from) => pace.refused(reason, from), publish: (event) => emit(event.name, event.body), act: (event) => actOn(client, event) }); await new Promise((resolve, reject) => { server.once("error", reject); - server.listen(port, process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0", () => resolve()); + server.listen(port, host, () => resolve()); }); - const did = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret); - console.log(`[gitea] listening for the forge's events on port ${port}; its system webhook ${did}, every event, signed`); + const hook = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret); + pace.listening(host, port, hook.url, hook); + console.log(`[gitea] listening for the forge's events on ${host}:${port}; its system webhook ${hook.id} ${hook.did} ` + + `at ${hook.url} (in place of ${hook.removed.system} system, ${hook.removed.default} default and ` + + `${hook.removed.repository} repository webhook(s)${hook.removed.repositories.length ? ` in ${hook.removed.repositories.join(", ")}` : ""}), ` + + `${hook.events.length} event families, signed`); return true; } catch (err) { - console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${err instanceof Error ? err.message : err}`); + const why = err instanceof Error ? err.message : String(err); + pace.unheard(why); + console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${why}`); return false; } } diff --git a/modules/gitea/test/webhook.test.ts b/modules/gitea/test/webhook.test.ts index 453ff2d0..1160de34 100644 --- a/modules/gitea/test/webhook.test.ts +++ b/modules/gitea/test/webhook.test.ts @@ -83,20 +83,235 @@ test("each forge event is named in the module's stream and trimmed", async () => assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other"); }); -test("the system webhook is made when none points at the receiver, and corrected when one does", async () => { - const { ensureSystemHook } = await import("../webhook.ts"); +// A forge's admin hooks as 1.27.3 keeps them: POST /admin/hooks makes a system webhook only when config.is_system_webhook +// is "true", and a default webhook otherwise; the listing names one kind at a time (type=system, the default, or default). +// Repositories hold their own webhooks (a default webhook's copies), listed a page at a time. +function fakeForge(start: { id: number; url: string; system: boolean }[] = [], + repoHooks: Record = {}, repoCount = 0) { + const hooks = start.map((h) => ({ ...h, events: ["push"], active: true })); + const repos = [...Object.keys(repoHooks)]; + for (let i = 0; repos.length < repoCount; i++) repos.push(`novox/r${i}`); + let next = 100; const calls: string[] = []; - let hooks: unknown[] = []; + const made: any[] = []; + const page = (items: any[], q: URLSearchParams) => { + const p = Number(q.get("page") ?? 1), l = Number(q.get("limit") ?? 50); + return items.slice((p - 1) * l, p * l); + }; const admin = { call: async (method: string, path: string, body?: any) => { calls.push(`${method} ${path}`); - if (method === "GET") return { status: 200, body: hooks }; - if (method === "POST") { assert.ok(body.events.includes("pull_request") && body.events.includes("push")); assert.equal(body.config.secret, secret); return { status: 201, body: {} }; } - return { status: 200, body: {} }; + const u = new URL(path, "http://f"); + const repoHook = u.pathname.match(/^\/repos\/([^/]+\/[^/]+)\/hooks(?:\/(\d+))?$/); + if (method === "GET" && u.pathname === "/repos/search") { + return { status: 200, body: { ok: true, data: page(repos.map((full_name) => ({ full_name })), u.searchParams) } }; + } + if (repoHook) { + const list = repoHooks[repoHook[1]] ?? []; + if (method === "GET") return { status: 200, body: page(list.map((h) => ({ id: h.id, config: { url: h.url } })), u.searchParams) }; + const i = list.findIndex((h) => h.id === Number(repoHook[2])); + if (i < 0) return { status: 404, body: {} }; + list.splice(i, 1); + return { status: 204, body: null }; + } + if (method === "GET") { + const type = u.searchParams.get("type") ?? "system"; + const listed = hooks.filter((h) => type === "all" || h.system === (type === "system")); + return { status: 200, body: listed.map((h) => ({ id: h.id, active: h.active, events: h.events, + config: { url: h.url, content_type: "json" } })) }; + } + if (method === "POST") { + made.push(body); + hooks.push({ id: next++, url: body.config.url, system: body.config.is_system_webhook === "true", events: body.events, + active: body.active }); + return { status: 201, body: {} }; + } + if (method === "DELETE") { + const id = Number(path.split("/").pop()); + const i = hooks.findIndex((h) => h.id === id); + if (i < 0) return { status: 404, body: {} }; + hooks.splice(i, 1); + return { status: 204, body: null }; + } + return { status: 405, body: {} }; } }; - assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "created"); - hooks = [{ id: 4, config: { url: "http://h:3047/forge-event" } }]; - assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "corrected"); - assert.ok(calls.includes("PATCH /admin/hooks/4")); + return { admin, hooks, calls, made, repoHooks }; +} + +test("the webhook is made a system webhook, which the forge sends every repository's events to", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const forge = fakeForge(); + const url = "http://h:3047/forge-event"; + const hook = await ensureSystemHook(forge.admin, url, secret); + assert.equal(forge.made.length, 1); + assert.equal(forge.made[0].config.is_system_webhook, "true", "without it the forge makes a default webhook, which fires for no repository"); + assert.equal(forge.made[0].config.secret, secret); + assert.ok(forge.made[0].events.includes("pull_request") && forge.made[0].events.includes("push")); + assert.deepEqual(forge.hooks.map((h) => h.system), [true]); + assert.equal(hook.kind, "system"); + assert.equal(hook.did, "created"); + assert.equal(hook.url, url); + assert.ok(hook.events.includes("pull_request")); +}); + +test("default webhooks left at the receiver by an earlier start are replaced by one system webhook", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const url = "http://h:3047/forge-event"; + // What the start at 23:05 left: a default webhook, invisible to the system listing, made again by every start. + const forge = fakeForge([{ id: 4, url, system: false }, { id: 5, url, system: false }, { id: 6, url: "http://other/x", system: true }]); + const hook = await ensureSystemHook(forge.admin, url, secret); + assert.deepEqual(hook.removed, { system: 0, default: 2, repository: 0, repositories: [] }); + assert.equal(hook.did, "replaced"); + assert.deepEqual(forge.hooks.map((h) => [h.url, h.system]), [["http://other/x", true], [url, true]], "another url is kept"); + // A second start replaces the system webhook, so the secret kept now is the one the forge signs with (its edit never + // changes a secret). + const again = await ensureSystemHook(forge.admin, url, secret); + assert.deepEqual(again.removed, { system: 1, default: 0, repository: 0, repositories: [] }); + assert.equal(forge.hooks.filter((h) => h.url === url).length, 1); + assert.ok(forge.calls.includes("DELETE /admin/hooks/4") && forge.calls.includes("DELETE /admin/hooks/5")); + // The new one is made before any old one is deleted, so no event falls in a gap. + const post = forge.calls.indexOf("POST /admin/hooks"); + assert.ok(post >= 0 && forge.calls.findIndex((c) => c.startsWith("DELETE")) > post); +}); + +test("a repository's copy of a default webhook at the receiver is deleted, on every page, and said", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const url = "http://h:3047/forge-event"; + // A repository made after the default webhook got its copy, which would send its events a second time; 120 + // repositories, so the copies sit beyond the first page. + const repoHooks: Record = {}; + for (let i = 0; i < 118; i++) repoHooks[`novox/r${i}`] = []; + repoHooks["novox/late"] = [{ id: 31, url }, { id: 32, url: "http://ci/hook" }]; + repoHooks["novox/later"] = [{ id: 33, url }]; + const forge = fakeForge([{ id: 4, url, system: false }], repoHooks); + const hook = await ensureSystemHook(forge.admin, url, secret); + assert.deepEqual(hook.removed, { system: 0, default: 1, repository: 2, repositories: ["novox/late", "novox/later"] }); + assert.deepEqual(forge.repoHooks["novox/late"].map((h) => h.id), [32], "a repository's other webhook is kept"); + assert.deepEqual(forge.repoHooks["novox/later"], []); + assert.ok(forge.calls.some((c) => /\/repos\/search\?.*page=3/.test(c)), "every page of repositories is read"); +}); + +test("a forge that makes no system webhook is said, never taken as heard", async () => { + const { ensureSystemHook } = await import("../webhook.ts"); + const forge = fakeForge(); + const admin = { call: async (method: string, path: string, body?: any) => { + // A forge that ignores the flag: what 23:05 met, had the module read back. + if (method === "POST") body = { ...body, config: { ...body.config, is_system_webhook: undefined } }; + return forge.admin.call(method, path, body); + } }; + await assert.rejects(ensureSystemHook(admin, "http://h:3047/forge-event", secret), /names 0 new system webhooks/); +}); + +test("the receiver tells what it verified and what it refused", async () => { + const { receiver, HOOK_PATH } = await import("../webhook.ts"); + const heard: string[] = [], refused: string[] = []; + const server = receiver({ secret, port: 0, log: () => {}, publish: async () => {}, + heard: (kind, delivery) => heard.push(`${kind} ${delivery}`), refused: (reason) => refused.push(reason) }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`; + try { + await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-9" } }); + await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "x") } }); + await server.idle(); + assert.deepEqual(heard, ["pull_request d-9"]); + assert.deepEqual(refused, ["its signature does not verify"]); + } finally { server.close(); } +}); + +test("a forge that never delivers is said loudly once, and unhealthy, when the poll finds what no event said", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const pace = new Pace(() => now, (l) => said.push(l)); + const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true, + did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } }; + pace.listening("172.17.0.1", 3047, hook.url, hook); + now += 5 * 60_000; + pace.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(said.length, 0, "five minutes after the start is not yet silence"); + assert.equal((pace.report() as any).healthy, true); + pace.refused("it is unsigned", "172.17.0.3"); + assert.equal((pace.report() as any).healthy, false, "refused, and none verified"); + now += 6 * 60_000; + pace.miss("the merge of novox/hq#7"); + pace.miss("novox/hq#8 at bbbbbbbb"); + assert.equal(said.filter((l) => /NOT HEARD: no delivery ever verified/.test(l)).length, 1, "said once"); + let r = pace.report() as any; + assert.equal(r.healthy, false); + assert.equal(r.state, "unhealthy"); + assert.match(r.why, /never delivered/); + assert.equal(r.polls.found_unsaid, 3); + assert.equal(r.refused.count, 1); + assert.equal(r.refused.last.reason, "it is unsigned"); + assert.equal(r.hook.kind, "system"); + assert.equal(r.receiver.port, 3047); + // The first delivery clears it. + now += 1000; + pace.heard("push", "d-1"); + r = pace.report() as any; + assert.equal(r.healthy, true); + assert.equal(r.verified.count, 1); + assert.equal(r.verified.last.event, "push"); + assert.equal(r.polls.every_seconds, 300); + assert.match(said.join("\n"), /heard again/); +}); + +test("a forge whose events stop after one delivery is said loudly once per silent spell, and refusals are unhealthy", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const pace = new Pace(() => now, (l) => said.push(l)); + const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true, + did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } }; + pace.listening("172.17.0.1", 3047, hook.url, hook); + now += 60_000; + pace.heard("push", "d-1"); + // The hook is deleted by hand: nothing more comes, and the poll finds what no event said. + now += 11 * 60_000; + pace.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 0, "one find alone may be the poll beating an event"); + pace.miss("the merge of novox/hq#7"); + pace.miss("novox/hq#8 at bbbbbbbb"); + assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 1, "said once"); + assert.match(said.find((l) => /NOT HEARD/.test(l)) ?? "", /since the last at/); + assert.equal((pace.report() as any).healthy, false); + // A delivery ends the spell; the next spell is said again. + now += 1000; + pace.heard("push", "d-2"); + assert.equal((pace.report() as any).healthy, true); + now += 11 * 60_000; + pace.miss("novox/hq#9 at cccccccc"); + pace.miss("novox/hq#10 at dddddddd"); + assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 2, "a second silent spell is said again"); + // Every delivery refused after a verified one (a secret that no longer matches) is unhealthy at once. + now += 1000; + pace.heard("push", "d-3"); + now += 1000; + pace.refused("its signature does not verify", "172.17.0.3"); + pace.refused("its signature does not verify", "172.17.0.3"); + const r = pace.report() as any; + assert.equal(r.healthy, false); + assert.equal(r.refused.since_verified, 2); + assert.match(r.why, /2 deliveries refused since the last verified one.*does not verify/); + now += 1000; + pace.heard("push", "d-4"); + assert.equal((pace.report() as any).healthy, true, "a verified delivery clears it"); +}); + +test("without a receiver by configuration the polls alone watch, healthy; a receiver that failed is unhealthy", async () => { + const { Pace } = await import("../webhook.ts"); + let now = 1_000_000; + const said: string[] = []; + const quiet = new Pace(() => now, (l) => said.push(l)); + quiet.unheard("MESH_GITEA_HOOK_URL unset", false); + now += 20 * 60_000; + quiet.miss("novox/hq#7 at aaaaaaaa"); + assert.equal(said.length, 0, "no deliveries were expected"); + assert.equal((quiet.report() as any).healthy, true); + const failed = new Pace(() => now, () => {}); + failed.unheard("the forge's system webhook could not be made: 403"); + const r = failed.report() as any; + assert.equal(r.healthy, false); + assert.match(r.why, /403/); }); test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => { diff --git a/modules/gitea/webhook.ts b/modules/gitea/webhook.ts index 130a4e01..7cb63671 100644 --- a/modules/gitea/webhook.ts +++ b/modules/gitea/webhook.ts @@ -140,8 +140,11 @@ export interface ReceiverOptions { publish: (event: ForgeEvent) => Promise; /** Acts on it at once, beside publishing it (the announcer). */ act?: (event: ForgeEvent) => Promise; - /** Told of every verified delivery, so the polls know events are heard. */ - heard?: () => void; + /** Told of every verified delivery — its kind (X-Gitea-Event) and its delivery id — so the polls know events are + * heard, and the module can say when it last heard one. */ + heard?: (kind: string, delivery: string) => void; + /** Told of every refused delivery, with the reason and where it came from. */ + refused?: (reason: string, from: string) => void; log?: (line: string) => void; } @@ -158,6 +161,10 @@ export const REMEMBERED = 2000; */ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise } { const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`)); + const refuse = (from: string, reason: string): void => { + log(`refused a forge event from ${from}: ${reason}`); + opts.refused?.(reason, from); + }; const seen = new Set(); let queue: Promise = Promise.resolve(); const server = createServer((req, res) => { @@ -168,14 +175,14 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise MAX_BODY) { - log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`); + refuse(from, `its body is over ${MAX_BODY} bytes`); res.writeHead(413, { Connection: "close" }); res.end("too large\n"); req.destroy(); @@ -189,7 +196,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise MAX_BODY) { if (!over) { over = true; - log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`); + refuse(from, `its body is over ${MAX_BODY} bytes`); res.writeHead(413, { Connection: "close" }); res.end("too large\n"); req.destroy(); @@ -200,7 +207,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise REMEMBERED) seen.delete(seen.values().next().value as string); } - const event = eventOf(String(req.headers["x-gitea-event"] ?? "other"), payload); + const event = eventOf(kind, payload); res.statusCode = 200; res.end("queued\n"); queue = queue.then(async () => { @@ -242,27 +251,113 @@ export interface HookAdmin { call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>; } +/** The forge's one system webhook as ensured at start: read back from the forge after it was made. */ +export interface HookState { + /** Always "system": a hook the forge's listing of system webhooks names, so it fires for every repository. */ + kind: "system"; + id: number; + url: string; + events: string[]; + active: boolean; + content_type?: string; + /** Made where none was, or made in place of the ones there (system, default, or a repository's) at the receiver's url. */ + did: "created" | "replaced"; + /** The hooks at the receiver's url it took the place of, by kind: a repository's is a copy of a default webhook, which + * would have sent that repository's events a second time. */ + removed: { system: number; default: number; repository: number; repositories: string[] }; +} + +/** How long without a verified delivery — since the start, or since the last one — a forge is said silent once the poll + * finds what no event said. */ +export const SILENT_AFTER = 10 * 60_000; + /** - * **How fast the polls look** (issue 729): every five minutes only while the forge's events are heard — a verified - * delivery within the last ten minutes, and none missed since. Until the first delivery, after ten minutes without one, - * or once the poll found something no event said, every thirty seconds. + * **How fast the polls look, and whether the forge's events are heard** (issue 729): every five minutes only while the + * forge's events are heard — a verified delivery within the last ten minutes, and none missed since. Until the first + * delivery, after ten minutes without one, or once the poll found something no event said, every thirty seconds. + * + * It also keeps what `gitea_events` says: where the receiver listens, the hook as ensured, the verified and refused + * deliveries, and the polls' finds no event said. **A forge whose events stop is said** (issue 729): once the receiver + * listens and the hook is ensured, when nothing verified for ten minutes — since the start, or since the last delivery — + * and the poll has found at least two heads or merges no event said in that quiet (one could be the poll beating an + * event by a second), it is logged loudly once per silent spell, and the module's events are said unhealthy until a + * delivery verifies. Deliveries refused since the last verified one are said unhealthy too. */ export class Pace { private last = 0; private missed = 0; + private readonly started: number; private readonly now: () => number; private readonly log: (l: string) => void; + private where: { host: string; port: number; url: string } | null = null; + private hook: HookState | null = null; + private notHeard: string | null = null; + private notHeardFails = false; + private verifiedCount = 0; + private lastVerified: { at: number; event: string; delivery: string } | null = null; + private refusedCount = 0; + private refusedSpell = 0; + private lastRefused: { at: number; reason: string; from: string } | null = null; + private missCount = 0; + private lastMiss: { at: number; what: string } | null = null; + private silentSince = 0; + private quietMisses = 0; constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) { this.now = now; this.log = log; + this.started = now(); } - heard(): void { + /** The receiver listens, and the hook at it was ensured: from now on, deliveries are expected. */ + listening(host: string, port: number, url: string, hook: HookState): void { + this.where = { host, port, url }; + this.hook = hook; + this.notHeard = null; + } + /** No events are listened for, and why: by configuration (`fails` false), or because the receiver could not listen + * or the hook could not be ensured (`fails` true, said unhealthy). The polls alone watch. */ + unheard(why: string, fails = true): void { + this.notHeard = why; + this.notHeardFails = fails; + } + heard(event = "", delivery = ""): void { this.last = this.now(); + this.verifiedCount++; + this.lastVerified = { at: this.last, event, delivery }; + this.refusedSpell = 0; + this.quietMisses = 0; + if (this.silentSince) this.log(`the forge's events are heard again: ${event || "a delivery"} verified`); + this.silentSince = 0; + } + refused(reason: string, from: string): void { + this.refusedCount++; + this.refusedSpell++; + this.lastRefused = { at: this.now(), reason, from }; } /** The poll found a head or merge no forge event announced. */ miss(what: string): void { + const t = this.now(); if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`); - this.missed = this.now(); + this.missed = t; + this.missCount++; + this.lastMiss = { at: t, what }; + // Silent: nothing verified for SILENT_AFTER — never since the start, or not since the last delivery (a hook deleted by + // hand, every delivery refused) — while the poll finds what no event said. Said once per silent spell. + const quietSince = this.last || this.started; + this.quietMisses++; + if (this.expecting() && t - quietSince >= SILENT_AFTER && this.quietMisses >= 2 && !this.silentSince) { + this.silentSince = t; + const minutes = Math.round((t - quietSince) / 60_000); + const refusedSince = this.refusedSinceHeard(); + this.log(`THE FORGE'S EVENTS ARE NOT HEARD: ${this.last ? `no delivery verified since the last at ` + + `${new Date(this.last).toISOString()}` : "no delivery ever verified since the start"}, ${minutes} min, from the ` + + `system webhook at ${this.hook?.url}${refusedSince ? `; ${refusedSince} refused since, the last because ` + + `${this.lastRefused?.reason}` : ""}; the poll found ${what} — the polls alone watch the forge, every 30 s ` + + `(novox/hq issue 729)`); + } + } + /** Whether a delivery was refused after the last verified one (or with none verified). */ + private refusedSinceHeard(): number { + return this.lastRefused && this.lastRefused.at >= this.last ? this.refusedSpell : 0; } hearing(): boolean { const t = this.now(); @@ -271,36 +366,146 @@ export class Pace { every(): number { return this.hearing() ? 5 * 60_000 : 30_000; } + private expecting(): boolean { + return this.where !== null && this.hook !== null; + } + /** What `gitea_events` answers: healthy, unhealthy or waits (ADR 0283's shape), with every fact behind it. */ + report(): Record { + const t = this.now(); + const iso = (ms: number): string => new Date(ms).toISOString(); + let state: "healthy" | "unhealthy" | "waits" = "healthy"; + let why: string; + const refusedSince = this.refusedSinceHeard(); + if (this.silentSince) { + state = "unhealthy"; + why = `${this.last ? `no delivery verified since the last at ${iso(this.last)}` : `the forge has never delivered an ` + + `event since the start at ${iso(this.started)}`}, while the poll found ${this.missCount} head(s) or merge(s) no ` + + `event said, the last ${this.lastMiss?.what} at ${iso(this.lastMiss?.at ?? t)}`; + } else if (refusedSince) { + state = "unhealthy"; + why = `${refusedSince} deliveries refused ${this.last ? `since the last verified one at ${iso(this.last)}` : + "and none verified"}, the last at ${iso(this.lastRefused?.at ?? t)} because ${this.lastRefused?.reason}`; + } else if (!this.expecting()) { + why = this.notHeard ? `no events are listened for, so the polls alone watch the forge: ${this.notHeard}` : + "the receiver is not listening yet"; + if (this.notHeard && this.notHeardFails) state = "unhealthy"; + } else if (this.hearing()) { + why = `the forge's events are heard: ${this.verifiedCount} verified, the last at ${iso(this.last)}`; + } else if (this.last === 0) { + why = `no delivery heard yet; the forge is said silent if the poll finds what no event said ${SILENT_AFTER / 60_000} min after the start`; + } else { + why = `the last delivery was at ${iso(this.last)}; the polls look every 30 s until the next`; + } + return { + healthy: state !== "unhealthy", state, why, + started: iso(this.started), + receiver: this.where ? { listening: true, ...this.where } : { listening: false, why: this.notHeard ?? "not started" }, + hook: this.hook, + verified: { count: this.verifiedCount, last: this.lastVerified && { ...this.lastVerified, at: iso(this.lastVerified.at) } }, + refused: { count: this.refusedCount, since_verified: refusedSince, last: this.lastRefused && { ...this.lastRefused, at: iso(this.lastRefused.at) } }, + polls: { every_seconds: this.every() / 1000, hearing: this.hearing(), found_unsaid: this.missCount, + last_found: this.lastMiss && { ...this.lastMiss, at: iso(this.lastMiss.at) } }, + silent: this.silentSince ? { since: iso(this.silentSince) } : null, + }; + } } /** Every event family the forge (1.27) can send a webhook. */ export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request", "pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"]; -/** - * **One system webhook, for every event, at the receiver** (issue 729): created when none points there, corrected - * when one does but differs (events, active, content type), and its secret set again each start — the forge never - * hands a secret back, so a corrected secret cannot be told from a kept one. Answers what it did. - */ -export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<"created" | "corrected"> { - // POST /admin/hooks makes a system webhook, and its options take no flag for it (the forge's 1.27.3 API schema). The - // forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it. - const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*", - config: { url, content_type: "json", secret } }; - const listed = await admin.call("GET", "/admin/hooks?type=system&limit=50"); - if (listed.status !== 200) throw new Error(`the forge's system webhooks could not be listed: ${listed.status}`); - const hooks: any[] = Array.isArray(listed.body) ? listed.body : []; - const ours = hooks.filter((h) => h?.config?.url === url); - if (ours.length === 0) { - const made = await admin.call("POST", "/admin/hooks", want); - if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`); - return "created"; +async function hooksAt(admin: HookAdmin, type: "system" | "default", url: string): Promise { + const listed = await admin.call("GET", `/admin/hooks?type=${type}&limit=50`); + if (listed.status !== 200) throw new Error(`the forge's ${type} webhooks could not be listed: ${listed.status}`); + return (Array.isArray(listed.body) ? listed.body : []).filter((h) => h?.config?.url === url); +} + +/** Every page of a listing that answers an array (or `{ data: [] }`, the repository search), PAGE at a time. */ +const PAGE = 50; +async function everyPage(admin: HookAdmin, path: string, what: string): Promise { + const all: any[] = []; + for (let page = 1; ; page++) { + const r = await admin.call("GET", `${path}${path.includes("?") ? "&" : "?"}page=${page}&limit=${PAGE}`); + if (r.status !== 200) throw new Error(`${what} could not be listed: ${r.status}`); + const items: any[] = Array.isArray(r.body) ? r.body : Array.isArray(r.body?.data) ? r.body.data : []; + all.push(...items); + if (items.length < PAGE) return all; } - for (const extra of ours.slice(1)) await admin.call("DELETE", `/admin/hooks/${extra.id}`); - const fixed = await admin.call("PATCH", `/admin/hooks/${ours[0].id}`, { active: true, events: HOOK_EVENTS, branch_filter: "*", - config: { url, content_type: "json", secret } }); - if (fixed.status !== 200) throw new Error(`the forge's system webhook could not be corrected: ${fixed.status}`); - return "corrected"; +} + +/** Every repository's own webhooks at the receiver's url: copies of a default webhook made at it. */ +async function repoHooksAt(admin: HookAdmin, url: string): Promise<{ repo: string; id: number }[]> { + const found: { repo: string; id: number }[] = []; + for (const repo of await everyPage(admin, "/repos/search?sort=id", "the forge's repositories")) { + const full = String(repo?.full_name ?? ""); + if (!full.includes("/")) continue; + const hooks = await everyPage(admin, `/repos/${full}/hooks`, `${full}'s webhooks`); + for (const h of hooks) if (h?.config?.url === url) found.push({ repo: full, id: Number(h.id) }); + } + return found; +} + +/** + * **One system webhook, for every event, at the receiver** (issue 729), made each start and read back. + * + * The forge (1.27.3, routers/api/v1/utils/hook.go addHook) makes a *default* webhook from POST /admin/hooks unless + * `config.is_system_webhook` is "true": a default webhook fires for no repository and is only copied into repositories + * made after it, and the listing of system webhooks does not name it — so without the flag every start made another + * one and no event came. Its edit (editHook) never changes a webhook's secret nor its kind, so a hook already there is + * not corrected but replaced: the new system webhook is made first, with the secret kept now, so no event falls in a + * gap; then every other hook at the receiver's url is deleted — system, default, and each repository's copy of a + * default one, which would send that repository's events twice. The forge's listing of system webhooks must then name + * exactly one at the url, or this fails, saying so. + */ +export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise { + // The forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it. + const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*", + config: { url, content_type: "json", secret, is_system_webhook: "true" } }; + const before = new Set((await hooksAt(admin, "system", url)).map((h) => Number(h.id))); + const made = await admin.call("POST", "/admin/hooks", want); + if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`); + const fresh = (await hooksAt(admin, "system", url)).filter((h) => !before.has(Number(h.id))); + if (fresh.length !== 1) { + throw new Error(`the forge names ${fresh.length} new system webhooks at ${url} after one was made, not one: its events ` + + `are not known to be sent`); + } + const ours = fresh[0]; + const defaults = await hooksAt(admin, "default", url); + const gone = async (path: string): Promise => { + const r = await admin.call("DELETE", path); + if (r.status !== 204 && r.status !== 200 && r.status !== 404) throw new Error(`the forge's webhook ${path} could not be deleted: ${r.status}`); + }; + for (const id of before) await gone(`/admin/hooks/${id}`); + for (const h of defaults) await gone(`/admin/hooks/${h.id}`); + const copies = await repoHooksAt(admin, url); + for (const c of copies) await gone(`/repos/${c.repo}/hooks/${c.id}`); + const kept = await hooksAt(admin, "system", url); + if (kept.length !== 1 || Number(kept[0].id) !== Number(ours.id)) { + throw new Error(`the forge names ${kept.length} system webhooks at ${url} once the others were deleted, not the one ` + + `made (${ours.id})`); + } + const removed = { system: before.size, default: defaults.length, repository: copies.length, + repositories: [...new Set(copies.map((c) => c.repo))] }; + return { kind: "system", id: Number(ours.id), url, events: Array.isArray(ours.events) ? ours.events : [], + active: !!ours.active, content_type: ours.config?.content_type, + did: removed.system + removed.default + removed.repository > 0 ? "replaced" : "created", removed }; +} + +/** The `gitea_events` tool: what the module knows of the forge's events, from its Pace. */ +export function eventsTool(pace: Pace): { name: string; description: string; input: Record; + run: (args: Readonly>) => Promise } { + return { + name: "gitea_events", + description: "Whether the forge's events reach the mesh (novox/hq issue 729): where the receiver listens, the " + + "system webhook as ensured at the start (its id, url, events, kind, and what it replaced), how many deliveries " + + "verified and the last one's time and event, how many were refused and the last reason, how often the polls look " + + "and whether events are heard, and how many heads or merges the poll found that no event said. Unhealthy when the " + + "forge has delivered nothing that verified for ten minutes (since the start or since the last delivery) while the " + + "poll found what no event said, when deliveries were refused since the last verified one, or when the receiver " + + "could not listen or the webhook could not be made.", + input: {}, + run: async () => pace.report(), + }; } /**