Merge pull request 'Make the forge's webhook a system one, and say whether its events are heard (hq issue 729)' (#309) from fix/729-the-forge-events-are-heard-or-said into main

modules/gitea/index.ts

@@ -44,7 +44,7 @@ import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
 import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked,
   type RepoCheckFacts, reposToLook } from "./pulls.js";
 import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
-import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
+import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
 import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
 
 // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
@@ -429,10 +429,16 @@ const gates = new GateWriter({
 // **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
 // process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
 // reach, nothing is served, as the tools' bundle serves nothing then.
-registerModuleTools("gitea", () => (gitea ? [commitStatusTool({
+/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what
+ *  gitea_events says of them (webhook.ts). */
+const pace = new Pace();
+if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
+
+// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
+registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
   setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
   gates,
-}), mergeTool(gitea, { gates })] : []));
+}), mergeTool(gitea, { gates })] : [])]);
 
 // **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
 // the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
@@ -451,9 +457,6 @@ async function buildStarted(event: { body: unknown }): Promise<void> {
   }
 }
 
-/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are (webhook.ts). */
-const pace = new Pace();
-
 /** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */
 async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> {
   const b = event.body as { owner?: string; repo?: string; number?: number };
@@ -476,23 +479,32 @@ async function listenForForge(client: GiteaClient): Promise<boolean> {
   const url = process.env.MESH_GITEA_HOOK_URL;
   if (!stateDir || !url) {
     console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge");
+    pace.unheard("MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset", false);
     return false;
   }
   try {
     const secret = keptSecret(join(stateDir, "webhook.secret"));
     const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT);
-    const server = receiver({ secret, port, heard: () => pace.heard(),
+    const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0";
+    const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery),
+      refused: (reason, from) => pace.refused(reason, from),
       publish: (event) => emit(event.name, event.body),
       act: (event) => actOn(client, event) });
     await new Promise<void>((resolve, reject) => {
       server.once("error", reject);
-      server.listen(port, process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0", () => resolve());
+      server.listen(port, host, () => resolve());
     });
-    const did = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
-    console.log(`[gitea] listening for the forge's events on port ${port}; its system webhook ${did}, every event, signed`);
+    const hook = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
+    pace.listening(host, port, hook.url, hook);
+    console.log(`[gitea] listening for the forge's events on ${host}:${port}; its system webhook ${hook.id} ${hook.did} ` +
+      `at ${hook.url} (in place of ${hook.removed.system} system, ${hook.removed.default} default and ` +
+      `${hook.removed.repository} repository webhook(s)${hook.removed.repositories.length ? ` in ${hook.removed.repositories.join(", ")}` : ""}), ` +
+      `${hook.events.length} event families, signed`);
     return true;
   } catch (err) {
-    console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${err instanceof Error ? err.message : err}`);
+    const why = err instanceof Error ? err.message : String(err);
+    pace.unheard(why);
+    console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${why}`);
     return false;
   }
 }

modules/gitea/test/webhook.test.ts

@@ -83,20 +83,235 @@ test("each forge event is named in the module's stream and trimmed", async () =>
   assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other");
 });
 
-test("the system webhook is made when none points at the receiver, and corrected when one does", async () => {
-  const { ensureSystemHook } = await import("../webhook.ts");
+// A forge's admin hooks as 1.27.3 keeps them: POST /admin/hooks makes a system webhook only when config.is_system_webhook
+// is "true", and a default webhook otherwise; the listing names one kind at a time (type=system, the default, or default).
+// Repositories hold their own webhooks (a default webhook's copies), listed a page at a time.
+function fakeForge(start: { id: number; url: string; system: boolean }[] = [],
+  repoHooks: Record<string, { id: number; url: string }[]> = {}, repoCount = 0) {
+  const hooks = start.map((h) => ({ ...h, events: ["push"], active: true }));
+  const repos = [...Object.keys(repoHooks)];
+  for (let i = 0; repos.length < repoCount; i++) repos.push(`novox/r${i}`);
+  let next = 100;
   const calls: string[] = [];
-  let hooks: unknown[] = [];
+  const made: any[] = [];
+  const page = (items: any[], q: URLSearchParams) => {
+    const p = Number(q.get("page") ?? 1), l = Number(q.get("limit") ?? 50);
+    return items.slice((p - 1) * l, p * l);
+  };
   const admin = { call: async (method: string, path: string, body?: any) => {
     calls.push(`${method} ${path}`);
-    if (method === "GET") return { status: 200, body: hooks };
-    if (method === "POST") { assert.ok(body.events.includes("pull_request") && body.events.includes("push")); assert.equal(body.config.secret, secret); return { status: 201, body: {} }; }
-    return { status: 200, body: {} };
+    const u = new URL(path, "http://f");
+    const repoHook = u.pathname.match(/^\/repos\/([^/]+\/[^/]+)\/hooks(?:\/(\d+))?$/);
+    if (method === "GET" && u.pathname === "/repos/search") {
+      return { status: 200, body: { ok: true, data: page(repos.map((full_name) => ({ full_name })), u.searchParams) } };
+    }
+    if (repoHook) {
+      const list = repoHooks[repoHook[1]] ?? [];
+      if (method === "GET") return { status: 200, body: page(list.map((h) => ({ id: h.id, config: { url: h.url } })), u.searchParams) };
+      const i = list.findIndex((h) => h.id === Number(repoHook[2]));
+      if (i < 0) return { status: 404, body: {} };
+      list.splice(i, 1);
+      return { status: 204, body: null };
+    }
+    if (method === "GET") {
+      const type = u.searchParams.get("type") ?? "system";
+      const listed = hooks.filter((h) => type === "all" || h.system === (type === "system"));
+      return { status: 200, body: listed.map((h) => ({ id: h.id, active: h.active, events: h.events,
+        config: { url: h.url, content_type: "json" } })) };
+    }
+    if (method === "POST") {
+      made.push(body);
+      hooks.push({ id: next++, url: body.config.url, system: body.config.is_system_webhook === "true", events: body.events,
+        active: body.active });
+      return { status: 201, body: {} };
+    }
+    if (method === "DELETE") {
+      const id = Number(path.split("/").pop());
+      const i = hooks.findIndex((h) => h.id === id);
+      if (i < 0) return { status: 404, body: {} };
+      hooks.splice(i, 1);
+      return { status: 204, body: null };
+    }
+    return { status: 405, body: {} };
   } };
-  assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "created");
-  hooks = [{ id: 4, config: { url: "http://h:3047/forge-event" } }];
-  assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "corrected");
-  assert.ok(calls.includes("PATCH /admin/hooks/4"));
+  return { admin, hooks, calls, made, repoHooks };
+}
+
+test("the webhook is made a system webhook, which the forge sends every repository's events to", async () => {
+  const { ensureSystemHook } = await import("../webhook.ts");
+  const forge = fakeForge();
+  const url = "http://h:3047/forge-event";
+  const hook = await ensureSystemHook(forge.admin, url, secret);
+  assert.equal(forge.made.length, 1);
+  assert.equal(forge.made[0].config.is_system_webhook, "true", "without it the forge makes a default webhook, which fires for no repository");
+  assert.equal(forge.made[0].config.secret, secret);
+  assert.ok(forge.made[0].events.includes("pull_request") && forge.made[0].events.includes("push"));
+  assert.deepEqual(forge.hooks.map((h) => h.system), [true]);
+  assert.equal(hook.kind, "system");
+  assert.equal(hook.did, "created");
+  assert.equal(hook.url, url);
+  assert.ok(hook.events.includes("pull_request"));
+});
+
+test("default webhooks left at the receiver by an earlier start are replaced by one system webhook", async () => {
+  const { ensureSystemHook } = await import("../webhook.ts");
+  const url = "http://h:3047/forge-event";
+  // What the start at 23:05 left: a default webhook, invisible to the system listing, made again by every start.
+  const forge = fakeForge([{ id: 4, url, system: false }, { id: 5, url, system: false }, { id: 6, url: "http://other/x", system: true }]);
+  const hook = await ensureSystemHook(forge.admin, url, secret);
+  assert.deepEqual(hook.removed, { system: 0, default: 2, repository: 0, repositories: [] });
+  assert.equal(hook.did, "replaced");
+  assert.deepEqual(forge.hooks.map((h) => [h.url, h.system]), [["http://other/x", true], [url, true]], "another url is kept");
+  // A second start replaces the system webhook, so the secret kept now is the one the forge signs with (its edit never
+  // changes a secret).
+  const again = await ensureSystemHook(forge.admin, url, secret);
+  assert.deepEqual(again.removed, { system: 1, default: 0, repository: 0, repositories: [] });
+  assert.equal(forge.hooks.filter((h) => h.url === url).length, 1);
+  assert.ok(forge.calls.includes("DELETE /admin/hooks/4") && forge.calls.includes("DELETE /admin/hooks/5"));
+  // The new one is made before any old one is deleted, so no event falls in a gap.
+  const post = forge.calls.indexOf("POST /admin/hooks");
+  assert.ok(post >= 0 && forge.calls.findIndex((c) => c.startsWith("DELETE")) > post);
+});
+
+test("a repository's copy of a default webhook at the receiver is deleted, on every page, and said", async () => {
+  const { ensureSystemHook } = await import("../webhook.ts");
+  const url = "http://h:3047/forge-event";
+  // A repository made after the default webhook got its copy, which would send its events a second time; 120
+  // repositories, so the copies sit beyond the first page.
+  const repoHooks: Record<string, { id: number; url: string }[]> = {};
+  for (let i = 0; i < 118; i++) repoHooks[`novox/r${i}`] = [];
+  repoHooks["novox/late"] = [{ id: 31, url }, { id: 32, url: "http://ci/hook" }];
+  repoHooks["novox/later"] = [{ id: 33, url }];
+  const forge = fakeForge([{ id: 4, url, system: false }], repoHooks);
+  const hook = await ensureSystemHook(forge.admin, url, secret);
+  assert.deepEqual(hook.removed, { system: 0, default: 1, repository: 2, repositories: ["novox/late", "novox/later"] });
+  assert.deepEqual(forge.repoHooks["novox/late"].map((h) => h.id), [32], "a repository's other webhook is kept");
+  assert.deepEqual(forge.repoHooks["novox/later"], []);
+  assert.ok(forge.calls.some((c) => /\/repos\/search\?.*page=3/.test(c)), "every page of repositories is read");
+});
+
+test("a forge that makes no system webhook is said, never taken as heard", async () => {
+  const { ensureSystemHook } = await import("../webhook.ts");
+  const forge = fakeForge();
+  const admin = { call: async (method: string, path: string, body?: any) => {
+    // A forge that ignores the flag: what 23:05 met, had the module read back.
+    if (method === "POST") body = { ...body, config: { ...body.config, is_system_webhook: undefined } };
+    return forge.admin.call(method, path, body);
+  } };
+  await assert.rejects(ensureSystemHook(admin, "http://h:3047/forge-event", secret), /names 0 new system webhooks/);
+});
+
+test("the receiver tells what it verified and what it refused", async () => {
+  const { receiver, HOOK_PATH } = await import("../webhook.ts");
+  const heard: string[] = [], refused: string[] = [];
+  const server = receiver({ secret, port: 0, log: () => {}, publish: async () => {},
+    heard: (kind, delivery) => heard.push(`${kind} ${delivery}`), refused: (reason) => refused.push(reason) });
+  await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
+  const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`;
+  try {
+    await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-9" } });
+    await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "x") } });
+    await server.idle();
+    assert.deepEqual(heard, ["pull_request d-9"]);
+    assert.deepEqual(refused, ["its signature does not verify"]);
+  } finally { server.close(); }
+});
+
+test("a forge that never delivers is said loudly once, and unhealthy, when the poll finds what no event said", async () => {
+  const { Pace } = await import("../webhook.ts");
+  let now = 1_000_000;
+  const said: string[] = [];
+  const pace = new Pace(() => now, (l) => said.push(l));
+  const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
+    did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
+  pace.listening("172.17.0.1", 3047, hook.url, hook);
+  now += 5 * 60_000;
+  pace.miss("novox/hq#7 at aaaaaaaa");
+  assert.equal(said.length, 0, "five minutes after the start is not yet silence");
+  assert.equal((pace.report() as any).healthy, true);
+  pace.refused("it is unsigned", "172.17.0.3");
+  assert.equal((pace.report() as any).healthy, false, "refused, and none verified");
+  now += 6 * 60_000;
+  pace.miss("the merge of novox/hq#7");
+  pace.miss("novox/hq#8 at bbbbbbbb");
+  assert.equal(said.filter((l) => /NOT HEARD: no delivery ever verified/.test(l)).length, 1, "said once");
+  let r = pace.report() as any;
+  assert.equal(r.healthy, false);
+  assert.equal(r.state, "unhealthy");
+  assert.match(r.why, /never delivered/);
+  assert.equal(r.polls.found_unsaid, 3);
+  assert.equal(r.refused.count, 1);
+  assert.equal(r.refused.last.reason, "it is unsigned");
+  assert.equal(r.hook.kind, "system");
+  assert.equal(r.receiver.port, 3047);
+  // The first delivery clears it.
+  now += 1000;
+  pace.heard("push", "d-1");
+  r = pace.report() as any;
+  assert.equal(r.healthy, true);
+  assert.equal(r.verified.count, 1);
+  assert.equal(r.verified.last.event, "push");
+  assert.equal(r.polls.every_seconds, 300);
+  assert.match(said.join("\n"), /heard again/);
+});
+
+test("a forge whose events stop after one delivery is said loudly once per silent spell, and refusals are unhealthy", async () => {
+  const { Pace } = await import("../webhook.ts");
+  let now = 1_000_000;
+  const said: string[] = [];
+  const pace = new Pace(() => now, (l) => said.push(l));
+  const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
+    did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
+  pace.listening("172.17.0.1", 3047, hook.url, hook);
+  now += 60_000;
+  pace.heard("push", "d-1");
+  // The hook is deleted by hand: nothing more comes, and the poll finds what no event said.
+  now += 11 * 60_000;
+  pace.miss("novox/hq#7 at aaaaaaaa");
+  assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 0, "one find alone may be the poll beating an event");
+  pace.miss("the merge of novox/hq#7");
+  pace.miss("novox/hq#8 at bbbbbbbb");
+  assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 1, "said once");
+  assert.match(said.find((l) => /NOT HEARD/.test(l)) ?? "", /since the last at/);
+  assert.equal((pace.report() as any).healthy, false);
+  // A delivery ends the spell; the next spell is said again.
+  now += 1000;
+  pace.heard("push", "d-2");
+  assert.equal((pace.report() as any).healthy, true);
+  now += 11 * 60_000;
+  pace.miss("novox/hq#9 at cccccccc");
+  pace.miss("novox/hq#10 at dddddddd");
+  assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 2, "a second silent spell is said again");
+  // Every delivery refused after a verified one (a secret that no longer matches) is unhealthy at once.
+  now += 1000;
+  pace.heard("push", "d-3");
+  now += 1000;
+  pace.refused("its signature does not verify", "172.17.0.3");
+  pace.refused("its signature does not verify", "172.17.0.3");
+  const r = pace.report() as any;
+  assert.equal(r.healthy, false);
+  assert.equal(r.refused.since_verified, 2);
+  assert.match(r.why, /2 deliveries refused since the last verified one.*does not verify/);
+  now += 1000;
+  pace.heard("push", "d-4");
+  assert.equal((pace.report() as any).healthy, true, "a verified delivery clears it");
+});
+
+test("without a receiver by configuration the polls alone watch, healthy; a receiver that failed is unhealthy", async () => {
+  const { Pace } = await import("../webhook.ts");
+  let now = 1_000_000;
+  const said: string[] = [];
+  const quiet = new Pace(() => now, (l) => said.push(l));
+  quiet.unheard("MESH_GITEA_HOOK_URL unset", false);
+  now += 20 * 60_000;
+  quiet.miss("novox/hq#7 at aaaaaaaa");
+  assert.equal(said.length, 0, "no deliveries were expected");
+  assert.equal((quiet.report() as any).healthy, true);
+  const failed = new Pace(() => now, () => {});
+  failed.unheard("the forge's system webhook could not be made: 403");
+  const r = failed.report() as any;
+  assert.equal(r.healthy, false);
+  assert.match(r.why, /403/);
 });
 
 test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => {

modules/gitea/webhook.ts

@@ -140,8 +140,11 @@ export interface ReceiverOptions {
   publish: (event: ForgeEvent) => Promise<void>;
   /** Acts on it at once, beside publishing it (the announcer). */
   act?: (event: ForgeEvent) => Promise<void>;
-  /** Told of every verified delivery, so the polls know events are heard. */
-  heard?: () => void;
+  /** Told of every verified delivery — its kind (X-Gitea-Event) and its delivery id — so the polls know events are
+   *  heard, and the module can say when it last heard one. */
+  heard?: (kind: string, delivery: string) => void;
+  /** Told of every refused delivery, with the reason and where it came from. */
+  refused?: (reason: string, from: string) => void;
   log?: (line: string) => void;
 }
 
@@ -158,6 +161,10 @@ export const REMEMBERED = 2000;
  */
 export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void> } {
   const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`));
+  const refuse = (from: string, reason: string): void => {
+    log(`refused a forge event from ${from}: ${reason}`);
+    opts.refused?.(reason, from);
+  };
   const seen = new Set<string>();
   let queue: Promise<void> = Promise.resolve();
   const server = createServer((req, res) => {
@@ -168,14 +175,14 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
     const from = req.socket.remoteAddress ?? "?";
     const signature = String(req.headers["x-gitea-signature"] ?? "").trim();
     if (!signature) {
-      log(`refused a forge event from ${from}: it is unsigned`);
+      refuse(from, "it is unsigned");
       res.writeHead(401, { Connection: "close" });
       res.end("unsigned\n");
       req.destroy();
       return;
     }
     if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) {
-      log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`);
+      refuse(from, `its body is over ${MAX_BODY} bytes`);
       res.writeHead(413, { Connection: "close" });
       res.end("too large\n");
       req.destroy();
@@ -189,7 +196,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
       if (size > MAX_BODY) {
         if (!over) {
           over = true;
-          log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`);
+          refuse(from, `its body is over ${MAX_BODY} bytes`);
           res.writeHead(413, { Connection: "close" });
           res.end("too large\n");
           req.destroy();
@@ -200,7 +207,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
       if (over) return;
       const body = Buffer.concat(chunks);
       if (!verified(opts.secret, body, signature)) {
-        log(`refused a forge event from ${from}: its signature does not verify`);
+        refuse(from, "its signature does not verify");
         res.statusCode = 401;
         return res.end("signature does not verify\n");
       }
@@ -208,11 +215,13 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
       try {
         payload = JSON.parse(body.toString("utf8"));
       } catch {
+        refuse(from, "it is signed but not JSON");
         res.statusCode = 400;
         return res.end("not JSON\n");
       }
-      opts.heard?.();
       const delivery = String(req.headers["x-gitea-delivery"] ?? "");
+      const kind = String(req.headers["x-gitea-event"] ?? "other");
+      opts.heard?.(kind, delivery);
       if (delivery && seen.has(delivery)) {
         res.statusCode = 200;
         return res.end("already heard\n");
@@ -221,7 +230,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
         seen.add(delivery);
         if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string);
       }
-      const event = eventOf(String(req.headers["x-gitea-event"] ?? "other"), payload);
+      const event = eventOf(kind, payload);
       res.statusCode = 200;
       res.end("queued\n");
       queue = queue.then(async () => {
@@ -242,27 +251,113 @@ export interface HookAdmin {
   call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>;
 }
 
+/** The forge's one system webhook as ensured at start: read back from the forge after it was made. */
+export interface HookState {
+  /** Always "system": a hook the forge's listing of system webhooks names, so it fires for every repository. */
+  kind: "system";
+  id: number;
+  url: string;
+  events: string[];
+  active: boolean;
+  content_type?: string;
+  /** Made where none was, or made in place of the ones there (system, default, or a repository's) at the receiver's url. */
+  did: "created" | "replaced";
+  /** The hooks at the receiver's url it took the place of, by kind: a repository's is a copy of a default webhook, which
+   *  would have sent that repository's events a second time. */
+  removed: { system: number; default: number; repository: number; repositories: string[] };
+}
+
+/** How long without a verified delivery — since the start, or since the last one — a forge is said silent once the poll
+ *  finds what no event said. */
+export const SILENT_AFTER = 10 * 60_000;
+
 /**
- * **How fast the polls look** (issue 729): every five minutes only while the forge's events are heard — a verified
- * delivery within the last ten minutes, and none missed since. Until the first delivery, after ten minutes without one,
- * or once the poll found something no event said, every thirty seconds.
+ * **How fast the polls look, and whether the forge's events are heard** (issue 729): every five minutes only while the
+ * forge's events are heard — a verified delivery within the last ten minutes, and none missed since. Until the first
+ * delivery, after ten minutes without one, or once the poll found something no event said, every thirty seconds.
+ *
+ * It also keeps what `gitea_events` says: where the receiver listens, the hook as ensured, the verified and refused
+ * deliveries, and the polls' finds no event said. **A forge whose events stop is said** (issue 729): once the receiver
+ * listens and the hook is ensured, when nothing verified for ten minutes — since the start, or since the last delivery —
+ * and the poll has found at least two heads or merges no event said in that quiet (one could be the poll beating an
+ * event by a second), it is logged loudly once per silent spell, and the module's events are said unhealthy until a
+ * delivery verifies. Deliveries refused since the last verified one are said unhealthy too.
  */
 export class Pace {
   private last = 0;
   private missed = 0;
+  private readonly started: number;
   private readonly now: () => number;
   private readonly log: (l: string) => void;
+  private where: { host: string; port: number; url: string } | null = null;
+  private hook: HookState | null = null;
+  private notHeard: string | null = null;
+  private notHeardFails = false;
+  private verifiedCount = 0;
+  private lastVerified: { at: number; event: string; delivery: string } | null = null;
+  private refusedCount = 0;
+  private refusedSpell = 0;
+  private lastRefused: { at: number; reason: string; from: string } | null = null;
+  private missCount = 0;
+  private lastMiss: { at: number; what: string } | null = null;
+  private silentSince = 0;
+  private quietMisses = 0;
   constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) {
     this.now = now;
     this.log = log;
+    this.started = now();
   }
-  heard(): void {
+  /** The receiver listens, and the hook at it was ensured: from now on, deliveries are expected. */
+  listening(host: string, port: number, url: string, hook: HookState): void {
+    this.where = { host, port, url };
+    this.hook = hook;
+    this.notHeard = null;
+  }
+  /** No events are listened for, and why: by configuration (`fails` false), or because the receiver could not listen
+   *  or the hook could not be ensured (`fails` true, said unhealthy). The polls alone watch. */
+  unheard(why: string, fails = true): void {
+    this.notHeard = why;
+    this.notHeardFails = fails;
+  }
+  heard(event = "", delivery = ""): void {
     this.last = this.now();
+    this.verifiedCount++;
+    this.lastVerified = { at: this.last, event, delivery };
+    this.refusedSpell = 0;
+    this.quietMisses = 0;
+    if (this.silentSince) this.log(`the forge's events are heard again: ${event || "a delivery"} verified`);
+    this.silentSince = 0;
+  }
+  refused(reason: string, from: string): void {
+    this.refusedCount++;
+    this.refusedSpell++;
+    this.lastRefused = { at: this.now(), reason, from };
   }
   /** The poll found a head or merge no forge event announced. */
   miss(what: string): void {
+    const t = this.now();
     if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`);
-    this.missed = this.now();
+    this.missed = t;
+    this.missCount++;
+    this.lastMiss = { at: t, what };
+    // Silent: nothing verified for SILENT_AFTER — never since the start, or not since the last delivery (a hook deleted by
+    // hand, every delivery refused) — while the poll finds what no event said. Said once per silent spell.
+    const quietSince = this.last || this.started;
+    this.quietMisses++;
+    if (this.expecting() && t - quietSince >= SILENT_AFTER && this.quietMisses >= 2 && !this.silentSince) {
+      this.silentSince = t;
+      const minutes = Math.round((t - quietSince) / 60_000);
+      const refusedSince = this.refusedSinceHeard();
+      this.log(`THE FORGE'S EVENTS ARE NOT HEARD: ${this.last ? `no delivery verified since the last at ` +
+        `${new Date(this.last).toISOString()}` : "no delivery ever verified since the start"}, ${minutes} min, from the ` +
+        `system webhook at ${this.hook?.url}${refusedSince ? `; ${refusedSince} refused since, the last because ` +
+        `${this.lastRefused?.reason}` : ""}; the poll found ${what} — the polls alone watch the forge, every 30 s ` +
+        `(novox/hq issue 729)`);
+    }
+  }
+  /** Whether a delivery was refused after the last verified one (or with none verified). */
+  private refusedSinceHeard(): number {
+    return this.lastRefused && this.lastRefused.at >= this.last ? this.refusedSpell : 0;
   }
   hearing(): boolean {
     const t = this.now();
@@ -271,36 +366,146 @@ export class Pace {
   every(): number {
     return this.hearing() ? 5 * 60_000 : 30_000;
   }
+  private expecting(): boolean {
+    return this.where !== null && this.hook !== null;
+  }
+  /** What `gitea_events` answers: healthy, unhealthy or waits (ADR 0283's shape), with every fact behind it. */
+  report(): Record<string, unknown> {
+    const t = this.now();
+    const iso = (ms: number): string => new Date(ms).toISOString();
+    let state: "healthy" | "unhealthy" | "waits" = "healthy";
+    let why: string;
+    const refusedSince = this.refusedSinceHeard();
+    if (this.silentSince) {
+      state = "unhealthy";
+      why = `${this.last ? `no delivery verified since the last at ${iso(this.last)}` : `the forge has never delivered an ` +
+        `event since the start at ${iso(this.started)}`}, while the poll found ${this.missCount} head(s) or merge(s) no ` +
+        `event said, the last ${this.lastMiss?.what} at ${iso(this.lastMiss?.at ?? t)}`;
+    } else if (refusedSince) {
+      state = "unhealthy";
+      why = `${refusedSince} deliveries refused ${this.last ? `since the last verified one at ${iso(this.last)}` :
+        "and none verified"}, the last at ${iso(this.lastRefused?.at ?? t)} because ${this.lastRefused?.reason}`;
+    } else if (!this.expecting()) {
+      why = this.notHeard ? `no events are listened for, so the polls alone watch the forge: ${this.notHeard}` :
+        "the receiver is not listening yet";
+      if (this.notHeard && this.notHeardFails) state = "unhealthy";
+    } else if (this.hearing()) {
+      why = `the forge's events are heard: ${this.verifiedCount} verified, the last at ${iso(this.last)}`;
+    } else if (this.last === 0) {
+      why = `no delivery heard yet; the forge is said silent if the poll finds what no event said ${SILENT_AFTER / 60_000} min after the start`;
+    } else {
+      why = `the last delivery was at ${iso(this.last)}; the polls look every 30 s until the next`;
+    }
+    return {
+      healthy: state !== "unhealthy", state, why,
+      started: iso(this.started),
+      receiver: this.where ? { listening: true, ...this.where } : { listening: false, why: this.notHeard ?? "not started" },
+      hook: this.hook,
+      verified: { count: this.verifiedCount, last: this.lastVerified && { ...this.lastVerified, at: iso(this.lastVerified.at) } },
+      refused: { count: this.refusedCount, since_verified: refusedSince, last: this.lastRefused && { ...this.lastRefused, at: iso(this.lastRefused.at) } },
+      polls: { every_seconds: this.every() / 1000, hearing: this.hearing(), found_unsaid: this.missCount,
+        last_found: this.lastMiss && { ...this.lastMiss, at: iso(this.lastMiss.at) } },
+      silent: this.silentSince ? { since: iso(this.silentSince) } : null,
+    };
+  }
 }
 
 /** Every event family the forge (1.27) can send a webhook. */
 export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request",
   "pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"];
 
-/**
- * **One system webhook, for every event, at the receiver** (issue 729): created when none points there, corrected
- * when one does but differs (events, active, content type), and its secret set again each start — the forge never
- * hands a secret back, so a corrected secret cannot be told from a kept one. Answers what it did.
- */
-export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<"created" | "corrected"> {
-  // POST /admin/hooks makes a system webhook, and its options take no flag for it (the forge's 1.27.3 API schema). The
-  // forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it.
-  const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*",
-    config: { url, content_type: "json", secret } };
-  const listed = await admin.call("GET", "/admin/hooks?type=system&limit=50");
-  if (listed.status !== 200) throw new Error(`the forge's system webhooks could not be listed: ${listed.status}`);
-  const hooks: any[] = Array.isArray(listed.body) ? listed.body : [];
-  const ours = hooks.filter((h) => h?.config?.url === url);
-  if (ours.length === 0) {
-    const made = await admin.call("POST", "/admin/hooks", want);
-    if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
-    return "created";
+async function hooksAt(admin: HookAdmin, type: "system" | "default", url: string): Promise<any[]> {
+  const listed = await admin.call("GET", `/admin/hooks?type=${type}&limit=50`);
+  if (listed.status !== 200) throw new Error(`the forge's ${type} webhooks could not be listed: ${listed.status}`);
+  return (Array.isArray(listed.body) ? listed.body : []).filter((h) => h?.config?.url === url);
+}
+
+/** Every page of a listing that answers an array (or `{ data: [] }`, the repository search), PAGE at a time. */
+const PAGE = 50;
+async function everyPage(admin: HookAdmin, path: string, what: string): Promise<any[]> {
+  const all: any[] = [];
+  for (let page = 1; ; page++) {
+    const r = await admin.call("GET", `${path}${path.includes("?") ? "&" : "?"}page=${page}&limit=${PAGE}`);
+    if (r.status !== 200) throw new Error(`${what} could not be listed: ${r.status}`);
+    const items: any[] = Array.isArray(r.body) ? r.body : Array.isArray(r.body?.data) ? r.body.data : [];
+    all.push(...items);
+    if (items.length < PAGE) return all;
   }
-  for (const extra of ours.slice(1)) await admin.call("DELETE", `/admin/hooks/${extra.id}`);
-  const fixed = await admin.call("PATCH", `/admin/hooks/${ours[0].id}`, { active: true, events: HOOK_EVENTS, branch_filter: "*",
-    config: { url, content_type: "json", secret } });
-  if (fixed.status !== 200) throw new Error(`the forge's system webhook could not be corrected: ${fixed.status}`);
-  return "corrected";
+}
+
+/** Every repository's own webhooks at the receiver's url: copies of a default webhook made at it. */
+async function repoHooksAt(admin: HookAdmin, url: string): Promise<{ repo: string; id: number }[]> {
+  const found: { repo: string; id: number }[] = [];
+  for (const repo of await everyPage(admin, "/repos/search?sort=id", "the forge's repositories")) {
+    const full = String(repo?.full_name ?? "");
+    if (!full.includes("/")) continue;
+    const hooks = await everyPage(admin, `/repos/${full}/hooks`, `${full}'s webhooks`);
+    for (const h of hooks) if (h?.config?.url === url) found.push({ repo: full, id: Number(h.id) });
+  }
+  return found;
+}
+
+/**
+ * **One system webhook, for every event, at the receiver** (issue 729), made each start and read back.
+ *
+ * The forge (1.27.3, routers/api/v1/utils/hook.go addHook) makes a *default* webhook from POST /admin/hooks unless
+ * `config.is_system_webhook` is "true": a default webhook fires for no repository and is only copied into repositories
+ * made after it, and the listing of system webhooks does not name it — so without the flag every start made another
+ * one and no event came. Its edit (editHook) never changes a webhook's secret nor its kind, so a hook already there is
+ * not corrected but replaced: the new system webhook is made first, with the secret kept now, so no event falls in a
+ * gap; then every other hook at the receiver's url is deleted — system, default, and each repository's copy of a
+ * default one, which would send that repository's events twice. The forge's listing of system webhooks must then name
+ * exactly one at the url, or this fails, saying so.
+ */
+export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<HookState> {
+  // The forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it.
+  const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*",
+    config: { url, content_type: "json", secret, is_system_webhook: "true" } };
+  const before = new Set((await hooksAt(admin, "system", url)).map((h) => Number(h.id)));
+  const made = await admin.call("POST", "/admin/hooks", want);
+  if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
+  const fresh = (await hooksAt(admin, "system", url)).filter((h) => !before.has(Number(h.id)));
+  if (fresh.length !== 1) {
+    throw new Error(`the forge names ${fresh.length} new system webhooks at ${url} after one was made, not one: its events ` +
+      `are not known to be sent`);
+  }
+  const ours = fresh[0];
+  const defaults = await hooksAt(admin, "default", url);
+  const gone = async (path: string): Promise<void> => {
+    const r = await admin.call("DELETE", path);
+    if (r.status !== 204 && r.status !== 200 && r.status !== 404) throw new Error(`the forge's webhook ${path} could not be deleted: ${r.status}`);
+  };
+  for (const id of before) await gone(`/admin/hooks/${id}`);
+  for (const h of defaults) await gone(`/admin/hooks/${h.id}`);
+  const copies = await repoHooksAt(admin, url);
+  for (const c of copies) await gone(`/repos/${c.repo}/hooks/${c.id}`);
+  const kept = await hooksAt(admin, "system", url);
+  if (kept.length !== 1 || Number(kept[0].id) !== Number(ours.id)) {
+    throw new Error(`the forge names ${kept.length} system webhooks at ${url} once the others were deleted, not the one ` +
+      `made (${ours.id})`);
+  }
+  const removed = { system: before.size, default: defaults.length, repository: copies.length,
+    repositories: [...new Set(copies.map((c) => c.repo))] };
+  return { kind: "system", id: Number(ours.id), url, events: Array.isArray(ours.events) ? ours.events : [],
+    active: !!ours.active, content_type: ours.config?.content_type,
+    did: removed.system + removed.default + removed.repository > 0 ? "replaced" : "created", removed };
+}
+
+/** The `gitea_events` tool: what the module knows of the forge's events, from its Pace. */
+export function eventsTool(pace: Pace): { name: string; description: string; input: Record<string, unknown>;
+  run: (args: Readonly<Record<string, unknown>>) => Promise<unknown> } {
+  return {
+    name: "gitea_events",
+    description: "Whether the forge's events reach the mesh (novox/hq issue 729): where the receiver listens, the " +
+      "system webhook as ensured at the start (its id, url, events, kind, and what it replaced), how many deliveries " +
+      "verified and the last one's time and event, how many were refused and the last reason, how often the polls look " +
+      "and whether events are heard, and how many heads or merges the poll found that no event said. Unhealthy when the " +
+      "forge has delivered nothing that verified for ten minutes (since the start or since the last delivery) while the " +
+      "poll found what no event said, when deliveries were refused since the last verified one, or when the receiver " +
+      "could not listen or the webhook could not be made.",
+    input: {},
+    run: async () => pace.report(),
+  };
 }
 
 /**