Make the forge's webhook a system one, and say whether its events are heard (hq issue 729)

POST /admin/hooks on gitea 1.27.3 makes a default webhook unless
config.is_system_webhook is "true": a default webhook fires for no existing
repository, and the system listing does not name it, so every start made
another and no event reached the receiver. Its edit never changes a secret
or a kind, so the new system webhook is made first and read back, then every
other hook at the receiver is deleted: system, default, and each
repository's copy of a default one, which would send its events twice.

gitea_events says where the receiver listens, the hook as ensured and what
it replaced, the verified and refused deliveries and the polls' pace. When
nothing verified for ten minutes, since the start or the last delivery,
while the poll found two heads or merges no event said, it is logged loudly
once per silent spell and answered unhealthy; so are deliveries refused
since the last verified one.

modules/gitea/index.ts

@@ -44,7 +44,7 @@ import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
 import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked,
   type RepoCheckFacts, reposToLook } from "./pulls.js";
 import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
-import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
+import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
 import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
 
 // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
@@ -429,10 +429,16 @@ const gates = new GateWriter({
 // **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
 // process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
 // reach, nothing is served, as the tools' bundle serves nothing then.
-registerModuleTools("gitea", () => (gitea ? [commitStatusTool({
+/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what
+ *  gitea_events says of them (webhook.ts). */
+const pace = new Pace();
+if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
+
+// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
+registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
   setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
   gates,
-}), mergeTool(gitea, { gates })] : []));
+}), mergeTool(gitea, { gates })] : [])]);
 
 // **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
 // the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
@@ -451,9 +457,6 @@ async function buildStarted(event: { body: unknown }): Promise<void> {
   }
 }
 
-/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are (webhook.ts). */
-const pace = new Pace();
-
 /** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */
 async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> {
   const b = event.body as { owner?: string; repo?: string; number?: number };
@@ -476,23 +479,32 @@ async function listenForForge(client: GiteaClient): Promise<boolean> {
   const url = process.env.MESH_GITEA_HOOK_URL;
   if (!stateDir || !url) {
     console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge");
+    pace.unheard("MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset", false);
     return false;
   }
   try {
     const secret = keptSecret(join(stateDir, "webhook.secret"));
     const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT);
-    const server = receiver({ secret, port, heard: () => pace.heard(),
+    const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0";
+    const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery),
+      refused: (reason, from) => pace.refused(reason, from),
       publish: (event) => emit(event.name, event.body),
       act: (event) => actOn(client, event) });
     await new Promise<void>((resolve, reject) => {
       server.once("error", reject);
-      server.listen(port, process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0", () => resolve());
+      server.listen(port, host, () => resolve());
     });
-    const did = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
-    console.log(`[gitea] listening for the forge's events on port ${port}; its system webhook ${did}, every event, signed`);
+    const hook = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
+    pace.listening(host, port, hook.url, hook);
+    console.log(`[gitea] listening for the forge's events on ${host}:${port}; its system webhook ${hook.id} ${hook.did} ` +
+      `at ${hook.url} (in place of ${hook.removed.system} system, ${hook.removed.default} default and ` +
+      `${hook.removed.repository} repository webhook(s)${hook.removed.repositories.length ? ` in ${hook.removed.repositories.join(", ")}` : ""}), ` +
+      `${hook.events.length} event families, signed`);
     return true;
   } catch (err) {
-    console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${err instanceof Error ? err.message : err}`);
+    const why = err instanceof Error ? err.message : String(err);
+    pace.unheard(why);
+    console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${why}`);
     return false;
   }
 }

modules/gitea/test/webhook.test.ts

@@ -83,20 +83,235 @@ test("each forge event is named in the module's stream and trimmed", async () =>
   assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other");
 });
 
-test("the system webhook is made when none points at the receiver, and corrected when one does", async () => {
-  const { ensureSystemHook } = await import("../webhook.ts");
+// A forge's admin hooks as 1.27.3 keeps them: POST /admin/hooks makes a system webhook only when config.is_system_webhook
+// is "true", and a default webhook otherwise; the listing names one kind at a time (type=system, the default, or default).
+// Repositories hold their own webhooks (a default webhook's copies), listed a page at a time.
+function fakeForge(start: { id: number; url: string; system: boolean }[] = [],
+  repoHooks: Record<string, { id: number; url: string }[]> = {}, repoCount = 0) {
+  const hooks = start.map((h) => ({ ...h, events: ["push"], active: true }));
+  const repos = [...Object.keys(repoHooks)];
+  for (let i = 0; repos.length < repoCount; i++) repos.push(`novox/r${i}`);
+  let next = 100;
   const calls: string[] = [];
-  let hooks: unknown[] = [];
+  const made: any[] = [];
+  const page = (items: any[], q: URLSearchParams) => {
+    const p = Number(q.get("page") ?? 1), l = Number(q.get("limit") ?? 50);
+    return items.slice((p - 1) * l, p * l);
+  };
   const admin = { call: async (method: string, path: string, body?: any) => {
     calls.push(`${method} ${path}`);
-    if (method === "GET") return { status: 200, body: hooks };
-    if (method === "POST") { assert.ok(body.events.includes("pull_request") && body.events.includes("push")); assert.equal(body.config.secret, secret); return { status: 201, body: {} }; }
-    return { status: 200, body: {} };
+    const u = new URL(path, "http://f");
+    const repoHook = u.pathname.match(/^\/repos\/([^/]+\/[^/]+)\/hooks(?:\/(\d+))?$/);
+    if (method === "GET" && u.pathname === "/repos/search") {
+      return { status: 200, body: { ok: true, data: page(repos.map((full_name) => ({ full_name })), u.searchParams) } };
+    }
+    if (repoHook) {
+      const list = repoHooks[repoHook[1]] ?? [];
+      if (method === "GET") return { status: 200, body: page(list.map((h) => ({ id: h.id, config: { url: h.url } })), u.searchParams) };
+      const i = list.findIndex((h) => h.id === Number(repoHook[2]));
+      if (i < 0) return { status: 404, body: {} };
+      list.splice(i, 1);
+      return { status: 204, body: null };
+    }
+    if (method === "GET") {
+      const type = u.searchParams.get("type") ?? "system";
+      const listed = hooks.filter((h) => type === "all" || h.system === (type === "system"));
+      return { status: 200, body: listed.map((h) => ({ id: h.id, active: h.active, events: h.events,
+        config: { url: h.url, content_type: "json" } })) };
+    }
+    if (method === "POST") {
+      made.push(body);
+      hooks.push({ id: next++, url: body.config.url, system: body.config.is_system_webhook === "true", events: body.events,
+        active: body.active });
+      return { status: 201, body: {} };
+    }
+    if (method === "DELETE") {
+      const id = Number(path.split("/").pop());
+      const i = hooks.findIndex((h) => h.id === id);
+      if (i < 0) return { status: 404, body: {} };
+      hooks.splice(i, 1);
+      return { status: 204, body: null };
+    }
+    return { status: 405, body: {} };
   } };
-  assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "created");
-  hooks = [{ id: 4, config: { url: "http://h:3047/forge-event" } }];
-  assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "corrected");
-  assert.ok(calls.includes("PATCH /admin/hooks/4"));
+  return { admin, hooks, calls, made, repoHooks };
+}
+
+test("the webhook is made a system webhook, which the forge sends every repository's events to", async () => {
+  const { ensureSystemHook } = await import("../webhook.ts");
+  const forge = fakeForge();
+  const url = "http://h:3047/forge-event";
+  const hook = await ensureSystemHook(forge.admin, url, secret);
+  assert.equal(forge.made.length, 1);
+  assert.equal(forge.made[0].config.is_system_webhook, "true", "without it the forge makes a default webhook, which fires for no repository");
+  assert.equal(forge.made[0].config.secret, secret);
+  assert.ok(forge.made[0].events.includes("pull_request") && forge.made[0].events.includes("push"));
+  assert.deepEqual(forge.hooks.map((h) => h.system), [true]);
+  assert.equal(hook.kind, "system");
+  assert.equal(hook.did, "created");
+  assert.equal(hook.url, url);
+  assert.ok(hook.events.includes("pull_request"));
+});
+
+test("default webhooks left at the receiver by an earlier start are replaced by one system webhook", async () => {
+  const { ensureSystemHook } = await import("../webhook.ts");
+  const url = "http://h:3047/forge-event";
+  // What the start at 23:05 left: a default webhook, invisible to the system listing, made again by every start.
+  const forge = fakeForge([{ id: 4, url, system: false }, { id: 5, url, system: false }, { id: 6, url: "http://other/x", system: true }]);
+  const hook = await ensureSystemHook(forge.admin, url, secret);
+  assert.deepEqual(hook.removed, { system: 0, default: 2, repository: 0, repositories: [] });
+  assert.equal(hook.did, "replaced");
+  assert.deepEqual(forge.hooks.map((h) => [h.url, h.system]), [["http://other/x", true], [url, true]], "another url is kept");
+  // A second start replaces the system webhook, so the secret kept now is the one the forge signs with (its edit never
+  // changes a secret).
+  const again = await ensureSystemHook(forge.admin, url, secret);
+  assert.deepEqual(again.removed, { system: 1, default: 0, repository: 0, repositories: [] });
+  assert.equal(forge.hooks.filter((h) => h.url === url).length, 1);
+  assert.ok(forge.calls.includes("DELETE /admin/hooks/4") && forge.calls.includes("DELETE /admin/hooks/5"));
+  // The new one is made before any old one is deleted, so no event falls in a gap.
+  const post = forge.calls.indexOf("POST /admin/hooks");
+  assert.ok(post >= 0 && forge.calls.findIndex((c) => c.startsWith("DELETE")) > post);
+});
+
+test("a repository's copy of a default webhook at the receiver is deleted, on every page, and said", async () => {
+  const { ensureSystemHook } = await import("../webhook.ts");
+  const url = "http://h:3047/forge-event";
+  // A repository made after the default webhook got its copy, which would send its events a second time; 120
+  // repositories, so the copies sit beyond the first page.
+  const repoHooks: Record<string, { id: number; url: string }[]> = {};
+  for (let i = 0; i < 118; i++) repoHooks[`novox/r${i}`] = [];
+  repoHooks["novox/late"] = [{ id: 31, url }, { id: 32, url: "http://ci/hook" }];
+  repoHooks["novox/later"] = [{ id: 33, url }];
+  const forge = fakeForge([{ id: 4, url, system: false }], repoHooks);
+  const hook = await ensureSystemHook(forge.admin, url, secret);
+  assert.deepEqual(hook.removed, { system: 0, default: 1, repository: 2, repositories: ["novox/late", "novox/later"] });
+  assert.deepEqual(forge.repoHooks["novox/late"].map((h) => h.id), [32], "a repository's other webhook is kept");
+  assert.deepEqual(forge.repoHooks["novox/later"], []);
+  assert.ok(forge.calls.some((c) => /\/repos\/search\?.*page=3/.test(c)), "every page of repositories is read");
+});
+
+test("a forge that makes no system webhook is said, never taken as heard", async () => {
+  const { ensureSystemHook } = await import("../webhook.ts");
+  const forge = fakeForge();
+  const admin = { call: async (method: string, path: string, body?: any) => {
+    // A forge that ignores the flag: what 23:05 met, had the module read back.
+    if (method === "POST") body = { ...body, config: { ...body.config, is_system_webhook: undefined } };
+    return forge.admin.call(method, path, body);
+  } };
+  await assert.rejects(ensureSystemHook(admin, "http://h:3047/forge-event", secret), /names 0 new system webhooks/);
+});
+
+test("the receiver tells what it verified and what it refused", async () => {
+  const { receiver, HOOK_PATH } = await import("../webhook.ts");
+  const heard: string[] = [], refused: string[] = [];
+  const server = receiver({ secret, port: 0, log: () => {}, publish: async () => {},
+    heard: (kind, delivery) => heard.push(`${kind} ${delivery}`), refused: (reason) => refused.push(reason) });
+  await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
+  const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`;
+  try {
+    await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-9" } });
+    await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "x") } });
+    await server.idle();
+    assert.deepEqual(heard, ["pull_request d-9"]);
+    assert.deepEqual(refused, ["its signature does not verify"]);
+  } finally { server.close(); }
+});
+
+test("a forge that never delivers is said loudly once, and unhealthy, when the poll finds what no event said", async () => {
+  const { Pace } = await import("../webhook.ts");
+  let now = 1_000_000;
+  const said: string[] = [];
+  const pace = new Pace(() => now, (l) => said.push(l));
+  const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
+    did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
+  pace.listening("172.17.0.1", 3047, hook.url, hook);
+  now += 5 * 60_000;
+  pace.miss("novox/hq#7 at aaaaaaaa");
+  assert.equal(said.length, 0, "five minutes after the start is not yet silence");
+  assert.equal((pace.report() as any).healthy, true);
+  pace.refused("it is unsigned", "172.17.0.3");
+  assert.equal((pace.report() as any).healthy, false, "refused, and none verified");
+  now += 6 * 60_000;
+  pace.miss("the merge of novox/hq#7");
+  pace.miss("novox/hq#8 at bbbbbbbb");
+  assert.equal(said.filter((l) => /NOT HEARD: no delivery ever verified/.test(l)).length, 1, "said once");
+  let r = pace.report() as any;
+  assert.equal(r.healthy, false);
+  assert.equal(r.state, "unhealthy");
+  assert.match(r.why, /never delivered/);
+  assert.equal(r.polls.found_unsaid, 3);
+  assert.equal(r.refused.count, 1);
+  assert.equal(r.refused.last.reason, "it is unsigned");
+  assert.equal(r.hook.kind, "system");
+  assert.equal(r.receiver.port, 3047);
+  // The first delivery clears it.
+  now += 1000;
+  pace.heard("push", "d-1");
+  r = pace.report() as any;
+  assert.equal(r.healthy, true);
+  assert.equal(r.verified.count, 1);
+  assert.equal(r.verified.last.event, "push");
+  assert.equal(r.polls.every_seconds, 300);
+  assert.match(said.join("\n"), /heard again/);
+});
+
+test("a forge whose events stop after one delivery is said loudly once per silent spell, and refusals are unhealthy", async () => {
+  const { Pace } = await import("../webhook.ts");
+  let now = 1_000_000;
+  const said: string[] = [];
+  const pace = new Pace(() => now, (l) => said.push(l));
+  const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
+    did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
+  pace.listening("172.17.0.1", 3047, hook.url, hook);
+  now += 60_000;
+  pace.heard("push", "d-1");
+  // The hook is deleted by hand: nothing more comes, and the poll finds what no event said.
+  now += 11 * 60_000;
+  pace.miss("novox/hq#7 at aaaaaaaa");
+  assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 0, "one find alone may be the poll beating an event");
+  pace.miss("the merge of novox/hq#7");
+  pace.miss("novox/hq#8 at bbbbbbbb");
+  assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 1, "said once");
+  assert.match(said.find((l) => /NOT HEARD/.test(l)) ?? "", /since the last at/);
+  assert.equal((pace.report() as any).healthy, false);
+  // A delivery ends the spell; the next spell is said again.
+  now += 1000;
+  pace.heard("push", "d-2");
+  assert.equal((pace.report() as any).healthy, true);
+  now += 11 * 60_000;
+  pace.miss("novox/hq#9 at cccccccc");
+  pace.miss("novox/hq#10 at dddddddd");
+  assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 2, "a second silent spell is said again");
+  // Every delivery refused after a verified one (a secret that no longer matches) is unhealthy at once.
+  now += 1000;
+  pace.heard("push", "d-3");
+  now += 1000;
+  pace.refused("its signature does not verify", "172.17.0.3");
+  pace.refused("its signature does not verify", "172.17.0.3");
+  const r = pace.report() as any;
+  assert.equal(r.healthy, false);
+  assert.equal(r.refused.since_verified, 2);
+  assert.match(r.why, /2 deliveries refused since the last verified one.*does not verify/);
+  now += 1000;
+  pace.heard("push", "d-4");
+  assert.equal((pace.report() as any).healthy, true, "a verified delivery clears it");
+});
+
+test("without a receiver by configuration the polls alone watch, healthy; a receiver that failed is unhealthy", async () => {
+  const { Pace } = await import("../webhook.ts");
+  let now = 1_000_000;
+  const said: string[] = [];
+  const quiet = new Pace(() => now, (l) => said.push(l));
+  quiet.unheard("MESH_GITEA_HOOK_URL unset", false);
+  now += 20 * 60_000;
+  quiet.miss("novox/hq#7 at aaaaaaaa");
+  assert.equal(said.length, 0, "no deliveries were expected");
+  assert.equal((quiet.report() as any).healthy, true);
+  const failed = new Pace(() => now, () => {});
+  failed.unheard("the forge's system webhook could not be made: 403");
+  const r = failed.report() as any;
+  assert.equal(r.healthy, false);
+  assert.match(r.why, /403/);
 });
 
 test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => {

modules/gitea/webhook.ts

@@ -140,8 +140,11 @@ export interface ReceiverOptions {
   publish: (event: ForgeEvent) => Promise<void>;
   /** Acts on it at once, beside publishing it (the announcer). */
   act?: (event: ForgeEvent) => Promise<void>;
-  /** Told of every verified delivery, so the polls know events are heard. */
-  heard?: () => void;
+  /** Told of every verified delivery — its kind (X-Gitea-Event) and its delivery id — so the polls know events are
+   *  heard, and the module can say when it last heard one. */
+  heard?: (kind: string, delivery: string) => void;
+  /** Told of every refused delivery, with the reason and where it came from. */
+  refused?: (reason: string, from: string) => void;
   log?: (line: string) => void;
 }
 
@@ -158,6 +161,10 @@ export const REMEMBERED = 2000;
  */
 export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void> } {
   const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`));
+  const refuse = (from: string, reason: string): void => {
+    log(`refused a forge event from ${from}: ${reason}`);
+    opts.refused?.(reason, from);
+  };
   const seen = new Set<string>();
   let queue: Promise<void> = Promise.resolve();
   const server = createServer((req, res) => {
@@ -168,14 +175,14 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
     const from = req.socket.remoteAddress ?? "?";
     const signature = String(req.headers["x-gitea-signature"] ?? "").trim();
     if (!signature) {
-      log(`refused a forge event from ${from}: it is unsigned`);
+      refuse(from, "it is unsigned");
       res.writeHead(401, { Connection: "close" });
       res.end("unsigned\n");
       req.destroy();
       return;
     }
     if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) {
-      log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`);
+      refuse(from, `its body is over ${MAX_BODY} bytes`);
       res.writeHead(413, { Connection: "close" });
       res.end("too large\n");
       req.destroy();
@@ -189,7 +196,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
       if (size > MAX_BODY) {
         if (!over) {
           over = true;
-          log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`);
+          refuse(from, `its body is over ${MAX_BODY} bytes`);
           res.writeHead(413, { Connection: "close" });
           res.end("too large\n");
           req.destroy();
@@ -200,7 +207,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
       if (over) return;
       const body = Buffer.concat(chunks);
       if (!verified(opts.secret, body, signature)) {
-        log(`refused a forge event from ${from}: its signature does not verify`);
+        refuse(from, "its signature does not verify");
         res.statusCode = 401;
         return res.end("signature does not verify\n");
       }
@@ -208,11 +215,13 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
       try {
         payload = JSON.parse(body.toString("utf8"));
       } catch {
+        refuse(from, "it is signed but not JSON");
         res.statusCode = 400;
         return res.end("not JSON\n");
       }
-      opts.heard?.();
       const delivery = String(req.headers["x-gitea-delivery"] ?? "");
+      const kind = String(req.headers["x-gitea-event"] ?? "other");
+      opts.heard?.(kind, delivery);
       if (delivery && seen.has(delivery)) {
         res.statusCode = 200;
         return res.end("already heard\n");
@@ -221,7 +230,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
         seen.add(delivery);
         if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string);
       }
-      const event = eventOf(String(req.headers["x-gitea-event"] ?? "other"), payload);
+      const event = eventOf(kind, payload);
       res.statusCode = 200;
       res.end("queued\n");
       queue = queue.then(async () => {
@@ -242,27 +251,113 @@ export interface HookAdmin {
   call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>;
 }
 
+/** The forge's one system webhook as ensured at start: read back from the forge after it was made. */
+export interface HookState {
+  /** Always "system": a hook the forge's listing of system webhooks names, so it fires for every repository. */
+  kind: "system";
+  id: number;
+  url: string;
+  events: string[];
+  active: boolean;
+  content_type?: string;
+  /** Made where none was, or made in place of the ones there (system, default, or a repository's) at the receiver's url. */
+  did: "created" | "replaced";
+  /** The hooks at the receiver's url it took the place of, by kind: a repository's is a copy of a default webhook, which
+   *  would have sent that repository's events a second time. */
+  removed: { system: number; default: number; repository: number; repositories: string[] };
+}
+
+/** How long without a verified delivery — since the start, or since the last one — a forge is said silent once the poll
+ *  finds what no event said. */
+export const SILENT_AFTER = 10 * 60_000;
+
 /**
- * **How fast the polls look** (issue 729): every five minutes only while the forge's events are heard — a verified
- * delivery within the last ten minutes, and none missed since. Until the first delivery, after ten minutes without one,
- * or once the poll found something no event said, every thirty seconds.
+ * **How fast the polls look, and whether the forge's events are heard** (issue 729): every five minutes only while the
+ * forge's events are heard — a verified delivery within the last ten minutes, and none missed since. Until the first
+ * delivery, after ten minutes without one, or once the poll found something no event said, every thirty seconds.
+ *
+ * It also keeps what `gitea_events` says: where the receiver listens, the hook as ensured, the verified and refused
+ * deliveries, and the polls' finds no event said. **A forge whose events stop is said** (issue 729): once the receiver
+ * listens and the hook is ensured, when nothing verified for ten minutes — since the start, or since the last delivery —
+ * and the poll has found at least two heads or merges no event said in that quiet (one could be the poll beating an
+ * event by a second), it is logged loudly once per silent spell, and the module's events are said unhealthy until a
+ * delivery verifies. Deliveries refused since the last verified one are said unhealthy too.
  */
 export class Pace {
   private last = 0;
   private missed = 0;
+  private readonly started: number;
   private readonly now: () => number;
   private readonly log: (l: string) => void;
+  private where: { host: string; port: number; url: string } | null = null;
+  private hook: HookState | null = null;
+  private notHeard: string | null = null;
+  private notHeardFails = false;
+  private verifiedCount = 0;
+  private lastVerified: { at: number; event: string; delivery: string } | null = null;
+  private refusedCount = 0;
+  private refusedSpell = 0;
+  private lastRefused: { at: number; reason: string; from: string } | null = null;
+  private missCount = 0;
+  private lastMiss: { at: number; what: string } | null = null;
+  private silentSince = 0;
+  private quietMisses = 0;
   constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) {
     this.now = now;
     this.log = log;
+    this.started = now();
   }
-  heard(): void {
+  /** The receiver listens, and the hook at it was ensured: from now on, deliveries are expected. */
+  listening(host: string, port: number, url: string, hook: HookState): void {
+    this.where = { host, port, url };
+    this.hook = hook;
+    this.notHeard = null;
+  }
+  /** No events are listened for, and why: by configuration (`fails` false), or because the receiver could not listen
+   *  or the hook could not be ensured (`fails` true, said unhealthy). The polls alone watch. */
+  unheard(why: string, fails = true): void {
+    this.notHeard = why;
+    this.notHeardFails = fails;
+  }
+  heard(event = "", delivery = ""): void {
     this.last = this.now();
+    this.verifiedCount++;
+    this.lastVerified = { at: this.last, event, delivery };
+    this.refusedSpell = 0;
+    this.quietMisses = 0;
+    if (this.silentSince) this.log(`the forge's events are heard again: ${event || "a delivery"} verified`);
+    this.silentSince = 0;
+  }
+  refused(reason: string, from: string): void {
+    this.refusedCount++;
+    this.refusedSpell++;
+    this.lastRefused = { at: this.now(), reason, from };
   }
   /** The poll found a head or merge no forge event announced. */
   miss(what: string): void {
+    const t = this.now();
     if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`);
-    this.missed = this.now();
+    this.missed = t;
+    this.missCount++;
+    this.lastMiss = { at: t, what };
+    // Silent: nothing verified for SILENT_AFTER — never since the start, or not since the last delivery (a hook deleted by
+    // hand, every delivery refused) — while the poll finds what no event said. Said once per silent spell.
+    const quietSince = this.last || this.started;
+    this.quietMisses++;
+    if (this.expecting() && t - quietSince >= SILENT_AFTER && this.quietMisses >= 2 && !this.silentSince) {
+      this.silentSince = t;
+      const minutes = Math.round((t - quietSince) / 60_000);
+      const refusedSince = this.refusedSinceHeard();
+      this.log(`THE FORGE'S EVENTS ARE NOT HEARD: ${this.last ? `no delivery verified since the last at ` +
+        `${new Date(this.last).toISOString()}` : "no delivery ever verified since the start"}, ${minutes} min, from the ` +
+        `system webhook at ${this.hook?.url}${refusedSince ? `; ${refusedSince} refused since, the last because ` +
+        `${this.lastRefused?.reason}` : ""}; the poll found ${what} — the polls alone watch the forge, every 30 s ` +
+        `(novox/hq issue 729)`);
+    }
+  }
+  /** Whether a delivery was refused after the last verified one (or with none verified). */
+  private refusedSinceHeard(): number {
+    return this.lastRefused && this.lastRefused.at >= this.last ? this.refusedSpell : 0;
   }
   hearing(): boolean {
     const t = this.now();
@@ -271,36 +366,146 @@ export class Pace {
   every(): number {
     return this.hearing() ? 5 * 60_000 : 30_000;
   }
+  private expecting(): boolean {
+    return this.where !== null && this.hook !== null;
+  }
+  /** What `gitea_events` answers: healthy, unhealthy or waits (ADR 0283's shape), with every fact behind it. */
+  report(): Record<string, unknown> {
+    const t = this.now();
+    const iso = (ms: number): string => new Date(ms).toISOString();
+    let state: "healthy" | "unhealthy" | "waits" = "healthy";
+    let why: string;
+    const refusedSince = this.refusedSinceHeard();
+    if (this.silentSince) {
+      state = "unhealthy";
+      why = `${this.last ? `no delivery verified since the last at ${iso(this.last)}` : `the forge has never delivered an ` +
+        `event since the start at ${iso(this.started)}`}, while the poll found ${this.missCount} head(s) or merge(s) no ` +
+        `event said, the last ${this.lastMiss?.what} at ${iso(this.lastMiss?.at ?? t)}`;
+    } else if (refusedSince) {
+      state = "unhealthy";
+      why = `${refusedSince} deliveries refused ${this.last ? `since the last verified one at ${iso(this.last)}` :
+        "and none verified"}, the last at ${iso(this.lastRefused?.at ?? t)} because ${this.lastRefused?.reason}`;
+    } else if (!this.expecting()) {
+      why = this.notHeard ? `no events are listened for, so the polls alone watch the forge: ${this.notHeard}` :
+        "the receiver is not listening yet";
+      if (this.notHeard && this.notHeardFails) state = "unhealthy";
+    } else if (this.hearing()) {
+      why = `the forge's events are heard: ${this.verifiedCount} verified, the last at ${iso(this.last)}`;
+    } else if (this.last === 0) {
+      why = `no delivery heard yet; the forge is said silent if the poll finds what no event said ${SILENT_AFTER / 60_000} min after the start`;
+    } else {
+      why = `the last delivery was at ${iso(this.last)}; the polls look every 30 s until the next`;
+    }
+    return {
+      healthy: state !== "unhealthy", state, why,
+      started: iso(this.started),
+      receiver: this.where ? { listening: true, ...this.where } : { listening: false, why: this.notHeard ?? "not started" },
+      hook: this.hook,
+      verified: { count: this.verifiedCount, last: this.lastVerified && { ...this.lastVerified, at: iso(this.lastVerified.at) } },
+      refused: { count: this.refusedCount, since_verified: refusedSince, last: this.lastRefused && { ...this.lastRefused, at: iso(this.lastRefused.at) } },
+      polls: { every_seconds: this.every() / 1000, hearing: this.hearing(), found_unsaid: this.missCount,
+        last_found: this.lastMiss && { ...this.lastMiss, at: iso(this.lastMiss.at) } },
+      silent: this.silentSince ? { since: iso(this.silentSince) } : null,
+    };
+  }
 }
 
 /** Every event family the forge (1.27) can send a webhook. */
 export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request",
   "pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"];
 
-/**
- * **One system webhook, for every event, at the receiver** (issue 729): created when none points there, corrected
- * when one does but differs (events, active, content type), and its secret set again each start — the forge never
- * hands a secret back, so a corrected secret cannot be told from a kept one. Answers what it did.
- */
-export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<"created" | "corrected"> {
-  // POST /admin/hooks makes a system webhook, and its options take no flag for it (the forge's 1.27.3 API schema). The
-  // forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it.
-  const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*",
-    config: { url, content_type: "json", secret } };
-  const listed = await admin.call("GET", "/admin/hooks?type=system&limit=50");
-  if (listed.status !== 200) throw new Error(`the forge's system webhooks could not be listed: ${listed.status}`);
-  const hooks: any[] = Array.isArray(listed.body) ? listed.body : [];
-  const ours = hooks.filter((h) => h?.config?.url === url);
-  if (ours.length === 0) {
-    const made = await admin.call("POST", "/admin/hooks", want);
-    if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
-    return "created";
+async function hooksAt(admin: HookAdmin, type: "system" | "default", url: string): Promise<any[]> {
+  const listed = await admin.call("GET", `/admin/hooks?type=${type}&limit=50`);
+  if (listed.status !== 200) throw new Error(`the forge's ${type} webhooks could not be listed: ${listed.status}`);
+  return (Array.isArray(listed.body) ? listed.body : []).filter((h) => h?.config?.url === url);
+}
+
+/** Every page of a listing that answers an array (or `{ data: [] }`, the repository search), PAGE at a time. */
+const PAGE = 50;
+async function everyPage(admin: HookAdmin, path: string, what: string): Promise<any[]> {
+  const all: any[] = [];
+  for (let page = 1; ; page++) {
+    const r = await admin.call("GET", `${path}${path.includes("?") ? "&" : "?"}page=${page}&limit=${PAGE}`);
+    if (r.status !== 200) throw new Error(`${what} could not be listed: ${r.status}`);
+    const items: any[] = Array.isArray(r.body) ? r.body : Array.isArray(r.body?.data) ? r.body.data : [];
+    all.push(...items);
+    if (items.length < PAGE) return all;
   }
-  for (const extra of ours.slice(1)) await admin.call("DELETE", `/admin/hooks/${extra.id}`);
-  const fixed = await admin.call("PATCH", `/admin/hooks/${ours[0].id}`, { active: true, events: HOOK_EVENTS, branch_filter: "*",
-    config: { url, content_type: "json", secret } });
-  if (fixed.status !== 200) throw new Error(`the forge's system webhook could not be corrected: ${fixed.status}`);
-  return "corrected";
+}
+
+/** Every repository's own webhooks at the receiver's url: copies of a default webhook made at it. */
+async function repoHooksAt(admin: HookAdmin, url: string): Promise<{ repo: string; id: number }[]> {
+  const found: { repo: string; id: number }[] = [];
+  for (const repo of await everyPage(admin, "/repos/search?sort=id", "the forge's repositories")) {
+    const full = String(repo?.full_name ?? "");
+    if (!full.includes("/")) continue;
+    const hooks = await everyPage(admin, `/repos/${full}/hooks`, `${full}'s webhooks`);
+    for (const h of hooks) if (h?.config?.url === url) found.push({ repo: full, id: Number(h.id) });
+  }
+  return found;
+}
+
+/**
+ * **One system webhook, for every event, at the receiver** (issue 729), made each start and read back.
+ *
+ * The forge (1.27.3, routers/api/v1/utils/hook.go addHook) makes a *default* webhook from POST /admin/hooks unless
+ * `config.is_system_webhook` is "true": a default webhook fires for no repository and is only copied into repositories
+ * made after it, and the listing of system webhooks does not name it — so without the flag every start made another
+ * one and no event came. Its edit (editHook) never changes a webhook's secret nor its kind, so a hook already there is
+ * not corrected but replaced: the new system webhook is made first, with the secret kept now, so no event falls in a
+ * gap; then every other hook at the receiver's url is deleted — system, default, and each repository's copy of a
+ * default one, which would send that repository's events twice. The forge's listing of system webhooks must then name
+ * exactly one at the url, or this fails, saying so.
+ */
+export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<HookState> {
+  // The forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it.
+  const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*",
+    config: { url, content_type: "json", secret, is_system_webhook: "true" } };
+  const before = new Set((await hooksAt(admin, "system", url)).map((h) => Number(h.id)));
+  const made = await admin.call("POST", "/admin/hooks", want);
+  if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
+  const fresh = (await hooksAt(admin, "system", url)).filter((h) => !before.has(Number(h.id)));
+  if (fresh.length !== 1) {
+    throw new Error(`the forge names ${fresh.length} new system webhooks at ${url} after one was made, not one: its events ` +
+      `are not known to be sent`);
+  }
+  const ours = fresh[0];
+  const defaults = await hooksAt(admin, "default", url);
+  const gone = async (path: string): Promise<void> => {
+    const r = await admin.call("DELETE", path);
+    if (r.status !== 204 && r.status !== 200 && r.status !== 404) throw new Error(`the forge's webhook ${path} could not be deleted: ${r.status}`);
+  };
+  for (const id of before) await gone(`/admin/hooks/${id}`);
+  for (const h of defaults) await gone(`/admin/hooks/${h.id}`);
+  const copies = await repoHooksAt(admin, url);
+  for (const c of copies) await gone(`/repos/${c.repo}/hooks/${c.id}`);
+  const kept = await hooksAt(admin, "system", url);
+  if (kept.length !== 1 || Number(kept[0].id) !== Number(ours.id)) {
+    throw new Error(`the forge names ${kept.length} system webhooks at ${url} once the others were deleted, not the one ` +
+      `made (${ours.id})`);
+  }
+  const removed = { system: before.size, default: defaults.length, repository: copies.length,
+    repositories: [...new Set(copies.map((c) => c.repo))] };
+  return { kind: "system", id: Number(ours.id), url, events: Array.isArray(ours.events) ? ours.events : [],
+    active: !!ours.active, content_type: ours.config?.content_type,
+    did: removed.system + removed.default + removed.repository > 0 ? "replaced" : "created", removed };
+}
+
+/** The `gitea_events` tool: what the module knows of the forge's events, from its Pace. */
+export function eventsTool(pace: Pace): { name: string; description: string; input: Record<string, unknown>;
+  run: (args: Readonly<Record<string, unknown>>) => Promise<unknown> } {
+  return {
+    name: "gitea_events",
+    description: "Whether the forge's events reach the mesh (novox/hq issue 729): where the receiver listens, the " +
+      "system webhook as ensured at the start (its id, url, events, kind, and what it replaced), how many deliveries " +
+      "verified and the last one's time and event, how many were refused and the last reason, how often the polls look " +
+      "and whether events are heard, and how many heads or merges the poll found that no event said. Unhealthy when the " +
+      "forge has delivered nothing that verified for ten minutes (since the start or since the last delivery) while the " +
+      "poll found what no event said, when deliveries were refused since the last verified one, or when the receiver " +
+      "could not listen or the webhook could not be made.",
+    input: {},
+    run: async () => pace.report(),
+  };
 }
 
 /**