Drop every mesh-tools token by id when the forge holds two, and stop re-minting on every call (hq issue 739)

On 2026-10-14 from 22:28 local the forge held two tokens named mesh-tools.
gitea refuses a delete by such a name with 422 "multiple matches", so every
gitea tool call re-entered the mint, made two basic-auth calls that hash the
admin password, and failed: several times a minute, on a control-node that was
already overloaded, and mesh-delivery could read neither open pulls nor views.

The watcher and the tools run as separate bundles, so the in-process sharing
of one token source does not cover them; two mints racing leave two tokens by
the name, because the forge's check that a name is unused is not atomic.

A 422 on the delete by name now lists the account's tokens and drops each one
carrying the name by its id, then mints. A failed mint is not asked of the
forge again for 30 s: the same error is answered without a call.

modules/gitea/test/token.test.ts

@@ -15,7 +15,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 
 import { collectTools } from "@novox/mesh-sdk/tools";
-import { AdminRefused, MintedToken, TOKEN_SCOPES } from "../dist/token.js";
+import { AdminRefused, MINT_RETRY_AFTER_MS, MintedToken, TOKEN_SCOPES } from "../dist/token.js";
 import { GiteaClient } from "../dist/client.js";
 import { getGiteaTools } from "../dist/tools/index.js";
 
@@ -29,6 +29,9 @@ interface Forge {
   mints: number;
   lastScopes: string[] | null;
   tokens: Map<string, string>;
+  /** Tokens beyond the one per name `tokens` holds, by id: what two mints racing leave (novox/hq issue 739). */
+  twins: Map<number, { name: string; sha1: string }>;
+  tokenCalls: number;
   scopesOf: Map<string, string[]>;
   admins: Map<string, string>;
   pullState: string;
@@ -42,6 +45,9 @@ function fakeForge(): Promise<Forge> {
     mints: 0,
     lastScopes: null as string[] | null,
     tokens: new Map<string, string>(), // name -> value
+    twins: new Map<number, { name: string; sha1: string }>(),
+    idOf: new Map<string, number>(), // value -> id
+    tokenCalls: 0,
     scopesOf: new Map<string, string[]>(), // value -> scopes, so a route can enforce them like gitea does
     admins: new Map([[ADMIN, PASSWORD]]),
     pullState: "open",
@@ -73,21 +79,41 @@ function fakeForge(): Promise<Forge> {
     const tokens = url.pathname.match(/^\/api\/v1\/users\/([^/]+)\/tokens(?:\/([^/]+))?$/);
     if (tokens) {
       const user = basic(req);
+      forge.tokenCalls++;
       if (user === null || user !== decodeURIComponent(tokens[1])) return json(res, 401, { message: "auth required" });
+      const named = (name: string): number =>
+        (forge.tokens.has(name) ? 1 : 0) + [...forge.twins.values()].filter((t) => t.name === name).length;
       if (req.method === "POST") {
         const { name, scopes } = await body(req);
-        if (forge.tokens.has(name)) return json(res, 400, { message: "token name has already been used" });
+        if (named(name) > 0) return json(res, 400, { message: "token name has already been used" });
         forge.mints++;
         forge.lastScopes = scopes;
         const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`;
         forge.tokens.set(name, sha1);
+        forge.idOf.set(sha1, forge.mints);
         forge.scopesOf.set(sha1, scopes);
         return json(res, 201, { id: forge.mints, name, sha1, scopes, token_last_eight: sha1.slice(-8) });
       }
+      if (req.method === "GET" && !tokens[2]) {
+        const page = Number(url.searchParams.get("page") ?? "1");
+        const all = [...[...forge.tokens].map(([name, v]) => ({ id: forge.idOf.get(v) ?? 0, name })),
+          ...[...forge.twins].map(([id, t]) => ({ id, name: t.name }))];
+        return json(res, 200, page === 1 ? all : []);
+      }
       if (req.method === "DELETE" && tokens[2]) {
-        const name = decodeURIComponent(tokens[2]);
-        if (!forge.tokens.has(name)) return json(res, 404, { message: "token not found" });
-        forge.tokens.delete(name);
+        const key = decodeURIComponent(tokens[2]);
+        // gitea: "identified by ID and if not available by name"
+        if (/^\d+$/.test(key)) {
+          const id = Number(key);
+          if (forge.twins.delete(id)) return json(res, 204, null);
+          const hit = [...forge.tokens].find(([, v]) => forge.idOf.get(v) === id);
+          if (!hit) return json(res, 404, { message: "token not found" });
+          forge.tokens.delete(hit[0]);
+          return json(res, 204, null);
+        }
+        if (named(key) > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` });
+        if (!forge.tokens.has(key)) return json(res, 404, { message: "token not found" });
+        forge.tokens.delete(key);
         return json(res, 204, null);
       }
       return json(res, 405, { message: "method not allowed" });
@@ -180,7 +206,9 @@ function fakeForge(): Promise<Forge> {
         get pullState() { return forge.pullState; },
         get pullTitle() { return forge.pullTitle; },
         get branchDeleted() { return forge.branchDeleted; },
+        get tokenCalls() { return forge.tokenCalls; },
         tokens: forge.tokens,
+        twins: forge.twins,
         scopesOf: forge.scopesOf,
         admins: forge.admins,
         close: () => new Promise((r) => server.close(() => r())),
@@ -209,8 +237,9 @@ async function delivered(forge: Forge): Promise<{ env: NodeJS.ProcessEnv; file:
 }
 
 /** A client as a fresh process would build it: a new source over the kept file, its log captured. */
-function minted(env: NodeJS.ProcessEnv, logs: string[]): GiteaClient {
+function minted(env: NodeJS.ProcessEnv, logs: string[], now?: () => number): GiteaClient {
   const source = new MintedToken({
+    now,
     url: env.MESH_GITEA_URL!,
     admin: env.MESH_GITEA_ADMIN_USER!,
     passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!,
@@ -325,6 +354,44 @@ test("the kept file is gone but the forge still holds a token by that name: repl
   assert.ok(logs.some((l) => l.includes('already holds a token named "mesh-tools"')), logs.join("\n"));
 });
 
+test("two tokens carry the name (two mints raced): each is dropped by its id and one is minted, not a 422 for good (issue 739)", async () => {
+  const { env, logs } = await delivered(forge);
+  await minted(env, logs).listRepos();
+  // What two processes minting at once leave: a second token by the same name, whose value nobody kept.
+  // Two tokens by the name, neither of them the one kept.
+  forge.tokens.clear();
+  forge.twins.set(9001, { name: "mesh-tools", sha1: "raced-first" });
+  forge.twins.set(9002, { name: "mesh-tools", sha1: "raced-second" });
+
+  const repos = await minted(env, logs).listRepos();
+
+  assert.equal(repos.length, 1);
+  assert.equal(forge.twins.size, 0);
+  assert.equal([...forge.tokens.keys()].filter((n) => n === "mesh-tools").length, 1);
+  assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n"));
+});
+
+test("a mint that failed is not asked of the forge again within the retry window: the same error, no basic-auth call (issue 739)", async () => {
+  const { env, logs } = await delivered(forge);
+  forge.admins.delete(ADMIN);
+  try {
+    let clock = 5_000_000;
+    const client = minted(env, logs, () => clock);
+    await assert.rejects(client.listRepos(), AdminRefused);
+    const calls = forge.tokenCalls;
+
+    await assert.rejects(client.listRepos(), AdminRefused);
+    await assert.rejects(client.listRepos(), AdminRefused);
+    assert.equal(forge.tokenCalls, calls);
+
+    clock += MINT_RETRY_AFTER_MS;
+    await assert.rejects(client.listRepos(), AdminRefused);
+    assert.equal(forge.tokenCalls, calls + 1);
+  } finally {
+    forge.admins.set(ADMIN, PASSWORD);
+  }
+});
+
 test("concurrent first calls share one mint", async () => {
   const { env, logs } = await delivered(forge);
   const client = minted(env, logs);
@@ -339,7 +406,8 @@ test("the admin account is refused: said plainly, nothing kept, and the next cal
   const { env, file, logs } = await delivered(forge);
   forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there
   try {
-    const client = minted(env, logs);
+    let clock = 1_000_000;
+    const client = minted(env, logs, () => clock);
     const before = forge.mints;
 
     await assert.rejects(client.listRepos(), (err: unknown) => {
@@ -354,8 +422,9 @@ test("the admin account is refused: said plainly, nothing kept, and the next cal
     assert.equal(forge.mints, before);
     await assert.rejects(stat(file), /ENOENT/);
 
-    // The account appears (the operator created it): the very next call mints and works.
+    // The account appears (the operator created it): the next call after the retry window mints and works.
     forge.admins.set(ADMIN, PASSWORD);
+    clock += MINT_RETRY_AFTER_MS;
     assert.equal((await client.listRepos()).length, 1);
     assert.equal(forge.mints, before + 1);
   } finally {