modules/gitea/test/token.test.ts

@@ -10,7 +10,7 @@
 import { test, after } from "node:test";
 import assert from "node:assert/strict";
 import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
-import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
@@ -29,6 +29,12 @@ interface Forge {
   mints: number;
   lastScopes: string[] | null;
   tokens: Map<string, string>;
+  /** Tokens beside `tokens` that carry a name already held there: what two mints at once leave behind. */
+  twins: { id: number; name: string; sha1: string }[];
+  /** How each DELETE named its token: "id" or "name". */
+  deletedBy: string[];
+  /** Plant a token by name, beside any other of the same name, as a raced mint would. */
+  plant(name: string): void;
   scopesOf: Map<string, string[]>;
   admins: Map<string, string>;
   pullState: string;
@@ -42,6 +48,10 @@ function fakeForge(): Promise<Forge> {
     mints: 0,
     lastScopes: null as string[] | null,
     tokens: new Map<string, string>(), // name -> value
+    twins: [] as { id: number; name: string; sha1: string }[],
+    deletedBy: [] as string[],
+    ids: new Map<string, number>(), // value -> id, as gitea numbers its tokens
+    nextId: 0,
     scopesOf: new Map<string, string[]>(), // value -> scopes, so a route can enforce them like gitea does
     admins: new Map([[ADMIN, PASSWORD]]),
     pullState: "open",
@@ -82,12 +92,35 @@ function fakeForge(): Promise<Forge> {
         const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`;
         forge.tokens.set(name, sha1);
         forge.scopesOf.set(sha1, scopes);
-        return json(res, 201, { id: forge.mints, name, sha1, scopes, token_last_eight: sha1.slice(-8) });
+        forge.ids.set(sha1, ++forge.nextId);
+        return json(res, 201, { id: forge.ids.get(sha1), name, sha1, scopes, token_last_eight: sha1.slice(-8) });
+      }
+      // gitea's list: every token of the account, a page at a time (at most three to a page here, so a
+      // client that reads only the first page misses some), the values never shown.
+      const listed = (): { id: number; name: string; sha1: string }[] => [
+        ...[...forge.tokens].map(([name, sha1]) => ({ id: forge.ids.get(sha1) ?? 0, name, sha1 })),
+        ...forge.twins,
+      ];
+      if (req.method === "GET" && !tokens[2]) {
+        const page = Math.max(1, Number(url.searchParams.get("page") ?? "1"));
+        const limit = Math.min(3, Math.max(1, Number(url.searchParams.get("limit") ?? "3")));
+        const all = listed();
+        res.setHeader("X-Total-Count", String(all.length));
+        return json(res, 200, all.slice((page - 1) * limit, page * limit)
+          .map((t) => ({ id: t.id, name: t.name, sha1: "", token_last_eight: t.sha1.slice(-8), scopes: [] })));
       }
       if (req.method === "DELETE" && tokens[2]) {
-        const name = decodeURIComponent(tokens[2]);
-        if (!forge.tokens.has(name)) return json(res, 404, { message: "token not found" });
-        forge.tokens.delete(name);
+        // gitea: "identified by ID and if not available by name" — and a name held twice is refused.
+        const key = decodeURIComponent(tokens[2]);
+        const byId = /^\d+$/.test(key);
+        const matches = listed().filter((t) => (byId ? String(t.id) === key : t.name === key));
+        if (matches.length === 0) return json(res, 404, { message: "token not found" });
+        if (matches.length > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` });
+        const t = matches[0];
+        forge.deletedBy.push(byId ? "id" : "name");
+        const twin = forge.twins.findIndex((w) => w.id === t.id);
+        if (twin >= 0) forge.twins.splice(twin, 1);
+        else forge.tokens.delete(t.name);
         return json(res, 204, null);
       }
       return json(res, 405, { message: "method not allowed" });
@@ -181,6 +214,14 @@ function fakeForge(): Promise<Forge> {
         get pullTitle() { return forge.pullTitle; },
         get branchDeleted() { return forge.branchDeleted; },
         tokens: forge.tokens,
+        twins: forge.twins,
+        deletedBy: forge.deletedBy,
+        plant: (name: string) => {
+          const sha1 = `planted-${++forge.nextId}`;
+          forge.ids.set(sha1, forge.nextId);
+          if (forge.tokens.has(name)) forge.twins.push({ id: forge.nextId, name, sha1 });
+          else forge.tokens.set(name, sha1);
+        },
         scopesOf: forge.scopesOf,
         admins: forge.admins,
         close: () => new Promise((r) => server.close(() => r())),
@@ -335,6 +376,56 @@ test("concurrent first calls share one mint", async () => {
   assert.equal(forge.mints, before + 1);
 });
 
+test("two tokens hold the name (two mints raced): both are deleted by their id, and one fresh token is minted", async () => {
+  const { env, file, logs } = await delivered(forge);
+  forge.tokens.clear();
+  forge.twins.length = 0;
+  for (const other of ["laptop", "backup", "ci"]) forge.plant(other); // the account's other tokens: kept
+  forge.plant("mesh-tools");
+  forge.plant("mesh-tools"); // gitea refuses to delete either by name from here on
+  forge.deletedBy.length = 0;
+  const before = forge.mints;
+
+  const repos = await minted(env, logs).listRepos();
+
+  assert.equal(repos.length, 1);
+  assert.equal(forge.mints, before + 1);
+  assert.deepEqual(forge.deletedBy, ["id", "id"]);
+  assert.equal(forge.twins.length, 0);
+  assert.deepEqual([...forge.tokens.keys()].sort(), ["backup", "ci", "laptop", "mesh-tools"]);
+  assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n");
+  assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n"));
+});
+
+test("two processes on one kept file, both without a token, mint once: the second waits for the first and reuses it", async () => {
+  const { env, logs } = await delivered(forge);
+  forge.tokens.clear();
+  forge.twins.length = 0;
+  const before = forge.mints;
+  // Two sources over one state directory, as the tools bundle and the watcher are two processes.
+  const first = minted(env, logs);
+  const second = minted(env, logs);
+
+  await Promise.all([first.listRepos(), second.listRepos()]);
+
+  assert.equal(forge.mints, before + 1);
+  assert.equal(forge.twins.length, 0);
+  assert.ok(logs.some((l) => l.includes("another process minted it")), logs.join("\n"));
+});
+
+test("a lock left by a process that died mid-mint is taken over, not waited on for ever", async () => {
+  const { env, file, logs } = await delivered(forge);
+  forge.tokens.clear();
+  forge.twins.length = 0;
+  await mkdir(join(env.MESH_GITEA_STATE_DIR!), { recursive: true });
+  await writeFile(`${file}.lock`, `2147483646 ${Date.now()} gone\n`); // a pid no process holds
+  const before = forge.mints;
+
+  assert.equal((await minted(env, logs).listRepos()).length, 1);
+  assert.equal(forge.mints, before + 1);
+  await assert.rejects(stat(`${file}.lock`), /ENOENT/);
+});
+
 test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => {
   const { env, file, logs } = await delivered(forge);
   forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there

modules/gitea/token.ts

@@ -12,7 +12,15 @@
 //     hands a token's value out exactly once, so a token not kept is a token lost;
 //   - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not
 //     a fault: the forge's data was restored from a predecessor and the token the file names never
-//     existed there.
+//     existed there;
+//   - minted by one process at a time. The runtime starts each of the bundle's entrypoints as a process
+//     of its own (novox/hq ADR 0193): the watcher (index.js) and the tools (tools/index.js) are two
+//     processes over one kept file. Two that minted at once left the forge two tokens by one name, which
+//     gitea then refuses to delete by name — for ever, and every forge tool with it (the 2026-10-14
+//     outage). So a mint holds a lock file beside the kept token, and reads the kept file again once it
+//     holds it: what another process minted meanwhile is reused, not minted over;
+//   - replaced by id: an old token by this name is found in the account's list and deleted by its id,
+//     every one of them, so a name held twice is cleared rather than refused.
 //
 // An explicitly configured token still wins, and is never minted over: if it is rejected, that is
 // reported, not repaired — somebody chose it.
@@ -20,7 +28,8 @@
 // The token is never logged. Lines say that one was minted, reused or renewed, and where it is
 // kept; never what it is.
 
-import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
+import { randomBytes } from "node:crypto";
 import { dirname, join } from "node:path";
 
 /** The name the token carries in the forge's own list — one per mesh runtime, found by name. */
@@ -39,7 +48,7 @@ export const TOKEN_NAME = "mesh-tools";
  * `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the
  * operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token.
  * A token kept with a different list is minted again with this one at the next start (the scopes are kept
- * beside it), and the forge drops the old one by name — so the scope a kept token had does not outlive this.
+ * beside it), and the old one is deleted from the forge by its id — so the scope a kept token had does not outlive this.
  *
  * A token kept from before a scope was added lacks it: the forge answers such a call with
  * `403 token does not have at least one of required scope(s)`, and the client treats that like a
@@ -124,9 +133,9 @@ export class MintedToken implements TokenSource {
    */
   static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken {
     const opts = MintedToken.optionsFromEnv(url, env);
-    // One source per kept file in a process. The watcher and the tools entrypoint both build a
-    // client in the same runtime; two sources over one file would each renew on a 401 and drop the
-    // other's token by name, forever. Shared, a renewal is one renewal.
+    // One source per kept file in a process: two sources in one process would each renew on a 401.
+    // This covers one process only. The watcher and the tools are two processes (the runtime starts
+    // each entrypoint as its own), and those are kept apart by the mint's lock file (mint, below).
     const shared = MintedToken.shared.get(opts.file);
     if (shared) return shared;
     const source = new MintedToken(opts);
@@ -161,9 +170,9 @@ export class MintedToken implements TokenSource {
         this.log(`reusing the token kept at ${this.opts.file}`);
         return kept;
       }
-      if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them");
+      if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept);
     }
-    return this.mint("no token kept — minting one");
+    return this.mint("no token kept — minting one", null);
   }
 
   async renew(rejected: string): Promise<string> {
@@ -177,7 +186,7 @@ export class MintedToken implements TokenSource {
       return kept;
     }
     this.held = null;
-    return this.mint("the forge rejected the kept token — minting a fresh one");
+    return this.mint("the forge rejected the kept token — minting a fresh one", rejected);
   }
 
   /** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
@@ -185,17 +194,95 @@ export class MintedToken implements TokenSource {
     return status === 403 && /required scope/i.test(body);
   }
 
-  /** One mint at a time: concurrent first calls share it, rather than each minting its own. */
-  private mint(why: string): Promise<string> {
+  /**
+   * One mint at a time: concurrent first calls in this process share it, and a mint in another process
+   * on this machine is waited for (the lock file). Once the lock is held, the kept file is read again:
+   * a token there that is not `stale` (the one rejected, or kept with other scopes) and carries the
+   * scopes asked for was minted by the process that held the lock before — it is reused.
+   */
+  private mint(why: string, stale: string | null): Promise<string> {
     if (this.inflight === null) {
       this.log(why);
-      this.inflight = this.doMint().finally(() => {
+      this.inflight = this.locked(async () => {
+        const kept = this.read();
+        if (kept !== null && kept !== stale && this.keptScopes() === this.scopesLine()) {
+          this.held = kept;
+          this.log(`another process minted it meanwhile — reusing the token kept at ${this.opts.file}`);
+          return kept;
+        }
+        return this.doMint();
+      }).finally(() => {
         this.inflight = null;
       });
     }
     return this.inflight;
   }
 
+  /** How long a mint may hold the lock before another process takes it as left behind, and how long one waits. */
+  static readonly LOCK_STALE_MS = 120_000;
+  static readonly LOCK_WAIT_MS = 150_000;
+
+  /**
+   * Run `work` holding the lock file beside the kept token: created exclusively, holding this process's
+   * pid, the time and a nonce; removed after, only when it is still ours. A lock whose process is gone,
+   * or older than LOCK_STALE_MS, was left by a process that died mid-mint and is taken over — never
+   * waited on for ever, which would be the same lock-out by another road.
+   */
+  private async locked<T>(work: () => Promise<T>): Promise<T> {
+    const lock = `${this.opts.file}.lock`;
+    mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 });
+    const mine = `${process.pid} ${Date.now()} ${randomBytes(8).toString("hex")}\n`;
+    const deadline = Date.now() + MintedToken.LOCK_WAIT_MS;
+    for (;;) {
+      try {
+        const fd = openSync(lock, "wx", 0o600);
+        try { writeSync(fd, mine); } finally { closeSync(fd); }
+        break;
+      } catch (err) {
+        if ((err as NodeJS.ErrnoException).code !== "EEXIST") {
+          throw new Error(`cannot take the mint lock at ${lock}: ${(err as Error).message}`);
+        }
+      }
+      if (this.leftBehind(lock)) {
+        this.log(`the mint lock at ${lock} was left by a process that is gone — taking it over`);
+        try { unlinkSync(lock); } catch { /* another waiter took it over first */ }
+        continue;
+      }
+      if (Date.now() > deadline) {
+        throw new Error(`another process has held the mint lock at ${lock} for over ${MintedToken.LOCK_WAIT_MS / 1000}s — not minting beside it`);
+      }
+      await new Promise((r) => setTimeout(r, 50 + Math.floor(Math.random() * 50)));
+    }
+    try {
+      return await work();
+    } finally {
+      try {
+        if (readFileSync(lock, "utf8") === mine) unlinkSync(lock);
+      } catch { /* already gone */ }
+    }
+  }
+
+  /** Whether the lock's holder is gone: its pid answers no signal, or it is older than LOCK_STALE_MS. */
+  private leftBehind(lock: string): boolean {
+    let text: string;
+    let age: number;
+    try {
+      text = readFileSync(lock, "utf8");
+      age = Date.now() - statSync(lock).mtimeMs;
+    } catch {
+      return false; // released meanwhile: the next attempt takes it
+    }
+    if (age > MintedToken.LOCK_STALE_MS) return true;
+    const pid = Number(text.split(" ")[0]);
+    if (!Number.isInteger(pid) || pid <= 0) return age > 1_000; // half-written, or not ours: give it a moment
+    try {
+      process.kill(pid, 0);
+      return false;
+    } catch (err) {
+      return (err as NodeJS.ErrnoException).code === "ESRCH";
+    }
+  }
+
   private read(): string | null {
     try {
       const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, "");
@@ -258,11 +345,26 @@ export class MintedToken implements TokenSource {
     if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status);
     if (res.status === 400 || res.status === 422) {
       // The forge still holds a token by this name whose value we no longer have — the kept file
-      // went while the forge's data stayed. It is ours to replace: drop it by name and mint again.
-      this.log(`the forge already holds a token named "${this.name}" — replacing it`);
-      const dropped = await call("DELETE", `/${encodeURIComponent(this.name)}`);
-      if (dropped.status !== 204 && dropped.status !== 404) {
-        throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${dropped.status} ${detail(dropped.body)}`);
+      // went while the forge's data stayed, or two processes once minted at once and left two. It is
+      // ours to replace: find every one by this name in the account's list and delete each by its id
+      // (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint.
+      const held: number[] = [];
+      for (let page = 1; page <= 100; page++) {
+        const listed = await call("GET", `?page=${page}&limit=50`);
+        if (listed.status !== 200 || !Array.isArray(listed.body)) {
+          throw new Error(`Gitea GET /users/${this.opts.admin}/tokens: ${listed.status} ${detail(listed.body)}`);
+        }
+        if (listed.body.length === 0) break;
+        for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id);
+      }
+      this.log(held.length === 1
+        ? `the forge already holds a token named "${this.name}" — replacing it`
+        : `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`);
+      for (const id of held) {
+        const dropped = await call("DELETE", `/${id}`);
+        if (dropped.status !== 204 && dropped.status !== 404) {
+          throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`);
+        }
       }
       res = await call("POST", "", { name: this.name, scopes: this.scopes });
     }