diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index 750c8122..59cacb90 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -10,7 +10,7 @@ import { test, after } from "node:test"; import assert from "node:assert/strict"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -29,6 +29,12 @@ interface Forge { mints: number; lastScopes: string[] | null; tokens: Map; + /** Tokens beside `tokens` that carry a name already held there: what two mints at once leave behind. */ + twins: { id: number; name: string; sha1: string }[]; + /** How each DELETE named its token: "id" or "name". */ + deletedBy: string[]; + /** Plant a token by name, beside any other of the same name, as a raced mint would. */ + plant(name: string): void; scopesOf: Map; admins: Map; pullState: string; @@ -42,6 +48,10 @@ function fakeForge(): Promise { mints: 0, lastScopes: null as string[] | null, tokens: new Map(), // name -> value + twins: [] as { id: number; name: string; sha1: string }[], + deletedBy: [] as string[], + ids: new Map(), // value -> id, as gitea numbers its tokens + nextId: 0, scopesOf: new Map(), // value -> scopes, so a route can enforce them like gitea does admins: new Map([[ADMIN, PASSWORD]]), pullState: "open", @@ -82,12 +92,35 @@ function fakeForge(): Promise { const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`; forge.tokens.set(name, sha1); forge.scopesOf.set(sha1, scopes); - return json(res, 201, { id: forge.mints, name, sha1, scopes, token_last_eight: sha1.slice(-8) }); + forge.ids.set(sha1, ++forge.nextId); + return json(res, 201, { id: forge.ids.get(sha1), name, sha1, scopes, token_last_eight: sha1.slice(-8) }); + } + // gitea's list: every token of the account, a page at a time (at most three to a page here, so a + // client that reads only the first page misses some), the values never shown. + const listed = (): { id: number; name: string; sha1: string }[] => [ + ...[...forge.tokens].map(([name, sha1]) => ({ id: forge.ids.get(sha1) ?? 0, name, sha1 })), + ...forge.twins, + ]; + if (req.method === "GET" && !tokens[2]) { + const page = Math.max(1, Number(url.searchParams.get("page") ?? "1")); + const limit = Math.min(3, Math.max(1, Number(url.searchParams.get("limit") ?? "3"))); + const all = listed(); + res.setHeader("X-Total-Count", String(all.length)); + return json(res, 200, all.slice((page - 1) * limit, page * limit) + .map((t) => ({ id: t.id, name: t.name, sha1: "", token_last_eight: t.sha1.slice(-8), scopes: [] }))); } if (req.method === "DELETE" && tokens[2]) { - const name = decodeURIComponent(tokens[2]); - if (!forge.tokens.has(name)) return json(res, 404, { message: "token not found" }); - forge.tokens.delete(name); + // gitea: "identified by ID and if not available by name" — and a name held twice is refused. + const key = decodeURIComponent(tokens[2]); + const byId = /^\d+$/.test(key); + const matches = listed().filter((t) => (byId ? String(t.id) === key : t.name === key)); + if (matches.length === 0) return json(res, 404, { message: "token not found" }); + if (matches.length > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` }); + const t = matches[0]; + forge.deletedBy.push(byId ? "id" : "name"); + const twin = forge.twins.findIndex((w) => w.id === t.id); + if (twin >= 0) forge.twins.splice(twin, 1); + else forge.tokens.delete(t.name); return json(res, 204, null); } return json(res, 405, { message: "method not allowed" }); @@ -181,6 +214,14 @@ function fakeForge(): Promise { get pullTitle() { return forge.pullTitle; }, get branchDeleted() { return forge.branchDeleted; }, tokens: forge.tokens, + twins: forge.twins, + deletedBy: forge.deletedBy, + plant: (name: string) => { + const sha1 = `planted-${++forge.nextId}`; + forge.ids.set(sha1, forge.nextId); + if (forge.tokens.has(name)) forge.twins.push({ id: forge.nextId, name, sha1 }); + else forge.tokens.set(name, sha1); + }, scopesOf: forge.scopesOf, admins: forge.admins, close: () => new Promise((r) => server.close(() => r())), @@ -335,6 +376,56 @@ test("concurrent first calls share one mint", async () => { assert.equal(forge.mints, before + 1); }); +test("two tokens hold the name (two mints raced): both are deleted by their id, and one fresh token is minted", async () => { + const { env, file, logs } = await delivered(forge); + forge.tokens.clear(); + forge.twins.length = 0; + for (const other of ["laptop", "backup", "ci"]) forge.plant(other); // the account's other tokens: kept + forge.plant("mesh-tools"); + forge.plant("mesh-tools"); // gitea refuses to delete either by name from here on + forge.deletedBy.length = 0; + const before = forge.mints; + + const repos = await minted(env, logs).listRepos(); + + assert.equal(repos.length, 1); + assert.equal(forge.mints, before + 1); + assert.deepEqual(forge.deletedBy, ["id", "id"]); + assert.equal(forge.twins.length, 0); + assert.deepEqual([...forge.tokens.keys()].sort(), ["backup", "ci", "laptop", "mesh-tools"]); + assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n"); + assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n")); +}); + +test("two processes on one kept file, both without a token, mint once: the second waits for the first and reuses it", async () => { + const { env, logs } = await delivered(forge); + forge.tokens.clear(); + forge.twins.length = 0; + const before = forge.mints; + // Two sources over one state directory, as the tools bundle and the watcher are two processes. + const first = minted(env, logs); + const second = minted(env, logs); + + await Promise.all([first.listRepos(), second.listRepos()]); + + assert.equal(forge.mints, before + 1); + assert.equal(forge.twins.length, 0); + assert.ok(logs.some((l) => l.includes("another process minted it")), logs.join("\n")); +}); + +test("a lock left by a process that died mid-mint is taken over, not waited on for ever", async () => { + const { env, file, logs } = await delivered(forge); + forge.tokens.clear(); + forge.twins.length = 0; + await mkdir(join(env.MESH_GITEA_STATE_DIR!), { recursive: true }); + await writeFile(`${file}.lock`, `2147483646 ${Date.now()} gone\n`); // a pid no process holds + const before = forge.mints; + + assert.equal((await minted(env, logs).listRepos()).length, 1); + assert.equal(forge.mints, before + 1); + await assert.rejects(stat(`${file}.lock`), /ENOENT/); +}); + test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => { const { env, file, logs } = await delivered(forge); forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there diff --git a/modules/gitea/token.ts b/modules/gitea/token.ts index 3aac30b1..d34c303e 100644 --- a/modules/gitea/token.ts +++ b/modules/gitea/token.ts @@ -12,7 +12,15 @@ // hands a token's value out exactly once, so a token not kept is a token lost; // - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not // a fault: the forge's data was restored from a predecessor and the token the file names never -// existed there. +// existed there; +// - minted by one process at a time. The runtime starts each of the bundle's entrypoints as a process +// of its own (novox/hq ADR 0193): the watcher (index.js) and the tools (tools/index.js) are two +// processes over one kept file. Two that minted at once left the forge two tokens by one name, which +// gitea then refuses to delete by name — for ever, and every forge tool with it (the 2026-10-14 +// outage). So a mint holds a lock file beside the kept token, and reads the kept file again once it +// holds it: what another process minted meanwhile is reused, not minted over; +// - replaced by id: an old token by this name is found in the account's list and deleted by its id, +// every one of them, so a name held twice is cleared rather than refused. // // An explicitly configured token still wins, and is never minted over: if it is rejected, that is // reported, not repaired — somebody chose it. @@ -20,7 +28,8 @@ // The token is never logged. Lines say that one was minted, reused or renewed, and where it is // kept; never what it is. -import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs"; +import { randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; /** The name the token carries in the forge's own list — one per mesh runtime, found by name. */ @@ -39,7 +48,7 @@ export const TOKEN_NAME = "mesh-tools"; * `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the * operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token. * A token kept with a different list is minted again with this one at the next start (the scopes are kept - * beside it), and the forge drops the old one by name — so the scope a kept token had does not outlive this. + * beside it), and the old one is deleted from the forge by its id — so the scope a kept token had does not outlive this. * * A token kept from before a scope was added lacks it: the forge answers such a call with * `403 token does not have at least one of required scope(s)`, and the client treats that like a @@ -124,9 +133,9 @@ export class MintedToken implements TokenSource { */ static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken { const opts = MintedToken.optionsFromEnv(url, env); - // One source per kept file in a process. The watcher and the tools entrypoint both build a - // client in the same runtime; two sources over one file would each renew on a 401 and drop the - // other's token by name, forever. Shared, a renewal is one renewal. + // One source per kept file in a process: two sources in one process would each renew on a 401. + // This covers one process only. The watcher and the tools are two processes (the runtime starts + // each entrypoint as its own), and those are kept apart by the mint's lock file (mint, below). const shared = MintedToken.shared.get(opts.file); if (shared) return shared; const source = new MintedToken(opts); @@ -161,9 +170,9 @@ export class MintedToken implements TokenSource { this.log(`reusing the token kept at ${this.opts.file}`); return kept; } - if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them"); + if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept); } - return this.mint("no token kept — minting one"); + return this.mint("no token kept — minting one", null); } async renew(rejected: string): Promise { @@ -177,7 +186,7 @@ export class MintedToken implements TokenSource { return kept; } this.held = null; - return this.mint("the forge rejected the kept token — minting a fresh one"); + return this.mint("the forge rejected the kept token — minting a fresh one", rejected); } /** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */ @@ -185,17 +194,95 @@ export class MintedToken implements TokenSource { return status === 403 && /required scope/i.test(body); } - /** One mint at a time: concurrent first calls share it, rather than each minting its own. */ - private mint(why: string): Promise { + /** + * One mint at a time: concurrent first calls in this process share it, and a mint in another process + * on this machine is waited for (the lock file). Once the lock is held, the kept file is read again: + * a token there that is not `stale` (the one rejected, or kept with other scopes) and carries the + * scopes asked for was minted by the process that held the lock before — it is reused. + */ + private mint(why: string, stale: string | null): Promise { if (this.inflight === null) { this.log(why); - this.inflight = this.doMint().finally(() => { + this.inflight = this.locked(async () => { + const kept = this.read(); + if (kept !== null && kept !== stale && this.keptScopes() === this.scopesLine()) { + this.held = kept; + this.log(`another process minted it meanwhile — reusing the token kept at ${this.opts.file}`); + return kept; + } + return this.doMint(); + }).finally(() => { this.inflight = null; }); } return this.inflight; } + /** How long a mint may hold the lock before another process takes it as left behind, and how long one waits. */ + static readonly LOCK_STALE_MS = 120_000; + static readonly LOCK_WAIT_MS = 150_000; + + /** + * Run `work` holding the lock file beside the kept token: created exclusively, holding this process's + * pid, the time and a nonce; removed after, only when it is still ours. A lock whose process is gone, + * or older than LOCK_STALE_MS, was left by a process that died mid-mint and is taken over — never + * waited on for ever, which would be the same lock-out by another road. + */ + private async locked(work: () => Promise): Promise { + const lock = `${this.opts.file}.lock`; + mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 }); + const mine = `${process.pid} ${Date.now()} ${randomBytes(8).toString("hex")}\n`; + const deadline = Date.now() + MintedToken.LOCK_WAIT_MS; + for (;;) { + try { + const fd = openSync(lock, "wx", 0o600); + try { writeSync(fd, mine); } finally { closeSync(fd); } + break; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "EEXIST") { + throw new Error(`cannot take the mint lock at ${lock}: ${(err as Error).message}`); + } + } + if (this.leftBehind(lock)) { + this.log(`the mint lock at ${lock} was left by a process that is gone — taking it over`); + try { unlinkSync(lock); } catch { /* another waiter took it over first */ } + continue; + } + if (Date.now() > deadline) { + throw new Error(`another process has held the mint lock at ${lock} for over ${MintedToken.LOCK_WAIT_MS / 1000}s — not minting beside it`); + } + await new Promise((r) => setTimeout(r, 50 + Math.floor(Math.random() * 50))); + } + try { + return await work(); + } finally { + try { + if (readFileSync(lock, "utf8") === mine) unlinkSync(lock); + } catch { /* already gone */ } + } + } + + /** Whether the lock's holder is gone: its pid answers no signal, or it is older than LOCK_STALE_MS. */ + private leftBehind(lock: string): boolean { + let text: string; + let age: number; + try { + text = readFileSync(lock, "utf8"); + age = Date.now() - statSync(lock).mtimeMs; + } catch { + return false; // released meanwhile: the next attempt takes it + } + if (age > MintedToken.LOCK_STALE_MS) return true; + const pid = Number(text.split(" ")[0]); + if (!Number.isInteger(pid) || pid <= 0) return age > 1_000; // half-written, or not ours: give it a moment + try { + process.kill(pid, 0); + return false; + } catch (err) { + return (err as NodeJS.ErrnoException).code === "ESRCH"; + } + } + private read(): string | null { try { const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, ""); @@ -258,11 +345,26 @@ export class MintedToken implements TokenSource { if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status); if (res.status === 400 || res.status === 422) { // The forge still holds a token by this name whose value we no longer have — the kept file - // went while the forge's data stayed. It is ours to replace: drop it by name and mint again. - this.log(`the forge already holds a token named "${this.name}" — replacing it`); - const dropped = await call("DELETE", `/${encodeURIComponent(this.name)}`); - if (dropped.status !== 204 && dropped.status !== 404) { - throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${dropped.status} ${detail(dropped.body)}`); + // went while the forge's data stayed, or two processes once minted at once and left two. It is + // ours to replace: find every one by this name in the account's list and delete each by its id + // (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint. + const held: number[] = []; + for (let page = 1; page <= 100; page++) { + const listed = await call("GET", `?page=${page}&limit=50`); + if (listed.status !== 200 || !Array.isArray(listed.body)) { + throw new Error(`Gitea GET /users/${this.opts.admin}/tokens: ${listed.status} ${detail(listed.body)}`); + } + if (listed.body.length === 0) break; + for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id); + } + this.log(held.length === 1 + ? `the forge already holds a token named "${this.name}" — replacing it` + : `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`); + for (const id of held) { + const dropped = await call("DELETE", `/${id}`); + if (dropped.status !== 204 && dropped.status !== 404) { + throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`); + } } res = await call("POST", "", { name: this.name, scopes: this.scopes }); }