Files
mesh-catalog/modules/gitea/token.ts
T
jochen ee2160d00c
mesh/merge-gate head ee2160d00c22 merged into trunk c71c513027e5 — fail: builds gitea → novox; no bus step; head ee2160d00c22 does not merge cleanly into t…
mesh/repo-check head ee2160d00c22 merged into trunk c71c513027e5 — fail: head ee2160d00c22 does not merge cleanly into trunk c71c513027e5: in conflict modu…
mesh/delivery stopped: the pull request closed unmerged
Drop every mesh-tools token by id when the forge holds two, and stop re-minting on every call (hq issue 739)
On 2026-10-14 from 22:28 local the forge held two tokens named mesh-tools.
gitea refuses a delete by such a name with 422 "multiple matches", so every
gitea tool call re-entered the mint, made two basic-auth calls that hash the
admin password, and failed: several times a minute, on a control-node that was
already overloaded, and mesh-delivery could read neither open pulls nor views.

The watcher and the tools run as separate bundles, so the in-process sharing
of one token source does not cover them; two mints racing leave two tokens by
the name, because the forge's check that a name is unused is not atomic.

A 422 on the delete by name now lists the account's tokens and drops each one
carrying the name by its id, then mints. A failed mint is not asked of the
forge again for 30 s: the same error is answered without a call.
2026-10-14 22:45:16 +02:00

341 lines
17 KiB
TypeScript

// The token the forge's tools and watcher authenticate with — and where it comes from.
//
// Nobody configures it. The forge is raised by the mesh, so there is no operator holding a token to
// paste in, and pasting one into settings would put a secret in the inventory in plaintext. What
// the mesh does deliver is the admin account: a login the manifest names and a password the vault
// minted and the host unsealed into a file (novox/hq ADR 0086). That account is enough to mint a
// token, so the module mints its own (hq issue 100, the forge's tools):
//
// - at first use, when none is kept: POST /users/{admin}/tokens over basic auth, with the two
// scopes the tools and the watcher need, and no more;
// - kept in the module's own state, a 0600 file, and read back on the next start — the forge
// hands a token's value out exactly once, so a token not kept is a token lost;
// - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not
// a fault: the forge's data was restored from a predecessor and the token the file names never
// existed there.
//
// An explicitly configured token still wins, and is never minted over: if it is rejected, that is
// reported, not repaired — somebody chose it.
//
// The token is never logged. Lines say that one was minted, reused or renewed, and where it is
// kept; never what it is.
import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
/** The name the token carries in the forge's own list — one per mesh runtime, found by name. */
export const TOKEN_NAME = "mesh-tools";
/**
* The least the fifteen tools and the watcher need (gitea's route groups, 1.20+ scoped tokens):
* write:repository — create/delete repositories, pull requests (list/get/open/merge);
* write:issue — issues, comments, labels;
* read:user — GET /user/repos, which the watcher's poll and gitea_list_repos both call.
* It sits under the `user` category despite listing repositories, not `repository`
* — confirmed against the running forge (1.27.3), which answered
* `required=[read:user]` to a token carrying only the other two.
*
* **Not write:admin** (novox/hq ADR 0266). It was here so a person could settle the forge's users through
* `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the
* operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token.
* A token kept with a different list is minted again with this one at the next start (the scopes are kept
* beside it), and the forge drops the old one by name — so the scope a kept token had does not outlive this.
*
* A token kept from before a scope was added lacks it: the forge answers such a call with
* `403 token does not have at least one of required scope(s)`, and the client treats that like a
* 401 — the source re-mints by name, with the whole list, and the call is retried once.
*/
export const TOKEN_SCOPES: readonly string[] = ["write:repository", "write:issue", "read:user"];
/** Where a client's token comes from, and what to do when the forge says it is wrong. */
export interface TokenSource {
/** The token to authenticate with now; minted, read or configured. */
current(): Promise<string>;
/** The forge answered 401 to `rejected`, or 403 for a scope it lacks. A fresh token, or a plain error when there is nothing to renew with. */
renew(rejected: string): Promise<string>;
}
/** A token somebody set — in settings or the environment. Never minted over. */
export class ConfiguredToken implements TokenSource {
constructor(private readonly token: string) {}
async current(): Promise<string> {
return this.token;
}
async renew(): Promise<string> {
throw new Error(
"the forge rejected the configured Gitea token (401). It was set explicitly (settings or MESH_GITEA_TOKEN), " +
"so the module does not mint over it — fix it, or unset it and the module mints its own",
);
}
}
/** The forge would not take the admin account: it is missing, or its password is not the one the mesh holds. */
export class AdminRefused extends Error {
constructor(admin: string, status: number) {
super(
`the forge refused the admin account "${admin}" (${status}) — it does not exist there, or its password is not ` +
`the one the vault delivered. The admin-bootstrap step creates it on a forge the mesh raised; a forge whose data ` +
`came from a predecessor does not have it. Create "${admin}" on the forge with the delivered password and the ` +
`token is minted on the next call — the tools stay registered and the watcher keeps trying`,
);
this.name = "AdminRefused";
}
}
export interface MintedTokenOptions {
/** The forge, e.g. http://127.0.0.1:3000. */
readonly url: string;
/** The admin login the manifest names. */
readonly admin: string;
/** The file the host unsealed the admin password into (ADR 0086). Read at mint time, so a rotation takes. */
readonly passwordFile: string;
/** Where the token is kept: a 0600 file in the module's own state. */
readonly file: string;
readonly name?: string;
readonly scopes?: readonly string[];
readonly log?: (line: string) => void;
readonly fetch?: typeof fetch;
/** The clock, in milliseconds; a test sets it. */
readonly now?: () => number;
}
/**
* **How long a failed mint is not tried again** (novox/hq issue 739). Every call that finds no token mints, and a
* mint is two or three calls over basic auth, each one hashing the admin password on the forge. A mint that keeps
* failing, once per tool call, is a loop the forge pays for in CPU: on 2026-10-14 it ran several times a minute
* on the control-node while it was already overloaded. Within this window the last failure is answered again,
* without asking the forge.
*/
export const MINT_RETRY_AFTER_MS = 30_000;
/** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */
export class MintedToken implements TokenSource {
private held: string | null = null;
private readFile = false;
private inflight: Promise<string> | null = null;
private failed: { at: number; error: unknown } | null = null;
private readonly name: string;
private readonly scopes: readonly string[];
private readonly log: (line: string) => void;
private readonly fetchImpl: typeof fetch;
constructor(private readonly opts: MintedTokenOptions) {
this.name = opts.name ?? TOKEN_NAME;
this.scopes = opts.scopes ?? TOKEN_SCOPES;
this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`));
this.fetchImpl = opts.fetch ?? fetch;
this.now = opts.now ?? Date.now;
}
private readonly now: () => number;
/**
* Build from the runtime's environment: the forge's URL, the admin login and password file the
* manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory
* the runtime mounts writable). Throws, naming what is missing, rather than hand back a source
* that cannot mint.
*/
static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken {
const opts = MintedToken.optionsFromEnv(url, env);
// One source per kept file in a process. The watcher and the tools entrypoint both build a
// client in the same runtime; two sources over one file would each renew on a 401 and drop the
// other's token by name, forever. Shared, a renewal is one renewal. Across processes it is not shared: the
// runtime serves the watcher (index.js) and the tools (tools/index.js) as separate bundles, so two sources over
// one file do exist. renew() reads the kept file before it mints, and a race that still leaves two tokens by
// the name is undone by dropping each by its id (doMint; novox/hq issue 739).
const shared = MintedToken.shared.get(opts.file);
if (shared) return shared;
const source = new MintedToken(opts);
MintedToken.shared.set(opts.file, source);
return source;
}
private static readonly shared = new Map<string, MintedToken>();
private static optionsFromEnv(url: string, env: NodeJS.ProcessEnv): MintedTokenOptions {
const admin = env.MESH_GITEA_ADMIN_USER;
const passwordFile = env.MESH_GITEA_ADMIN_PASSWORD_FILE;
const stateDir = env.MESH_GITEA_STATE_DIR;
const missing = [
admin ? null : "MESH_GITEA_ADMIN_USER",
passwordFile ? null : "MESH_GITEA_ADMIN_PASSWORD_FILE",
stateDir ? null : "MESH_GITEA_STATE_DIR",
].filter((v): v is string => v !== null);
if (missing.length) {
throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`);
}
return { url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token") };
}
async current(): Promise<string> {
if (this.held !== null) return this.held;
if (!this.readFile) {
this.readFile = true;
const kept = this.read();
if (kept !== null && this.keptScopes() === this.scopesLine()) {
this.held = kept;
this.log(`reusing the token kept at ${this.opts.file}`);
return kept;
}
if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them");
}
return this.mint(this.failed ? "no token held since the last mint failed — minting one" : "no token kept — minting one");
}
async renew(rejected: string): Promise<string> {
// Another caller already renewed while this one was in flight with the old token.
if (this.held !== null && this.held !== rejected) return this.held;
// Or another process did, and kept it: use what is kept before minting over it.
const kept = this.read();
if (kept !== null && kept !== rejected) {
this.held = kept;
this.log(`the forge rejected the token held; the kept one at ${this.opts.file} is newer — reusing it`);
return kept;
}
this.held = null;
return this.mint("the forge rejected the kept token — minting a fresh one");
}
/** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
static lacksScope(status: number, body: string): boolean {
return status === 403 && /required scope/i.test(body);
}
/** One mint at a time: concurrent first calls share it, rather than each minting its own. */
private mint(why: string): Promise<string> {
if (this.inflight === null) {
// A mint that failed a moment ago is not asked of the forge again yet (novox/hq issue 739): its error stands.
if (this.failed !== null && this.now() - this.failed.at < MINT_RETRY_AFTER_MS) return Promise.reject(this.failed.error);
this.log(why);
this.inflight = this.doMint().then(
(token) => {
this.failed = null;
return token;
},
(err) => {
this.failed = { at: this.now(), error: err };
throw err;
},
).finally(() => {
this.inflight = null;
});
}
return this.inflight;
}
private read(): string | null {
try {
const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, "");
return token.length ? token : null;
} catch (err) {
if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
throw new Error(`cannot read the kept Gitea token at ${this.opts.file}: ${(err as Error).message}`);
}
}
/** The scopes as kept beside the token: sorted, one line. */
private scopesLine(): string {
return [...this.scopes].sort().join(",");
}
/** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */
private keptScopes(): string {
try {
return readFileSync(`${this.opts.file}.scopes`, "utf8").trim();
} catch {
return "";
}
}
/** Write the token at 0600, whole or not at all: a temp file beside it, then a rename. Its scopes beside it. */
private keep(token: string): void {
mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 });
const tmp = `${this.opts.file}.tmp`;
writeFileSync(tmp, token + "\n", { mode: 0o600 });
chmodSync(tmp, 0o600);
renameSync(tmp, this.opts.file);
writeFileSync(`${this.opts.file}.scopes.tmp`, this.scopesLine() + "\n", { mode: 0o600 });
renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`);
}
private async doMint(): Promise<string> {
let password: string;
try {
password = readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, "");
} catch (err) {
throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`);
}
const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64");
const tokens = `${this.opts.url.replace(/\/+$/, "")}/api/v1/users/${encodeURIComponent(this.opts.admin)}/tokens`;
const call = async (method: string, path = "", body?: unknown): Promise<{ status: number; body: any }> => {
const res = await this.fetchImpl(tokens + path, {
method,
headers: { "Content-Type": "application/json", Authorization: authorization },
...(body === undefined ? {} : { body: JSON.stringify(body) }),
});
const text = await res.text();
let parsed: any = null;
if (text) {
try { parsed = JSON.parse(text); } catch { parsed = text; }
}
return { status: res.status, body: parsed };
};
let res = await call("POST", "", { name: this.name, scopes: this.scopes });
if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status);
if (res.status === 400 || res.status === 422) {
// The forge still holds a token by this name whose value we no longer have — the kept file
// went while the forge's data stayed. It is ours to replace: drop it by name and mint again.
this.log(`the forge already holds a token named "${this.name}" — replacing it`);
const dropped = await call("DELETE", `/${encodeURIComponent(this.name)}`);
if (dropped.status === 422) {
// More than one token carries the name (novox/hq issue 739): two processes minted at once, and the forge's
// check that a name is unused is not atomic. The forge refuses a delete by such a name for good, so every
// one of them is dropped by its id — all are ours, and none of their values is kept.
await this.dropEveryNamed(call, dropped);
} else if (dropped.status !== 204 && dropped.status !== 404) {
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${dropped.status} ${detail(dropped.body)}`);
}
res = await call("POST", "", { name: this.name, scopes: this.scopes });
}
if (res.status !== 201 && res.status !== 200) {
throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} ${detail(res.body)}`);
}
const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null;
if (!token) throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} but no token in the reply`);
this.keep(token);
this.held = token;
this.log(`minted a token for "${this.opts.admin}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`);
return token;
}
/** Drop, by id, every token of the admin account that carries this name; the forge listed them, page by page. */
private async dropEveryNamed(call: (method: string, path?: string) => Promise<{ status: number; body: any }>,
refused: { status: number; body: any }): Promise<void> {
const ids: number[] = [];
for (let page = 1; page <= 20; page++) {
const listed = await call("GET", `?page=${page}&limit=50`);
if (listed.status !== 200 || !Array.isArray(listed.body)) {
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${refused.status} ${detail(refused.body)}; ` +
`and its tokens could not be listed to drop them one by one: ${listed.status} ${detail(listed.body)}`);
}
for (const t of listed.body) if (t?.name === this.name && Number.isInteger(t?.id)) ids.push(t.id);
if (listed.body.length < 50) break;
}
this.log(`the forge holds ${ids.length} tokens named "${this.name}" — dropping each by its id`);
for (const id of ids) {
const dropped = await call("DELETE", `/${id}`);
if (dropped.status !== 204 && dropped.status !== 404) {
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id}: ${dropped.status} ${detail(dropped.body)}`);
}
}
}
}
function detail(body: unknown): string {
return typeof body === "string" ? body : JSON.stringify(body);
}