mesh/merge-gate head ee2160d00c22 merged into trunk c71c513027e5 — fail: builds gitea → novox; no bus step; head ee2160d00c22 does not merge cleanly into t…
mesh/repo-check head ee2160d00c22 merged into trunk c71c513027e5 — fail: head ee2160d00c22 does not merge cleanly into trunk c71c513027e5: in conflict modu…
mesh/delivery stopped: the pull request closed unmerged
On 2026-10-14 from 22:28 local the forge held two tokens named mesh-tools. gitea refuses a delete by such a name with 422 "multiple matches", so every gitea tool call re-entered the mint, made two basic-auth calls that hash the admin password, and failed: several times a minute, on a control-node that was already overloaded, and mesh-delivery could read neither open pulls nor views. The watcher and the tools run as separate bundles, so the in-process sharing of one token source does not cover them; two mints racing leave two tokens by the name, because the forge's check that a name is unused is not atomic. A 422 on the delete by name now lists the account's tokens and drops each one carrying the name by its id, then mints. A failed mint is not asked of the forge again for 30 s: the same error is answered without a call.
341 lines
17 KiB
TypeScript
341 lines
17 KiB
TypeScript
// The token the forge's tools and watcher authenticate with — and where it comes from.
|
|
//
|
|
// Nobody configures it. The forge is raised by the mesh, so there is no operator holding a token to
|
|
// paste in, and pasting one into settings would put a secret in the inventory in plaintext. What
|
|
// the mesh does deliver is the admin account: a login the manifest names and a password the vault
|
|
// minted and the host unsealed into a file (novox/hq ADR 0086). That account is enough to mint a
|
|
// token, so the module mints its own (hq issue 100, the forge's tools):
|
|
//
|
|
// - at first use, when none is kept: POST /users/{admin}/tokens over basic auth, with the two
|
|
// scopes the tools and the watcher need, and no more;
|
|
// - kept in the module's own state, a 0600 file, and read back on the next start — the forge
|
|
// hands a token's value out exactly once, so a token not kept is a token lost;
|
|
// - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not
|
|
// a fault: the forge's data was restored from a predecessor and the token the file names never
|
|
// existed there.
|
|
//
|
|
// An explicitly configured token still wins, and is never minted over: if it is rejected, that is
|
|
// reported, not repaired — somebody chose it.
|
|
//
|
|
// The token is never logged. Lines say that one was minted, reused or renewed, and where it is
|
|
// kept; never what it is.
|
|
|
|
import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
|
|
/** The name the token carries in the forge's own list — one per mesh runtime, found by name. */
|
|
export const TOKEN_NAME = "mesh-tools";
|
|
|
|
/**
|
|
* The least the fifteen tools and the watcher need (gitea's route groups, 1.20+ scoped tokens):
|
|
* write:repository — create/delete repositories, pull requests (list/get/open/merge);
|
|
* write:issue — issues, comments, labels;
|
|
* read:user — GET /user/repos, which the watcher's poll and gitea_list_repos both call.
|
|
* It sits under the `user` category despite listing repositories, not `repository`
|
|
* — confirmed against the running forge (1.27.3), which answered
|
|
* `required=[read:user]` to a token carrying only the other two.
|
|
*
|
|
* **Not write:admin** (novox/hq ADR 0266). It was here so a person could settle the forge's users through
|
|
* `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the
|
|
* operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token.
|
|
* A token kept with a different list is minted again with this one at the next start (the scopes are kept
|
|
* beside it), and the forge drops the old one by name — so the scope a kept token had does not outlive this.
|
|
*
|
|
* A token kept from before a scope was added lacks it: the forge answers such a call with
|
|
* `403 token does not have at least one of required scope(s)`, and the client treats that like a
|
|
* 401 — the source re-mints by name, with the whole list, and the call is retried once.
|
|
*/
|
|
export const TOKEN_SCOPES: readonly string[] = ["write:repository", "write:issue", "read:user"];
|
|
|
|
/** Where a client's token comes from, and what to do when the forge says it is wrong. */
|
|
export interface TokenSource {
|
|
/** The token to authenticate with now; minted, read or configured. */
|
|
current(): Promise<string>;
|
|
/** The forge answered 401 to `rejected`, or 403 for a scope it lacks. A fresh token, or a plain error when there is nothing to renew with. */
|
|
renew(rejected: string): Promise<string>;
|
|
}
|
|
|
|
/** A token somebody set — in settings or the environment. Never minted over. */
|
|
export class ConfiguredToken implements TokenSource {
|
|
constructor(private readonly token: string) {}
|
|
|
|
async current(): Promise<string> {
|
|
return this.token;
|
|
}
|
|
|
|
async renew(): Promise<string> {
|
|
throw new Error(
|
|
"the forge rejected the configured Gitea token (401). It was set explicitly (settings or MESH_GITEA_TOKEN), " +
|
|
"so the module does not mint over it — fix it, or unset it and the module mints its own",
|
|
);
|
|
}
|
|
}
|
|
|
|
/** The forge would not take the admin account: it is missing, or its password is not the one the mesh holds. */
|
|
export class AdminRefused extends Error {
|
|
constructor(admin: string, status: number) {
|
|
super(
|
|
`the forge refused the admin account "${admin}" (${status}) — it does not exist there, or its password is not ` +
|
|
`the one the vault delivered. The admin-bootstrap step creates it on a forge the mesh raised; a forge whose data ` +
|
|
`came from a predecessor does not have it. Create "${admin}" on the forge with the delivered password and the ` +
|
|
`token is minted on the next call — the tools stay registered and the watcher keeps trying`,
|
|
);
|
|
this.name = "AdminRefused";
|
|
}
|
|
}
|
|
|
|
export interface MintedTokenOptions {
|
|
/** The forge, e.g. http://127.0.0.1:3000. */
|
|
readonly url: string;
|
|
/** The admin login the manifest names. */
|
|
readonly admin: string;
|
|
/** The file the host unsealed the admin password into (ADR 0086). Read at mint time, so a rotation takes. */
|
|
readonly passwordFile: string;
|
|
/** Where the token is kept: a 0600 file in the module's own state. */
|
|
readonly file: string;
|
|
readonly name?: string;
|
|
readonly scopes?: readonly string[];
|
|
readonly log?: (line: string) => void;
|
|
readonly fetch?: typeof fetch;
|
|
/** The clock, in milliseconds; a test sets it. */
|
|
readonly now?: () => number;
|
|
}
|
|
|
|
/**
|
|
* **How long a failed mint is not tried again** (novox/hq issue 739). Every call that finds no token mints, and a
|
|
* mint is two or three calls over basic auth, each one hashing the admin password on the forge. A mint that keeps
|
|
* failing, once per tool call, is a loop the forge pays for in CPU: on 2026-10-14 it ran several times a minute
|
|
* on the control-node while it was already overloaded. Within this window the last failure is answered again,
|
|
* without asking the forge.
|
|
*/
|
|
export const MINT_RETRY_AFTER_MS = 30_000;
|
|
|
|
/** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */
|
|
export class MintedToken implements TokenSource {
|
|
private held: string | null = null;
|
|
private readFile = false;
|
|
private inflight: Promise<string> | null = null;
|
|
private failed: { at: number; error: unknown } | null = null;
|
|
private readonly name: string;
|
|
private readonly scopes: readonly string[];
|
|
private readonly log: (line: string) => void;
|
|
private readonly fetchImpl: typeof fetch;
|
|
|
|
constructor(private readonly opts: MintedTokenOptions) {
|
|
this.name = opts.name ?? TOKEN_NAME;
|
|
this.scopes = opts.scopes ?? TOKEN_SCOPES;
|
|
this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`));
|
|
this.fetchImpl = opts.fetch ?? fetch;
|
|
this.now = opts.now ?? Date.now;
|
|
}
|
|
|
|
private readonly now: () => number;
|
|
|
|
/**
|
|
* Build from the runtime's environment: the forge's URL, the admin login and password file the
|
|
* manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory
|
|
* the runtime mounts writable). Throws, naming what is missing, rather than hand back a source
|
|
* that cannot mint.
|
|
*/
|
|
static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken {
|
|
const opts = MintedToken.optionsFromEnv(url, env);
|
|
// One source per kept file in a process. The watcher and the tools entrypoint both build a
|
|
// client in the same runtime; two sources over one file would each renew on a 401 and drop the
|
|
// other's token by name, forever. Shared, a renewal is one renewal. Across processes it is not shared: the
|
|
// runtime serves the watcher (index.js) and the tools (tools/index.js) as separate bundles, so two sources over
|
|
// one file do exist. renew() reads the kept file before it mints, and a race that still leaves two tokens by
|
|
// the name is undone by dropping each by its id (doMint; novox/hq issue 739).
|
|
const shared = MintedToken.shared.get(opts.file);
|
|
if (shared) return shared;
|
|
const source = new MintedToken(opts);
|
|
MintedToken.shared.set(opts.file, source);
|
|
return source;
|
|
}
|
|
|
|
private static readonly shared = new Map<string, MintedToken>();
|
|
|
|
private static optionsFromEnv(url: string, env: NodeJS.ProcessEnv): MintedTokenOptions {
|
|
const admin = env.MESH_GITEA_ADMIN_USER;
|
|
const passwordFile = env.MESH_GITEA_ADMIN_PASSWORD_FILE;
|
|
const stateDir = env.MESH_GITEA_STATE_DIR;
|
|
const missing = [
|
|
admin ? null : "MESH_GITEA_ADMIN_USER",
|
|
passwordFile ? null : "MESH_GITEA_ADMIN_PASSWORD_FILE",
|
|
stateDir ? null : "MESH_GITEA_STATE_DIR",
|
|
].filter((v): v is string => v !== null);
|
|
if (missing.length) {
|
|
throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`);
|
|
}
|
|
return { url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token") };
|
|
}
|
|
|
|
async current(): Promise<string> {
|
|
if (this.held !== null) return this.held;
|
|
if (!this.readFile) {
|
|
this.readFile = true;
|
|
const kept = this.read();
|
|
if (kept !== null && this.keptScopes() === this.scopesLine()) {
|
|
this.held = kept;
|
|
this.log(`reusing the token kept at ${this.opts.file}`);
|
|
return kept;
|
|
}
|
|
if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them");
|
|
}
|
|
return this.mint(this.failed ? "no token held since the last mint failed — minting one" : "no token kept — minting one");
|
|
}
|
|
|
|
async renew(rejected: string): Promise<string> {
|
|
// Another caller already renewed while this one was in flight with the old token.
|
|
if (this.held !== null && this.held !== rejected) return this.held;
|
|
// Or another process did, and kept it: use what is kept before minting over it.
|
|
const kept = this.read();
|
|
if (kept !== null && kept !== rejected) {
|
|
this.held = kept;
|
|
this.log(`the forge rejected the token held; the kept one at ${this.opts.file} is newer — reusing it`);
|
|
return kept;
|
|
}
|
|
this.held = null;
|
|
return this.mint("the forge rejected the kept token — minting a fresh one");
|
|
}
|
|
|
|
/** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
|
|
static lacksScope(status: number, body: string): boolean {
|
|
return status === 403 && /required scope/i.test(body);
|
|
}
|
|
|
|
/** One mint at a time: concurrent first calls share it, rather than each minting its own. */
|
|
private mint(why: string): Promise<string> {
|
|
if (this.inflight === null) {
|
|
// A mint that failed a moment ago is not asked of the forge again yet (novox/hq issue 739): its error stands.
|
|
if (this.failed !== null && this.now() - this.failed.at < MINT_RETRY_AFTER_MS) return Promise.reject(this.failed.error);
|
|
this.log(why);
|
|
this.inflight = this.doMint().then(
|
|
(token) => {
|
|
this.failed = null;
|
|
return token;
|
|
},
|
|
(err) => {
|
|
this.failed = { at: this.now(), error: err };
|
|
throw err;
|
|
},
|
|
).finally(() => {
|
|
this.inflight = null;
|
|
});
|
|
}
|
|
return this.inflight;
|
|
}
|
|
|
|
private read(): string | null {
|
|
try {
|
|
const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, "");
|
|
return token.length ? token : null;
|
|
} catch (err) {
|
|
if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
throw new Error(`cannot read the kept Gitea token at ${this.opts.file}: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
|
|
/** The scopes as kept beside the token: sorted, one line. */
|
|
private scopesLine(): string {
|
|
return [...this.scopes].sort().join(",");
|
|
}
|
|
|
|
/** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */
|
|
private keptScopes(): string {
|
|
try {
|
|
return readFileSync(`${this.opts.file}.scopes`, "utf8").trim();
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
/** Write the token at 0600, whole or not at all: a temp file beside it, then a rename. Its scopes beside it. */
|
|
private keep(token: string): void {
|
|
mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 });
|
|
const tmp = `${this.opts.file}.tmp`;
|
|
writeFileSync(tmp, token + "\n", { mode: 0o600 });
|
|
chmodSync(tmp, 0o600);
|
|
renameSync(tmp, this.opts.file);
|
|
writeFileSync(`${this.opts.file}.scopes.tmp`, this.scopesLine() + "\n", { mode: 0o600 });
|
|
renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`);
|
|
}
|
|
|
|
private async doMint(): Promise<string> {
|
|
let password: string;
|
|
try {
|
|
password = readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, "");
|
|
} catch (err) {
|
|
throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`);
|
|
}
|
|
const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64");
|
|
const tokens = `${this.opts.url.replace(/\/+$/, "")}/api/v1/users/${encodeURIComponent(this.opts.admin)}/tokens`;
|
|
const call = async (method: string, path = "", body?: unknown): Promise<{ status: number; body: any }> => {
|
|
const res = await this.fetchImpl(tokens + path, {
|
|
method,
|
|
headers: { "Content-Type": "application/json", Authorization: authorization },
|
|
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
});
|
|
const text = await res.text();
|
|
let parsed: any = null;
|
|
if (text) {
|
|
try { parsed = JSON.parse(text); } catch { parsed = text; }
|
|
}
|
|
return { status: res.status, body: parsed };
|
|
};
|
|
|
|
let res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
|
if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status);
|
|
if (res.status === 400 || res.status === 422) {
|
|
// The forge still holds a token by this name whose value we no longer have — the kept file
|
|
// went while the forge's data stayed. It is ours to replace: drop it by name and mint again.
|
|
this.log(`the forge already holds a token named "${this.name}" — replacing it`);
|
|
const dropped = await call("DELETE", `/${encodeURIComponent(this.name)}`);
|
|
if (dropped.status === 422) {
|
|
// More than one token carries the name (novox/hq issue 739): two processes minted at once, and the forge's
|
|
// check that a name is unused is not atomic. The forge refuses a delete by such a name for good, so every
|
|
// one of them is dropped by its id — all are ours, and none of their values is kept.
|
|
await this.dropEveryNamed(call, dropped);
|
|
} else if (dropped.status !== 204 && dropped.status !== 404) {
|
|
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${dropped.status} ${detail(dropped.body)}`);
|
|
}
|
|
res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
|
}
|
|
if (res.status !== 201 && res.status !== 200) {
|
|
throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} ${detail(res.body)}`);
|
|
}
|
|
const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null;
|
|
if (!token) throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} but no token in the reply`);
|
|
|
|
this.keep(token);
|
|
this.held = token;
|
|
this.log(`minted a token for "${this.opts.admin}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`);
|
|
return token;
|
|
}
|
|
|
|
/** Drop, by id, every token of the admin account that carries this name; the forge listed them, page by page. */
|
|
private async dropEveryNamed(call: (method: string, path?: string) => Promise<{ status: number; body: any }>,
|
|
refused: { status: number; body: any }): Promise<void> {
|
|
const ids: number[] = [];
|
|
for (let page = 1; page <= 20; page++) {
|
|
const listed = await call("GET", `?page=${page}&limit=50`);
|
|
if (listed.status !== 200 || !Array.isArray(listed.body)) {
|
|
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${refused.status} ${detail(refused.body)}; ` +
|
|
`and its tokens could not be listed to drop them one by one: ${listed.status} ${detail(listed.body)}`);
|
|
}
|
|
for (const t of listed.body) if (t?.name === this.name && Number.isInteger(t?.id)) ids.push(t.id);
|
|
if (listed.body.length < 50) break;
|
|
}
|
|
this.log(`the forge holds ${ids.length} tokens named "${this.name}" — dropping each by its id`);
|
|
for (const id of ids) {
|
|
const dropped = await call("DELETE", `/${id}`);
|
|
if (dropped.status !== 204 && dropped.status !== 404) {
|
|
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id}: ${dropped.status} ${detail(dropped.body)}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function detail(body: unknown): string {
|
|
return typeof body === "string" ? body : JSON.stringify(body);
|
|
}
|