jschoubben
d1debdcce8
mesh/merge-gate head d1debdcce816 merged into trunk 0e7d9e74d22a — pass: builds gitea → novox; no bus step; every machine composes with the change as it di…
mesh/repo-check head d1debdcce816 merged into trunk 0e7d9e74d22a — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
The runtime starts the watcher and the tools as two processes over one kept token. Both minted at once on 2026-10-14 and left two tokens named mesh-tools; gitea refuses a delete by a name held twice, so every mint failed and every forge tool with it. A replace now deletes each token by the name by its id, from the account's whole list, and a mint holds a lock file beside the kept token and reuses what another process minted.
387 lines
18 KiB
TypeScript
387 lines
18 KiB
TypeScript
// The token the forge's tools and watcher authenticate with — and where it comes from.
|
|
//
|
|
// Nobody configures it. The forge is raised by the mesh, so there is no operator holding a token to
|
|
// paste in, and pasting one into settings would put a secret in the inventory in plaintext. What
|
|
// the mesh does deliver is the admin account: a login the manifest names and a password the vault
|
|
// minted and the host unsealed into a file (novox/hq ADR 0086). That account is enough to mint a
|
|
// token, so the module mints its own (hq issue 100, the forge's tools):
|
|
//
|
|
// - at first use, when none is kept: POST /users/{admin}/tokens over basic auth, with the two
|
|
// scopes the tools and the watcher need, and no more;
|
|
// - kept in the module's own state, a 0600 file, and read back on the next start — the forge
|
|
// hands a token's value out exactly once, so a token not kept is a token lost;
|
|
// - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not
|
|
// a fault: the forge's data was restored from a predecessor and the token the file names never
|
|
// existed there;
|
|
// - minted by one process at a time. The runtime starts each of the bundle's entrypoints as a process
|
|
// of its own (novox/hq ADR 0193): the watcher (index.js) and the tools (tools/index.js) are two
|
|
// processes over one kept file. Two that minted at once left the forge two tokens by one name, which
|
|
// gitea then refuses to delete by name — for ever, and every forge tool with it (the 2026-10-14
|
|
// outage). So a mint holds a lock file beside the kept token, and reads the kept file again once it
|
|
// holds it: what another process minted meanwhile is reused, not minted over;
|
|
// - replaced by id: an old token by this name is found in the account's list and deleted by its id,
|
|
// every one of them, so a name held twice is cleared rather than refused.
|
|
//
|
|
// An explicitly configured token still wins, and is never minted over: if it is rejected, that is
|
|
// reported, not repaired — somebody chose it.
|
|
//
|
|
// The token is never logged. Lines say that one was minted, reused or renewed, and where it is
|
|
// kept; never what it is.
|
|
|
|
import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
|
import { randomBytes } from "node:crypto";
|
|
import { dirname, join } from "node:path";
|
|
|
|
/** The name the token carries in the forge's own list — one per mesh runtime, found by name. */
|
|
export const TOKEN_NAME = "mesh-tools";
|
|
|
|
/**
|
|
* The least the fifteen tools and the watcher need (gitea's route groups, 1.20+ scoped tokens):
|
|
* write:repository — create/delete repositories, pull requests (list/get/open/merge);
|
|
* write:issue — issues, comments, labels;
|
|
* read:user — GET /user/repos, which the watcher's poll and gitea_list_repos both call.
|
|
* It sits under the `user` category despite listing repositories, not `repository`
|
|
* — confirmed against the running forge (1.27.3), which answered
|
|
* `required=[read:user]` to a token carrying only the other two.
|
|
*
|
|
* **Not write:admin** (novox/hq ADR 0266). It was here so a person could settle the forge's users through
|
|
* `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the
|
|
* operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token.
|
|
* A token kept with a different list is minted again with this one at the next start (the scopes are kept
|
|
* beside it), and the old one is deleted from the forge by its id — so the scope a kept token had does not outlive this.
|
|
*
|
|
* A token kept from before a scope was added lacks it: the forge answers such a call with
|
|
* `403 token does not have at least one of required scope(s)`, and the client treats that like a
|
|
* 401 — the source re-mints by name, with the whole list, and the call is retried once.
|
|
*/
|
|
export const TOKEN_SCOPES: readonly string[] = ["write:repository", "write:issue", "read:user"];
|
|
|
|
/** Where a client's token comes from, and what to do when the forge says it is wrong. */
|
|
export interface TokenSource {
|
|
/** The token to authenticate with now; minted, read or configured. */
|
|
current(): Promise<string>;
|
|
/** The forge answered 401 to `rejected`, or 403 for a scope it lacks. A fresh token, or a plain error when there is nothing to renew with. */
|
|
renew(rejected: string): Promise<string>;
|
|
}
|
|
|
|
/** A token somebody set — in settings or the environment. Never minted over. */
|
|
export class ConfiguredToken implements TokenSource {
|
|
constructor(private readonly token: string) {}
|
|
|
|
async current(): Promise<string> {
|
|
return this.token;
|
|
}
|
|
|
|
async renew(): Promise<string> {
|
|
throw new Error(
|
|
"the forge rejected the configured Gitea token (401). It was set explicitly (settings or MESH_GITEA_TOKEN), " +
|
|
"so the module does not mint over it — fix it, or unset it and the module mints its own",
|
|
);
|
|
}
|
|
}
|
|
|
|
/** The forge would not take the admin account: it is missing, or its password is not the one the mesh holds. */
|
|
export class AdminRefused extends Error {
|
|
constructor(admin: string, status: number) {
|
|
super(
|
|
`the forge refused the admin account "${admin}" (${status}) — it does not exist there, or its password is not ` +
|
|
`the one the vault delivered. The admin-bootstrap step creates it on a forge the mesh raised; a forge whose data ` +
|
|
`came from a predecessor does not have it. Create "${admin}" on the forge with the delivered password and the ` +
|
|
`token is minted on the next call — the tools stay registered and the watcher keeps trying`,
|
|
);
|
|
this.name = "AdminRefused";
|
|
}
|
|
}
|
|
|
|
export interface MintedTokenOptions {
|
|
/** The forge, e.g. http://127.0.0.1:3000. */
|
|
readonly url: string;
|
|
/** The admin login the manifest names. */
|
|
readonly admin: string;
|
|
/** The file the host unsealed the admin password into (ADR 0086). Read at mint time, so a rotation takes. */
|
|
readonly passwordFile: string;
|
|
/** Where the token is kept: a 0600 file in the module's own state. */
|
|
readonly file: string;
|
|
readonly name?: string;
|
|
readonly scopes?: readonly string[];
|
|
readonly log?: (line: string) => void;
|
|
readonly fetch?: typeof fetch;
|
|
}
|
|
|
|
/** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */
|
|
export class MintedToken implements TokenSource {
|
|
private held: string | null = null;
|
|
private readFile = false;
|
|
private inflight: Promise<string> | null = null;
|
|
private readonly name: string;
|
|
private readonly scopes: readonly string[];
|
|
private readonly log: (line: string) => void;
|
|
private readonly fetchImpl: typeof fetch;
|
|
|
|
constructor(private readonly opts: MintedTokenOptions) {
|
|
this.name = opts.name ?? TOKEN_NAME;
|
|
this.scopes = opts.scopes ?? TOKEN_SCOPES;
|
|
this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`));
|
|
this.fetchImpl = opts.fetch ?? fetch;
|
|
}
|
|
|
|
/**
|
|
* Build from the runtime's environment: the forge's URL, the admin login and password file the
|
|
* manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory
|
|
* the runtime mounts writable). Throws, naming what is missing, rather than hand back a source
|
|
* that cannot mint.
|
|
*/
|
|
static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken {
|
|
const opts = MintedToken.optionsFromEnv(url, env);
|
|
// One source per kept file in a process: two sources in one process would each renew on a 401.
|
|
// This covers one process only. The watcher and the tools are two processes (the runtime starts
|
|
// each entrypoint as its own), and those are kept apart by the mint's lock file (mint, below).
|
|
const shared = MintedToken.shared.get(opts.file);
|
|
if (shared) return shared;
|
|
const source = new MintedToken(opts);
|
|
MintedToken.shared.set(opts.file, source);
|
|
return source;
|
|
}
|
|
|
|
private static readonly shared = new Map<string, MintedToken>();
|
|
|
|
private static optionsFromEnv(url: string, env: NodeJS.ProcessEnv): MintedTokenOptions {
|
|
const admin = env.MESH_GITEA_ADMIN_USER;
|
|
const passwordFile = env.MESH_GITEA_ADMIN_PASSWORD_FILE;
|
|
const stateDir = env.MESH_GITEA_STATE_DIR;
|
|
const missing = [
|
|
admin ? null : "MESH_GITEA_ADMIN_USER",
|
|
passwordFile ? null : "MESH_GITEA_ADMIN_PASSWORD_FILE",
|
|
stateDir ? null : "MESH_GITEA_STATE_DIR",
|
|
].filter((v): v is string => v !== null);
|
|
if (missing.length) {
|
|
throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`);
|
|
}
|
|
return { url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token") };
|
|
}
|
|
|
|
async current(): Promise<string> {
|
|
if (this.held !== null) return this.held;
|
|
if (!this.readFile) {
|
|
this.readFile = true;
|
|
const kept = this.read();
|
|
if (kept !== null && this.keptScopes() === this.scopesLine()) {
|
|
this.held = kept;
|
|
this.log(`reusing the token kept at ${this.opts.file}`);
|
|
return kept;
|
|
}
|
|
if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept);
|
|
}
|
|
return this.mint("no token kept — minting one", null);
|
|
}
|
|
|
|
async renew(rejected: string): Promise<string> {
|
|
// Another caller already renewed while this one was in flight with the old token.
|
|
if (this.held !== null && this.held !== rejected) return this.held;
|
|
// Or another process did, and kept it: use what is kept before minting over it.
|
|
const kept = this.read();
|
|
if (kept !== null && kept !== rejected) {
|
|
this.held = kept;
|
|
this.log(`the forge rejected the token held; the kept one at ${this.opts.file} is newer — reusing it`);
|
|
return kept;
|
|
}
|
|
this.held = null;
|
|
return this.mint("the forge rejected the kept token — minting a fresh one", rejected);
|
|
}
|
|
|
|
/** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
|
|
static lacksScope(status: number, body: string): boolean {
|
|
return status === 403 && /required scope/i.test(body);
|
|
}
|
|
|
|
/**
|
|
* One mint at a time: concurrent first calls in this process share it, and a mint in another process
|
|
* on this machine is waited for (the lock file). Once the lock is held, the kept file is read again:
|
|
* a token there that is not `stale` (the one rejected, or kept with other scopes) and carries the
|
|
* scopes asked for was minted by the process that held the lock before — it is reused.
|
|
*/
|
|
private mint(why: string, stale: string | null): Promise<string> {
|
|
if (this.inflight === null) {
|
|
this.log(why);
|
|
this.inflight = this.locked(async () => {
|
|
const kept = this.read();
|
|
if (kept !== null && kept !== stale && this.keptScopes() === this.scopesLine()) {
|
|
this.held = kept;
|
|
this.log(`another process minted it meanwhile — reusing the token kept at ${this.opts.file}`);
|
|
return kept;
|
|
}
|
|
return this.doMint();
|
|
}).finally(() => {
|
|
this.inflight = null;
|
|
});
|
|
}
|
|
return this.inflight;
|
|
}
|
|
|
|
/** How long a mint may hold the lock before another process takes it as left behind, and how long one waits. */
|
|
static readonly LOCK_STALE_MS = 120_000;
|
|
static readonly LOCK_WAIT_MS = 150_000;
|
|
|
|
/**
|
|
* Run `work` holding the lock file beside the kept token: created exclusively, holding this process's
|
|
* pid, the time and a nonce; removed after, only when it is still ours. A lock whose process is gone,
|
|
* or older than LOCK_STALE_MS, was left by a process that died mid-mint and is taken over — never
|
|
* waited on for ever, which would be the same lock-out by another road.
|
|
*/
|
|
private async locked<T>(work: () => Promise<T>): Promise<T> {
|
|
const lock = `${this.opts.file}.lock`;
|
|
mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 });
|
|
const mine = `${process.pid} ${Date.now()} ${randomBytes(8).toString("hex")}\n`;
|
|
const deadline = Date.now() + MintedToken.LOCK_WAIT_MS;
|
|
for (;;) {
|
|
try {
|
|
const fd = openSync(lock, "wx", 0o600);
|
|
try { writeSync(fd, mine); } finally { closeSync(fd); }
|
|
break;
|
|
} catch (err) {
|
|
if ((err as NodeJS.ErrnoException).code !== "EEXIST") {
|
|
throw new Error(`cannot take the mint lock at ${lock}: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
if (this.leftBehind(lock)) {
|
|
this.log(`the mint lock at ${lock} was left by a process that is gone — taking it over`);
|
|
try { unlinkSync(lock); } catch { /* another waiter took it over first */ }
|
|
continue;
|
|
}
|
|
if (Date.now() > deadline) {
|
|
throw new Error(`another process has held the mint lock at ${lock} for over ${MintedToken.LOCK_WAIT_MS / 1000}s — not minting beside it`);
|
|
}
|
|
await new Promise((r) => setTimeout(r, 50 + Math.floor(Math.random() * 50)));
|
|
}
|
|
try {
|
|
return await work();
|
|
} finally {
|
|
try {
|
|
if (readFileSync(lock, "utf8") === mine) unlinkSync(lock);
|
|
} catch { /* already gone */ }
|
|
}
|
|
}
|
|
|
|
/** Whether the lock's holder is gone: its pid answers no signal, or it is older than LOCK_STALE_MS. */
|
|
private leftBehind(lock: string): boolean {
|
|
let text: string;
|
|
let age: number;
|
|
try {
|
|
text = readFileSync(lock, "utf8");
|
|
age = Date.now() - statSync(lock).mtimeMs;
|
|
} catch {
|
|
return false; // released meanwhile: the next attempt takes it
|
|
}
|
|
if (age > MintedToken.LOCK_STALE_MS) return true;
|
|
const pid = Number(text.split(" ")[0]);
|
|
if (!Number.isInteger(pid) || pid <= 0) return age > 1_000; // half-written, or not ours: give it a moment
|
|
try {
|
|
process.kill(pid, 0);
|
|
return false;
|
|
} catch (err) {
|
|
return (err as NodeJS.ErrnoException).code === "ESRCH";
|
|
}
|
|
}
|
|
|
|
private read(): string | null {
|
|
try {
|
|
const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, "");
|
|
return token.length ? token : null;
|
|
} catch (err) {
|
|
if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
throw new Error(`cannot read the kept Gitea token at ${this.opts.file}: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
|
|
/** The scopes as kept beside the token: sorted, one line. */
|
|
private scopesLine(): string {
|
|
return [...this.scopes].sort().join(",");
|
|
}
|
|
|
|
/** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */
|
|
private keptScopes(): string {
|
|
try {
|
|
return readFileSync(`${this.opts.file}.scopes`, "utf8").trim();
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
/** Write the token at 0600, whole or not at all: a temp file beside it, then a rename. Its scopes beside it. */
|
|
private keep(token: string): void {
|
|
mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 });
|
|
const tmp = `${this.opts.file}.tmp`;
|
|
writeFileSync(tmp, token + "\n", { mode: 0o600 });
|
|
chmodSync(tmp, 0o600);
|
|
renameSync(tmp, this.opts.file);
|
|
writeFileSync(`${this.opts.file}.scopes.tmp`, this.scopesLine() + "\n", { mode: 0o600 });
|
|
renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`);
|
|
}
|
|
|
|
private async doMint(): Promise<string> {
|
|
let password: string;
|
|
try {
|
|
password = readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, "");
|
|
} catch (err) {
|
|
throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`);
|
|
}
|
|
const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64");
|
|
const tokens = `${this.opts.url.replace(/\/+$/, "")}/api/v1/users/${encodeURIComponent(this.opts.admin)}/tokens`;
|
|
const call = async (method: string, path = "", body?: unknown): Promise<{ status: number; body: any }> => {
|
|
const res = await this.fetchImpl(tokens + path, {
|
|
method,
|
|
headers: { "Content-Type": "application/json", Authorization: authorization },
|
|
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
});
|
|
const text = await res.text();
|
|
let parsed: any = null;
|
|
if (text) {
|
|
try { parsed = JSON.parse(text); } catch { parsed = text; }
|
|
}
|
|
return { status: res.status, body: parsed };
|
|
};
|
|
|
|
let res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
|
if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status);
|
|
if (res.status === 400 || res.status === 422) {
|
|
// The forge still holds a token by this name whose value we no longer have — the kept file
|
|
// went while the forge's data stayed, or two processes once minted at once and left two. It is
|
|
// ours to replace: find every one by this name in the account's list and delete each by its id
|
|
// (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint.
|
|
const held: number[] = [];
|
|
for (let page = 1; page <= 100; page++) {
|
|
const listed = await call("GET", `?page=${page}&limit=50`);
|
|
if (listed.status !== 200 || !Array.isArray(listed.body)) {
|
|
throw new Error(`Gitea GET /users/${this.opts.admin}/tokens: ${listed.status} ${detail(listed.body)}`);
|
|
}
|
|
if (listed.body.length === 0) break;
|
|
for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id);
|
|
}
|
|
this.log(held.length === 1
|
|
? `the forge already holds a token named "${this.name}" — replacing it`
|
|
: `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`);
|
|
for (const id of held) {
|
|
const dropped = await call("DELETE", `/${id}`);
|
|
if (dropped.status !== 204 && dropped.status !== 404) {
|
|
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`);
|
|
}
|
|
}
|
|
res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
|
}
|
|
if (res.status !== 201 && res.status !== 200) {
|
|
throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} ${detail(res.body)}`);
|
|
}
|
|
const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null;
|
|
if (!token) throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} but no token in the reply`);
|
|
|
|
this.keep(token);
|
|
this.held = token;
|
|
this.log(`minted a token for "${this.opts.admin}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`);
|
|
return token;
|
|
}
|
|
}
|
|
|
|
function detail(body: unknown): string {
|
|
return typeof body === "string" ? body : JSON.stringify(body);
|
|
}
|