// What the alert watcher does when Grafana refuses the vault's admin password (the sweep of 2026-10-30, issue 774: // a 401 every 30 s on ace since at least 2026-10-05, 145 times in the sweep's window, said only in the journal). // // The cause on ace: Grafana keeps the admin password it was first given in its own database, and applies // GF_SECURITY_ADMIN_PASSWORD__FILE only when it creates that database. ace's Grafana was adopted with its database // before the mesh minted `admin` (2026-09-30), so the minted value never took (the same kind as keycloak's admin, // hq issue 179). Asking again every 30 s changes nothing until the password in Grafana's database is the vault's. // So a refusal is said once, with that cause and what ends it, and asked again hourly, not every look; an answer // after it is said too. /** How long a refused credential waits before the watcher asks again. */ export const AskAgainRefusedMs = 60 * 60_000; /** Whether err is Grafana refusing the credential (the client's "Grafana API : 401 ..."). */ export function refusedCredential(err: unknown): boolean { return /^Error: Grafana API \S+: 401 |^Grafana API \S+: 401 /.test(String(err)); } export class Refusal { private since: number | undefined; private lastAsk = 0; /** Whether the watcher asks Grafana now. */ due(now: number): boolean { return this.since === undefined || now - this.lastAsk >= AskAgainRefusedMs; } /** What to say after one ask at now: its error, the refusal once, the recovery once; undefined for nothing. */ after(now: number, err?: unknown): string | undefined { this.lastAsk = now; if (err === undefined) { if (this.since === undefined) return undefined; this.since = undefined; return "[grafana] Grafana accepts the vault's admin password again: watching for firing alerts every 30 s"; } if (!refusedCredential(err)) return `[grafana] ${err}`; if (this.since !== undefined) return undefined; this.since = now; return ( "[grafana] Grafana refuses the vault's admin password (401), so no firing alert is seen; asked again hourly. " + "Grafana keeps the admin password it was first given in its database and takes the vault's only when it creates " + "that database: an install adopted with its database keeps its old one. It ends when Grafana's admin password is " + "the vault's (grafana cli admin reset-admin-password in the container, with the value of own//grafana/admin)." ); } }