diff --git a/modules/grafana/index.ts b/modules/grafana/index.ts index 6e174b23..ab541dd3 100644 --- a/modules/grafana/index.ts +++ b/modules/grafana/index.ts @@ -9,6 +9,7 @@ import { emit } from "@novox/mesh-sdk/events"; import { GrafanaClient, type GrafanaAlert } from "./client.js"; +import { Refusal } from "./refused.js"; // Constructed lazily so an unconfigured node (no auth) loads this entrypoint without crashing the // events host — it simply watches nothing. @@ -51,7 +52,13 @@ async function pollAlerts(client: GrafanaClient): Promise { if (grafana) { const client = grafana; - const run = (): void => void pollAlerts(client).catch((err) => console.error(`[grafana] ${err}`)); + // A refused credential is said once and asked again hourly (refused.ts), not every 30 s. + const refusal = new Refusal(); + const say = (line: string | undefined): void => { if (line) console.error(line); }; + const run = (): void => { + if (!refusal.due(Date.now())) return; + void pollAlerts(client).then(() => say(refusal.after(Date.now())), (err) => say(refusal.after(Date.now(), err))); + }; setInterval(run, 30_000); run(); console.log("[grafana] watching for firing alerts"); diff --git a/modules/grafana/refused.ts b/modules/grafana/refused.ts new file mode 100644 index 00000000..1610a265 --- /dev/null +++ b/modules/grafana/refused.ts @@ -0,0 +1,46 @@ +// What the alert watcher does when Grafana refuses the vault's admin password (the sweep of 2026-10-30, issue 774: +// a 401 every 30 s on ace since at least 2026-10-05, 145 times in the sweep's window, said only in the journal). +// +// The cause on ace: Grafana keeps the admin password it was first given in its own database, and applies +// GF_SECURITY_ADMIN_PASSWORD__FILE only when it creates that database. ace's Grafana was adopted with its database +// before the mesh minted `admin` (2026-09-30), so the minted value never took (the same kind as keycloak's admin, +// hq issue 179). Asking again every 30 s changes nothing until the password in Grafana's database is the vault's. +// So a refusal is said once, with that cause and what ends it, and asked again hourly, not every look; an answer +// after it is said too. + +/** How long a refused credential waits before the watcher asks again. */ +export const AskAgainRefusedMs = 60 * 60_000; + +/** Whether err is Grafana refusing the credential (the client's "Grafana API : 401 ..."). */ +export function refusedCredential(err: unknown): boolean { + return /^Error: Grafana API \S+: 401 |^Grafana API \S+: 401 /.test(String(err)); +} + +export class Refusal { + private since: number | undefined; + private lastAsk = 0; + + /** Whether the watcher asks Grafana now. */ + due(now: number): boolean { + return this.since === undefined || now - this.lastAsk >= AskAgainRefusedMs; + } + + /** What to say after one ask at now: its error, the refusal once, the recovery once; undefined for nothing. */ + after(now: number, err?: unknown): string | undefined { + this.lastAsk = now; + if (err === undefined) { + if (this.since === undefined) return undefined; + this.since = undefined; + return "[grafana] Grafana accepts the vault's admin password again: watching for firing alerts every 30 s"; + } + if (!refusedCredential(err)) return `[grafana] ${err}`; + if (this.since !== undefined) return undefined; + this.since = now; + return ( + "[grafana] Grafana refuses the vault's admin password (401), so no firing alert is seen; asked again hourly. " + + "Grafana keeps the admin password it was first given in its database and takes the vault's only when it creates " + + "that database: an install adopted with its database keeps its old one. It ends when Grafana's admin password is " + + "the vault's (grafana cli admin reset-admin-password in the container, with the value of own//grafana/admin)." + ); + } +} diff --git a/modules/grafana/test/refused.test.ts b/modules/grafana/test/refused.test.ts new file mode 100644 index 00000000..8d88f736 --- /dev/null +++ b/modules/grafana/test/refused.test.ts @@ -0,0 +1,27 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +// The sweep of 2026-10-30 (issue 774): a refused admin password was said every 30 s, forever. Now it is said once with +// its cause, asked again hourly, and its end is said; any other error is still said each time. +test("a refused credential is said once, asked again hourly, and its end said", async () => { + const { Refusal, AskAgainRefusedMs } = await import("../refused.ts"); + const r = new Refusal(); + const refused = new Error('Grafana API /api/prometheus/grafana/api/v1/alerts: 401 {"message":"Invalid username or password"}'); + const rows: [number, Error | undefined, boolean, RegExp | undefined][] = [ + // at, error of the ask, due before it, what is said + [0, refused, true, /refuses the vault's admin password.*reset-admin-password/], + [30_000, undefined, false, undefined], + [AskAgainRefusedMs, refused, true, undefined], + [AskAgainRefusedMs + 1, new Error("Grafana API /api/health: 502 bad gateway"), false, undefined], + [2 * AskAgainRefusedMs, undefined, true, /accepts the vault's admin password again/], + [2 * AskAgainRefusedMs + 30_000, new Error("Grafana API /x: 502 bad gateway"), true, /502 bad gateway/], + [2 * AskAgainRefusedMs + 60_000, new Error("Grafana API /x: 502 bad gateway"), true, /502 bad gateway/], + ]; + for (const [i, [at, err, due, said]] of rows.entries()) { + assert.equal(r.due(at), due, `row ${i}: due`); + if (!due) continue; + const line = r.after(at, err); + if (said) assert.match(line ?? "", said, `row ${i}`); + else assert.equal(line, undefined, `row ${i}`); + } +});