jschoubben
0e6ca453be
Grafana on ace answers 401 to the vault's admin password, every 30 s since at least 2026-10-05. I think the cause is the adoption: Grafana keeps the admin password it was first given in its database and applies GF_SECURITY_ADMIN_PASSWORD__FILE only when it creates it; ace's Grafana came with its database before the mesh minted own/ace/grafana/admin (2026-09-30, generation 1, unchanged since), so the minted value never took (keycloak's admin, hq issue 179, is the same kind). Quick fix: none in code; the password in Grafana's database must become the vault's (grafana cli admin reset-admin-password in the container), an act on ace no tool offers yet. Thorough fix: a step that converges the admin password at every start, which needs a container entrypoint the node-engine's declaration does not carry. This commit does what the catalogue can now: the refusal is said once with its cause and what ends it, asked again hourly, and its end is said.
66 lines
2.4 KiB
TypeScript
66 lines
2.4 KiB
TypeScript
// grafana's events. The tool runtime imports this once the broker is bound. It watches unified
|
|
// alerting and announces when an alert instance starts firing.
|
|
//
|
|
// Emits (novox/hq ADR 0041/0042):
|
|
// module.grafana.alert.firing — an alert instance entered the Alerting state
|
|
//
|
|
// A Grafana with no alerting configured simply never has a firing alert, so this observes nothing
|
|
// and emits nothing — no error, no noise.
|
|
|
|
import { emit } from "@novox/mesh-sdk/events";
|
|
import { GrafanaClient, type GrafanaAlert } from "./client.js";
|
|
import { Refusal } from "./refused.js";
|
|
|
|
// Constructed lazily so an unconfigured node (no auth) loads this entrypoint without crashing the
|
|
// events host — it simply watches nothing.
|
|
let grafana: GrafanaClient | undefined;
|
|
try {
|
|
grafana = GrafanaClient.fromEnv();
|
|
} catch (err) {
|
|
console.log(`[grafana] not configured, not watching alerts: ${err}`);
|
|
}
|
|
|
|
// Firing alerts, by diffing the set currently in the Alerting state. Primed silently on the first
|
|
// look so alerts already firing when this started are not announced as freshly firing.
|
|
const firing = new Set<string>();
|
|
let primed = false;
|
|
|
|
const alertKey = (a: GrafanaAlert): string =>
|
|
`${a.name}:${Object.entries(a.labels).sort().map(([k, v]) => `${k}=${v}`).join(",")}`;
|
|
|
|
async function pollAlerts(client: GrafanaClient): Promise<void> {
|
|
const now = new Set<string>();
|
|
const byKey = new Map<string, GrafanaAlert>();
|
|
for (const a of await client.listAlerts()) {
|
|
if (a.state.toLowerCase() !== "alerting") continue;
|
|
const key = alertKey(a);
|
|
now.add(key);
|
|
byKey.set(key, a);
|
|
}
|
|
if (primed) {
|
|
for (const key of now) {
|
|
if (!firing.has(key)) {
|
|
const a = byKey.get(key)!;
|
|
await emit("alert.firing", { name: a.name, labels: a.labels, activeAt: a.activeAt });
|
|
}
|
|
}
|
|
}
|
|
firing.clear();
|
|
for (const key of now) firing.add(key);
|
|
primed = true;
|
|
}
|
|
|
|
if (grafana) {
|
|
const client = grafana;
|
|
// A refused credential is said once and asked again hourly (refused.ts), not every 30 s.
|
|
const refusal = new Refusal();
|
|
const say = (line: string | undefined): void => { if (line) console.error(line); };
|
|
const run = (): void => {
|
|
if (!refusal.due(Date.now())) return;
|
|
void pollAlerts(client).then(() => say(refusal.after(Date.now())), (err) => say(refusal.after(Date.now(), err)));
|
|
};
|
|
setInterval(run, 30_000);
|
|
run();
|
|
console.log("[grafana] watching for firing alerts");
|
|
}
|