Say grafana's refused admin password once with its cause, and ask again hourly, not every 30 s (issue 774)

Grafana on ace answers 401 to the vault's admin password, every 30 s since at least 2026-10-05. I think the cause is
the adoption: Grafana keeps the admin password it was first given in its database and applies
GF_SECURITY_ADMIN_PASSWORD__FILE only when it creates it; ace's Grafana came with its database before the mesh minted
own/ace/grafana/admin (2026-09-30, generation 1, unchanged since), so the minted value never took (keycloak's
admin, hq issue 179, is the same kind).

Quick fix: none in code; the password in Grafana's database must become the vault's (grafana cli admin
reset-admin-password in the container), an act on ace no tool offers yet. Thorough fix: a step that converges the
admin password at every start, which needs a container entrypoint the node-engine's declaration does not carry.
This commit does what the catalogue can now: the refusal is said once with its cause and what ends it, asked again
hourly, and its end is said.

modules/grafana/index.ts

@@ -9,6 +9,7 @@
 
 import { emit } from "@novox/mesh-sdk/events";
 import { GrafanaClient, type GrafanaAlert } from "./client.js";
+import { Refusal } from "./refused.js";
 
 // Constructed lazily so an unconfigured node (no auth) loads this entrypoint without crashing the
 // events host — it simply watches nothing.
@@ -51,7 +52,13 @@ async function pollAlerts(client: GrafanaClient): Promise<void> {
 
 if (grafana) {
   const client = grafana;
-  const run = (): void => void pollAlerts(client).catch((err) => console.error(`[grafana] ${err}`));
+  // A refused credential is said once and asked again hourly (refused.ts), not every 30 s.
+  const refusal = new Refusal();
+  const say = (line: string | undefined): void => { if (line) console.error(line); };
+  const run = (): void => {
+    if (!refusal.due(Date.now())) return;
+    void pollAlerts(client).then(() => say(refusal.after(Date.now())), (err) => say(refusal.after(Date.now(), err)));
+  };
   setInterval(run, 30_000);
   run();
   console.log("[grafana] watching for firing alerts");

modules/grafana/refused.ts

@@ -0,0 +1,46 @@
+// What the alert watcher does when Grafana refuses the vault's admin password (the sweep of 2026-10-30, issue 774:
+// a 401 every 30 s on ace since at least 2026-10-05, 145 times in the sweep's window, said only in the journal).
+//
+// The cause on ace: Grafana keeps the admin password it was first given in its own database, and applies
+// GF_SECURITY_ADMIN_PASSWORD__FILE only when it creates that database. ace's Grafana was adopted with its database
+// before the mesh minted `admin` (2026-09-30), so the minted value never took (the same kind as keycloak's admin,
+// hq issue 179). Asking again every 30 s changes nothing until the password in Grafana's database is the vault's.
+// So a refusal is said once, with that cause and what ends it, and asked again hourly, not every look; an answer
+// after it is said too.
+
+/** How long a refused credential waits before the watcher asks again. */
+export const AskAgainRefusedMs = 60 * 60_000;
+
+/** Whether err is Grafana refusing the credential (the client's "Grafana API <path>: 401 ..."). */
+export function refusedCredential(err: unknown): boolean {
+  return /^Error: Grafana API \S+: 401 |^Grafana API \S+: 401 /.test(String(err));
+}
+
+export class Refusal {
+  private since: number | undefined;
+  private lastAsk = 0;
+
+  /** Whether the watcher asks Grafana now. */
+  due(now: number): boolean {
+    return this.since === undefined || now - this.lastAsk >= AskAgainRefusedMs;
+  }
+
+  /** What to say after one ask at now: its error, the refusal once, the recovery once; undefined for nothing. */
+  after(now: number, err?: unknown): string | undefined {
+    this.lastAsk = now;
+    if (err === undefined) {
+      if (this.since === undefined) return undefined;
+      this.since = undefined;
+      return "[grafana] Grafana accepts the vault's admin password again: watching for firing alerts every 30 s";
+    }
+    if (!refusedCredential(err)) return `[grafana] ${err}`;
+    if (this.since !== undefined) return undefined;
+    this.since = now;
+    return (
+      "[grafana] Grafana refuses the vault's admin password (401), so no firing alert is seen; asked again hourly. " +
+      "Grafana keeps the admin password it was first given in its database and takes the vault's only when it creates " +
+      "that database: an install adopted with its database keeps its old one. It ends when Grafana's admin password is " +
+      "the vault's (grafana cli admin reset-admin-password in the container, with the value of own/<node>/grafana/admin)."
+    );
+  }
+}

modules/grafana/test/refused.test.ts

@@ -0,0 +1,27 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+// The sweep of 2026-10-30 (issue 774): a refused admin password was said every 30 s, forever. Now it is said once with
+// its cause, asked again hourly, and its end is said; any other error is still said each time.
+test("a refused credential is said once, asked again hourly, and its end said", async () => {
+  const { Refusal, AskAgainRefusedMs } = await import("../refused.ts");
+  const r = new Refusal();
+  const refused = new Error('Grafana API /api/prometheus/grafana/api/v1/alerts: 401 {"message":"Invalid username or password"}');
+  const rows: [number, Error | undefined, boolean, RegExp | undefined][] = [
+    // at, error of the ask, due before it, what is said
+    [0, refused, true, /refuses the vault's admin password.*reset-admin-password/],
+    [30_000, undefined, false, undefined],
+    [AskAgainRefusedMs, refused, true, undefined],
+    [AskAgainRefusedMs + 1, new Error("Grafana API /api/health: 502 bad gateway"), false, undefined],
+    [2 * AskAgainRefusedMs, undefined, true, /accepts the vault's admin password again/],
+    [2 * AskAgainRefusedMs + 30_000, new Error("Grafana API /x: 502 bad gateway"), true, /502 bad gateway/],
+    [2 * AskAgainRefusedMs + 60_000, new Error("Grafana API /x: 502 bad gateway"), true, /502 bad gateway/],
+  ];
+  for (const [i, [at, err, due, said]] of rows.entries()) {
+    assert.equal(r.due(at), due, `row ${i}: due`);
+    if (!due) continue;
+    const line = r.after(at, err);
+    if (said) assert.match(line ?? "", said, `row ${i}`);
+    else assert.equal(line, undefined, `row ${i}`);
+  }
+});