From d49d3a14d82a5dd83f7d9e683a969287e96875cd Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 08:26:33 +0200 Subject: [PATCH] 0.1.14: the provider loop says a consumer it keeps failing on the bus, and its recovery (hq ADR 0224, issue 272) No provider on runProvisioner passed announce, and the loop had no standing per consumer, so the object store, the SQL and document stores, the mail server and the others failed their consumers in their journals alone. The loop now keeps the Go copy's standing (failing after five minutes, again every fifteen, recovered on success, when unasked or retired, and on the first success since start) and says it through the SDK's emit unless given another announce. --- README.md | 12 +- package-lock.json | 4 +- package.json | 2 +- src/provisioner/index.ts | 171 ++++++++++++++++++++++-- test/retire.test.ts | 3 +- test/standing.test.ts | 272 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 448 insertions(+), 16 deletions(-) create mode 100644 test/standing.test.ts diff --git a/README.md b/README.md index bdb0b69..81b4e03 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,21 @@ never called to retire**; it is the delete of an adapter without `delete`, reach person's `provisioner_delete`. A stable set of more than 3, or more than half of those held where more than one is held, is not retired without a person: the provider waits for `provisioner_retire_approve` or `provisioner_retire_reject`, and says `provisioner.retirement` -(`waiting`) through the `announce` option. Asked for again, a retired consumer is created — re-enabled +(`waiting`). Asked for again, a retired consumer is created — re-enabled — at once. Given `module`, the four tools (`provisioner_retirement`, `provisioner_retire_approve`, `provisioner_retire_reject`, `provisioner_delete`) are registered for it; `provisionerTools(handle)` builds them otherwise. Additions and changes are never delayed. +**A provider that keeps failing a consumer says so on the bus** (novox/hq ADR 0224). A consumer whose +create, check (`holds`) or secret has failed for five minutes without one success between +(`failingAfterMs`) is announced `provisioner.failing` — the consumer, its machine, the class of error +(`credentials-rejected`, `unreachable`, `secret-unreadable`, `refused`; an adapter's `classify` is +believed over the words), the error with the password removed, since when and how many attempts — and +again every fifteen minutes while it lasts. The first success after it is `provisioner.recovered`, as is +a consumer no longer asked for or retired, and the first success for each consumer since the process +started. Every event goes through the `announce` option, which defaults to the SDK's `emit`: a provider +says them without asking, and a failed publish is said in its journal, never failing the act. + ### `tools` — the tool-serving harness `registerModuleTools`, the `ToolDefinition` type, and the registry/worker that loads a module's tools and exposes them through the mesh's command surface. *How* a tool is declared and served is diff --git a/package-lock.json b/package-lock.json index a109d0a..6f9a353 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@novox/mesh-sdk", - "version": "0.1.13", + "version": "0.1.14", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@novox/mesh-sdk", - "version": "0.1.13", + "version": "0.1.14", "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" diff --git a/package.json b/package.json index 43be6bf..cd796d0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@novox/mesh-sdk", - "version": "0.1.13", + "version": "0.1.14", "description": "The stable spine a Novox Mesh module's own code builds against.", "type": "module", "exports": { diff --git a/src/provisioner/index.ts b/src/provisioner/index.ts index 3514e17..8a8d77f 100644 --- a/src/provisioner/index.ts +++ b/src/provisioner/index.ts @@ -10,10 +10,20 @@ // does not hand one back — the consumer already receives its copy through the mesh's own channel. // This is why the old $MESH_SEAL_KEY, the symmetric seal, and the *.grant.json / *.credential files // are gone: they described a second, contradictory credential path the mesh does not have. +// +// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer +// whose create, check or secret has failed without one success in between for `failingAfterMs` is +// announced as `provisioner.failing` — naming the consumer, its machine and the class of error — and +// again every `sayAgainEveryMs` while it lasts; the first success after that is +// `provisioner.recovered`, as is a consumer the mesh stopped asking for, and the first success for +// each consumer since this process started. The controller keeps the newest per provider and consumer +// and `status` names it. Until novox/hq issue 272 only the Go copy of this loop (postgres, keycloak) +// said it; every provider on this one failed its consumers in its journal alone. import { readFile } from "node:fs/promises"; import type { ToolDefinition } from "../contracts/index.js"; +import { emit } from "../events/index.js"; import { registerModuleTools } from "../tools/index.js"; /** One consumer's resource to bring into being — everything the mesh derived and delivered. */ @@ -86,6 +96,9 @@ export interface Adapter { /** Optional: every consumer the backend holds ACTIVE that the mesh made, read at start, so one the * mesh stopped asking for while this process was down is still retired (by the same rules). */ held?(): Promise; + /** Optional: the class of one of the adapter's own errors (novox/hq ADR 0224), when its code knows + * better than the error's words; "" or undefined falls back to `classOf` of the text. */ + classify?(err: unknown): string | undefined; } /** One retired consumer, as `provisioner_retirement` answers it. */ @@ -139,6 +152,21 @@ export type Provisioner = (() => void) & { /** The one event a provider says about retirement (novox/hq ADR 0230). */ export const RETIREMENT_EVENT = "provisioner.retirement"; +/** The events a provider's standing toward one consumer is announced as (novox/hq ADR 0224). The + * controller derives the permission to emit them for every module that receives contributions. */ +export const FAILING_EVENT = "provisioner.failing"; +export const RECOVERED_EVENT = "provisioner.recovered"; + +/** The classes a failing consumer is announced with: what a person reading `status` needs to know + * before reading the journal (novox/hq ADR 0224). An adapter may say better (`classify`). */ +export const CLASS_CREDENTIALS = "credentials-rejected"; +export const CLASS_UNREACHABLE = "unreachable"; +export const CLASS_SECRET = "secret-unreadable"; +export const CLASS_REFUSED = "refused"; + +/** How long a consumer fails without one success before it is announced failing (ADR 0224). */ +export const FAILING_AFTER_MS = 5 * 60_000; + export interface ProvisionerOptions { /** The contributions file the mesh writes for this resource (the module's `receives` path). * Defaults to $MESH_RECEIVES. */ @@ -165,10 +193,16 @@ export interface ProvisionerOptions { /** Nor a set larger than this share of the consumers held, where more than one is held. Defaults * to 0.5. */ retireFraction?: number; - /** How often a waiting retirement is said again, in ms. Defaults to 15 minutes. */ + /** How often a waiting retirement, and a consumer still failing, is said again, in ms. Defaults to + * 15 minutes. */ sayAgainEveryMs?: number; - /** Publishes one of the provider's events (`provisioner.retirement`); usually the SDK's `emit`. - * Without it the provider's retirement is said in its journal only. */ + /** How long, in ms, a consumer fails without one success before it is announced + * `provisioner.failing` (novox/hq ADR 0224). Defaults to FAILING_AFTER_MS, five minutes. */ + failingAfterMs?: number; + /** Publishes one of the provider's events (`provisioner.failing`, `provisioner.recovered`, + * `provisioner.retirement`). Defaults to the SDK's `emit`, so every provider says them on the bus + * without asking (novox/hq issue 272); a failed publish is said in the journal and never fails + * the act it reports. */ announce?: (event: string, body: Record) => Promise | void; /** The machine this provider runs on, said in each event. Defaults to $MESH_NODE. */ node?: string; @@ -227,6 +261,8 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi const retireAtOnce = opts.retireAtOnce ?? 3; const retireFraction = opts.retireFraction ?? 0.5; const sayAgainEveryMs = opts.sayAgainEveryMs ?? 15 * 60_000; + const failingAfterMs = opts.failingAfterMs ?? FAILING_AFTER_MS; + const announce = opts.announce ?? ((event: string, body: Record) => emit(event, body)); const node = opts.node ?? process.env.MESH_NODE ?? ""; const say = (line: string): void => console.error(`[provisioner:${resource}] ${line}`); // Retirement (novox/hq ADR 0230). The unasked set last seen and in how many consecutive readable @@ -245,6 +281,13 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi // itself. Each repeat doubles the wait before asking again, so such a bug costs one re-apply an // hour rather than one a minute, and it is said loudly rather than quietly repeated. const lost = new Map(); + // Each consumer's unbroken run of failures — since when, how often, the last error, and when it was + // last announced (0: not yet) — and every consumer that has succeeded since this process started + // (novox/hq ADR 0224). + const trouble = new Map(); + const cleared = new Set(); // login (`as`) -> what was last applied: the hash that says whether it changed, and the // derivation to hand back on withdrawal, which by then is no longer in the file. @@ -311,6 +354,7 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi `file and who this process runs as (novox/hq issue 225)`, ); } + await failed(g.as, g.node, CLASS_SECRET, `secret not readable (${g.secret}): ${err}`); continue; } waiting.delete(g.as); @@ -328,11 +372,15 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi try { if (await withTimeout(adapter.holds!(p), holdsTimeoutMs)) { lost.delete(g.as); + await succeeded(g.as); continue; } } catch (err) { - // Unable to ask is not evidence of loss. Kept as applied, asked again next time. - console.error(`[provisioner:${resource}] ${g.as}: could not check the backend, will ask again: ${scrub(err, password)}`); + // Unable to ask is not evidence of loss. Kept as applied, asked again next time — but a + // check that keeps failing is a consumer failed all the same (ADR 0224, issue 179). + const text = scrub(err, password); + console.error(`[provisioner:${resource}] ${g.as}: could not check the backend, will ask again: ${text}`); + await failed(g.as, g.node, classed(err, text), text); // A backend that did not answer in time will not answer for the next consumer either, and // each would cost a timeout, so the rest of this pass is not asked. Any other failure may // be this consumer's alone, and the others are still asked. @@ -356,6 +404,7 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi await adapter.reenable({ as: g.as, derived: was.derived ?? p.derived }); } await adapter.create(p); + await succeeded(g.as); applied.set(g.as, { hash: h, derived: p.derived, node: g.node }); if (retired.has(g.as)) { // Asked for again: the ordinary create re-enabled it and cleared its mark (ADR 0230). @@ -376,7 +425,9 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi } } } catch (err) { - console.error(`[provisioner:${resource}] ${g.as}: create failed, will retry: ${scrub(err, password)}`); + const text = scrub(err, password); + console.error(`[provisioner:${resource}] ${g.as}: create failed, will retry: ${text}`); + await failed(g.as, g.node, classed(err, text), text); // A failed create is not a disagreement: asked again at the next check, the count unchanged. if (reapplying !== undefined) lost.set(g.as, { times: reapplying - 1, nextAt: 0 }); } @@ -386,6 +437,12 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi // `stablePasses` passes, and within the bound, or with a person's word (novox/hq ADR 0230). // Only consumers this harness applied, or the backend says the mesh made, are touched. const unasked = [...applied.keys()].filter((as) => !wantByAs.has(as)).sort(); + // A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed + // by anyone: said, so the standing the controller keeps for it is cleared rather than left naming a + // consumer that is gone. One still held is said recovered when it is retired (ADR 0224, 0230). + for (const as of [...trouble.keys()]) { + if (!wantByAs.has(as) && !applied.has(as)) await recovered(as, "no longer asked for"); + } const key = unasked.join(","); if (pending && pending.consumers.join(",") !== key) { say(`the retirement waiting for a person (${pending.consumers.join(", ")}) is no longer what the mesh says; settled, nothing retired`); @@ -464,6 +521,7 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi } applied.delete(as); lost.delete(as); + await recovered(as, "retired"); retired.set(as, { node: was.node, at: at.getTime(), why, derived: was.derived, access: adapter.retire ? "disabled" : "kept" }); done.push(as); } catch (err) { @@ -485,16 +543,86 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi // Said on the bus, never failing the act it reports (the journal still has it). async function tell(change: string, body: Record): Promise { - if (!opts.announce) return; + await said(RETIREMENT_EVENT, { + provider: resource, "provider-node": node, change, bound: bound(applied.size), at: iso(Date.now()), ...body, + }, change); + } + + async function said(event: string, body: Record, what = ""): Promise { try { - await opts.announce(RETIREMENT_EVENT, { - provider: resource, "provider-node": node, change, bound: bound(applied.size), at: iso(Date.now()), ...body, - }); + await announce(event, body); } catch (err) { - say(`could not announce ${RETIREMENT_EVENT} ${change}: ${err}`); + say(`could not announce ${event}${what ? ` ${what}` : ""}: ${err}`); } } + // An error's class: the adapter's word when it has one, else read from the text (ADR 0224). + function classed(err: unknown, text: string): string { + try { + const own = adapter.classify?.(err); + if (own) return own; + } catch { + // An adapter that cannot class its own error is read by the words. + } + // A check that did not answer in time is a backend out of reach, whatever its words. + if (err instanceof HoldsTimeout) return CLASS_UNREACHABLE; + return classOf(text); + } + + // One more failure in a consumer's unbroken run, announced once the run has lasted failingAfterMs, + // then again every sayAgainEveryMs while it lasts (novox/hq ADR 0224). + async function failed(as: string, consumerNode: string | undefined, cls: string, text: string): Promise { + const now = Date.now(); + let s = trouble.get(as); + if (!s) { + s = { since: now, attempts: 0, cls, text, saidAt: 0 }; + trouble.set(as, s); + } + s.node = consumerNode; + s.cls = cls; + s.text = text; + s.attempts++; + if (now - s.since < failingAfterMs) return; + if (s.saidAt !== 0 && now - s.saidAt < sayAgainEveryMs) return; + if (s.saidAt === 0) { + say( + `${as}: FAILING for ${Math.round((now - s.since) / 1000)}s (${s.attempts} attempts, ${cls}): ${text}. ` + + `Announced as ${FAILING_EVENT}; \`status\` names it until it succeeds (novox/hq ADR 0224)`, + ); + } + s.saidAt = now; + await said(FAILING_EVENT, { + provider: resource, "provider-node": node, consumer: as, node: consumerNode ?? "", + class: cls, error: clip(text), since: iso(s.since), attempts: s.attempts, + }); + } + + // A success ends a consumer's run of failures; one that was announced is announced recovered. And + // the first success for a consumer since this process started is announced too, failing or not: a + // provider that announced a failure and was restarted has forgotten it, and without this the + // controller would name the consumer failing for ever after it recovered unheard. + async function succeeded(as: string): Promise { + if (!trouble.has(as) && !cleared.has(as)) { + cleared.add(as); + await said(RECOVERED_EVENT, { provider: resource, "provider-node": node, consumer: as, why: "first-success" }); + return; + } + cleared.add(as); + await recovered(as, ""); + } + + async function recovered(as: string, why: string): Promise { + const s = trouble.get(as); + if (!s) return; + trouble.delete(as); + if (s.saidAt === 0) return; // never announced, so there is nothing to take back + if (!why) say(`${as}: recovered after ${Math.round((Date.now() - s.since) / 1000)}s and ${s.attempts} failed attempt(s)`); + await said(RECOVERED_EVENT, { + provider: resource, "provider-node": node, consumer: as, node: s.node ?? "", + since: iso(s.since), attempts: s.attempts, ...(why ? { why } : {}), + }); + } + // What the backend holds that the mesh made: active ones become held, so one the mesh stopped asking // for while this process was down is still retired, by the same rules; retired ones are known. async function seed(): Promise { @@ -713,6 +841,27 @@ async function readContributions( return (doc.given as Contribution[]).filter((g) => g.as && g.secret); } +/** An error's class read from its text — the words the backends the mesh runs use (novox/hq ADR + * 0224): the same words, in the same order, as the Go copy of this loop reads. */ +export function classOf(text: string): string { + const t = text.toLowerCase(); + const credentials = ["invalid_grant", "invalid user credentials", "password authentication failed", + "authentication failed", "unauthorized", " 401"]; + if (credentials.some((w) => t.includes(w))) return CLASS_CREDENTIALS; + const unreachable = ["connection refused", "no such host", "i/o timeout", "deadline exceeded", + "connection reset", "network is unreachable", "no route to host", "eof", + // Node's own words for the same faults. + "econnrefused", "enotfound", "etimedout", "econnreset", "ehostunreach", "enetunreach", "eai_again"]; + if (unreachable.some((w) => t.includes(w))) return CLASS_UNREACHABLE; + return CLASS_REFUSED; +} + +/** An announced error kept to what belongs in a status line. */ +function clip(text: string): string { + const most = 300; + return text.length <= most ? text : `${text.slice(0, most)}…`; +} + /** A check that did not answer in time: the backend, not the consumer, is the likely cause. */ class HoldsTimeout extends Error {} diff --git a/test/retire.test.ts b/test/retire.test.ts index 63550ee..9d660a4 100644 --- a/test/retire.test.ts +++ b/test/retire.test.ts @@ -64,7 +64,8 @@ async function world( }; w.p = runProvisioner("postgres-database", adapter, { receives, everyMs: EVERY, node: "anchor", stableForMs: opts.stableForMs ?? 0, - announce: (event, body) => { assert.equal(event, RETIREMENT_EVENT); w.events.push(body); }, + // Only retirement is these tests'; the standing (provisioner.failing, .recovered) is standing.test.ts's. + announce: (event, body) => { if (event === RETIREMENT_EVENT) w.events.push(body); }, }); // A pass is counted by the file being read: wait wall time for n passes and some slack. w.done = () => { w.p(); console.error = was; }; diff --git a/test/standing.test.ts b/test/standing.test.ts new file mode 100644 index 0000000..538ae86 --- /dev/null +++ b/test/standing.test.ts @@ -0,0 +1,272 @@ +// A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224, issue 272): not on +// the first failure, which may be a restart; after failingAfterMs of failures with no success between; +// again every sayAgainEveryMs while it lasts; and recovered on the first success, or when the consumer +// goes. The same behaviour as the Go copy of the loop's standing_test.go, with the clock shortened. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + runProvisioner, classOf, FAILING_EVENT, RECOVERED_EVENT, RETIREMENT_EVENT, + CLASS_CREDENTIALS, CLASS_UNREACHABLE, CLASS_SECRET, CLASS_REFUSED, type Adapter, +} from "../dist/provisioner/index.js"; +import { useBroker } from "../dist/messaging/index.js"; + +const EVERY = 10; +const FAILING_AFTER = 150; +const SAY_AGAIN = 400; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +async function until(cond: () => boolean, ms = 4000): Promise { + const start = Date.now(); + while (!cond()) { + if (Date.now() - start > ms) throw new Error("timed out waiting"); + await sleep(5); + } +} + +interface Said { event: string; body: Record } + +interface World { + dir: string; + ask(list: { as: string; node?: string; values?: Record }[]): Promise; + failing: Error | null; + holdsErr: Error | null; + created: string[]; + /** Every event but the first success since start, which has its own test. */ + said: Said[]; + all: Said[]; + stop(): void; +} + +async function world(opts: { classify?: boolean; announce?: false; stableForMs?: number } = {}): Promise { + const dir = await mkdtemp(join(tmpdir(), "standing-")); + const receives = join(dir, "postgres-database.json"); + const w = { dir, failing: null, holdsErr: null, created: [], said: [], all: [] } as unknown as World; + w.ask = async (list) => { + const given = []; + for (const g of list) { + await writeFile(join(dir, `${g.as}.secret`), `pw-${g.as}\n`); + given.push({ from: g.as, node: g.node ?? "anchor", as: g.as, secret: join(dir, `${g.as}.secret`), values: g.values }); + } + await writeFile(receives, JSON.stringify({ contributions: 1, requirement: "postgres-database", given })); + }; + const adapter: Adapter = { + async create(p) { + if (w.failing) throw w.failing; + w.created.push(p.as); + }, + async remove() {}, + async retire() {}, + async holds() { + if (w.holdsErr) throw w.holdsErr; + return true; + }, + ...(opts.classify ? { classify: () => "its-own" } : {}), + }; + const was = console.error; + console.error = () => {}; + const stop = runProvisioner("postgres-database", adapter, { + receives, everyMs: EVERY, verifyEveryMs: 0, node: "home-server", + failingAfterMs: FAILING_AFTER, sayAgainEveryMs: SAY_AGAIN, stableForMs: opts.stableForMs ?? 0, + ...(opts.announce === false ? {} : { + announce: (event: string, body: Record) => { + w.all.push({ event, body }); + if (body.why !== "first-success") w.said.push({ event, body }); + }, + }), + }); + w.stop = () => { + stop(); + console.error = was; + void rm(dir, { recursive: true, force: true }); + }; + return w; +} + +const events = (said: Said[]) => said.filter((s) => s.event !== RETIREMENT_EVENT).map((s) => s.event).join(","); + +test("a consumer failed for minutes is announced, naming it, its machine and the class, again while it lasts, and recovered", async () => { + const w = await world(); + try { + w.failing = new Error(`token request failed: 401 {"error":"invalid_grant"} for pw-a`); + await w.ask([{ as: "a", node: "anchor" }]); + await sleep(FAILING_AFTER / 2); + assert.equal(events(w.said), "", "announced before failingAfterMs"); + await until(() => w.said.length === 1); + const b = w.said[0].body; + assert.equal(w.said[0].event, FAILING_EVENT); + assert.equal(b.consumer, "a"); + assert.equal(b.node, "anchor"); + assert.equal(b.class, CLASS_CREDENTIALS); + assert.equal(b.provider, "postgres-database"); + assert.equal(b["provider-node"], "home-server"); + assert.ok((b.attempts as number) > 3, `attempts ${b.attempts}`); + assert.match(String(b.since), /^\d{4}-\d{2}-\d{2}T/); + assert.doesNotMatch(String(b.error), /pw-a/, "the consumer's password travelled"); + + // Said again while it lasts, not every pass. + await sleep(SAY_AGAIN / 2); + assert.equal(events(w.said), FAILING_EVENT, "repeated too soon"); + await until(() => w.said.length === 2); + assert.equal(events(w.said), `${FAILING_EVENT},${FAILING_EVENT}`); + + // The first success takes it back. + w.failing = null; + await until(() => w.said.length === 3); + assert.equal(w.said[2].event, RECOVERED_EVENT); + assert.equal(w.said[2].body.consumer, "a"); + assert.equal(w.said[2].body.node, "anchor"); + } finally { + w.stop(); + } +}); + +test("one success between failures starts the run again", async () => { + const w = await world(); + try { + w.failing = new Error("connection refused"); + await w.ask([{ as: "a" }]); + await sleep(FAILING_AFTER * 0.6); + w.failing = null; + await until(() => w.created.length === 1); + w.failing = new Error("connection refused"); + await w.ask([{ as: "a", values: { name: "changed" } }]); + await sleep(FAILING_AFTER * 0.6); + assert.equal(events(w.said), "", "two short runs are not one long one"); + } finally { + w.stop(); + } +}); + +// The check that failed for a day on 2026-10-05 (issue 179): consumers already made, every check +// refused. A check that cannot be asked is a failure too. +test("a check that keeps failing is a failure too", async () => { + const w = await world(); + try { + await w.ask([{ as: "a" }]); + await until(() => w.created.length === 1); + w.holdsErr = new Error("401 invalid_grant"); + await until(() => w.said.length === 1); + assert.equal(w.said[0].event, FAILING_EVENT); + assert.equal(w.said[0].body.class, CLASS_CREDENTIALS); + w.holdsErr = null; + await until(() => w.said.length === 2); + assert.equal(events(w.said), `${FAILING_EVENT},${RECOVERED_EVENT}`); + } finally { + w.stop(); + } +}); + +test("an unreadable secret is announced as such", async () => { + const w = await world(); + try { + await w.ask([{ as: "a" }]); + await rm(join(w.dir, "a.secret")); + await until(() => w.said.length === 1); + assert.equal(w.said[0].event, FAILING_EVENT); + assert.equal(w.said[0].body.class, CLASS_SECRET); + } finally { + w.stop(); + } +}); + +test("a consumer no longer asked for is no longer failing", async () => { + const w = await world(); + try { + w.failing = new Error("boom"); + await w.ask([{ as: "a" }, { as: "b" }]); + await until(() => w.said.length === 2); + assert.equal(events(w.said), `${FAILING_EVENT},${FAILING_EVENT}`); + await w.ask([{ as: "a" }]); + await until(() => w.said.some((s) => s.event === RECOVERED_EVENT)); + const last = w.said.find((s) => s.event === RECOVERED_EVENT)!; + assert.equal(last.body.consumer, "b"); + assert.equal(last.body.why, "no longer asked for"); + } finally { + w.stop(); + } +}); + +test("a failing consumer the mesh stopped asking for is said recovered when it is retired", async () => { + const w = await world(); + try { + await w.ask([{ as: "a" }, { as: "b" }, { as: "c" }]); + await until(() => w.created.length === 3); + w.holdsErr = new Error("boom"); + await until(() => w.said.filter((s) => s.event === FAILING_EVENT).length >= 3); + // c is no longer asked for while it is still failing; the others then recover. + await w.ask([{ as: "a" }, { as: "b" }]); + w.holdsErr = null; + await until(() => w.said.some((s) => s.event === RECOVERED_EVENT && s.body.consumer === "c")); + const c = w.said.find((s) => s.event === RECOVERED_EVENT && s.body.consumer === "c")!; + assert.equal(c.body.why, "retired"); + } finally { + w.stop(); + } +}); + +test("an error is classed by its words", () => { + const table: Record = { + 'Keycloak token request failed: 401 {"error":"invalid_grant"}': CLASS_CREDENTIALS, + 'FATAL: password authentication failed for user "postgres"': CLASS_CREDENTIALS, + "dial tcp 127.0.0.1:5432: connect: connection refused": CLASS_UNREACHABLE, + "context deadline exceeded": CLASS_UNREACHABLE, + "Error: connect ECONNREFUSED 127.0.0.1:9000": CLASS_UNREACHABLE, + "getaddrinfo ENOTFOUND minio": CLASS_UNREACHABLE, + 'extension "nope" is not available': CLASS_REFUSED, + }; + for (const [text, want] of Object.entries(table)) assert.equal(classOf(text), want, text); +}); + +test("an adapter that classes its own errors is believed", async () => { + const w = await world({ classify: true }); + try { + w.failing = new Error("anything"); + await w.ask([{ as: "a" }]); + await until(() => w.said.length === 1); + assert.equal(w.said[0].body.class, "its-own"); + } finally { + w.stop(); + } +}); + +// A provider restarted after announcing a failure has forgotten it; its first success for each +// consumer is announced, so the controller clears what it kept rather than naming it for ever. +test("the first success since start is announced once", async () => { + const w = await world(); + try { + await w.ask([{ as: "a" }]); + await until(() => w.all.length >= 1); + await sleep(EVERY * 10); + assert.equal(w.all.length, 1, JSON.stringify(w.all)); + assert.equal(w.all[0].event, RECOVERED_EVENT); + assert.equal(w.all[0].body.why, "first-success"); + assert.equal(w.all[0].body.consumer, "a"); + } finally { + w.stop(); + } +}); + +// No provider on main passed `announce`, so none said anything (issue 272): the loop says it through +// the SDK's own emit unless it is given another way. +test("without announce the standing is said through the SDK's emit", async () => { + const published: { key: string; body: Record }[] = []; + useBroker(() => ({ + async publish(env: { key: string; body: Record }) { published.push({ key: env.key, body: env.body }); }, + async subscribe() { return () => {}; }, + }) as never); + const w = await world({ announce: false }); + try { + w.failing = new Error("connection refused"); + await w.ask([{ as: "a" }]); + await until(() => published.some((p) => p.key === FAILING_EVENT)); + const p = published.find((x) => x.key === FAILING_EVENT)!; + assert.equal(p.body.consumer, "a"); + assert.equal(p.body.class, CLASS_UNREACHABLE); + } finally { + w.stop(); + } +}); -- 2.54.0