Files
mesh-sdk/test/standing.test.ts
T
jschoubben f90189353c Fold H2's new tests into existing ones: go/ at 327 unit tests (from 331), TypeScript at 48 (from 49)
The eight Go test functions H2 added become subtests of existing tests
(catalogue's state test, the bundle's state test, the views test, the process
state test, the testbus bus test, the seal byte-for-byte test), and four more
are folded beside them (two seal, two process watch); the two TypeScript seal
tests become one, and two more TypeScript tests are folded into their
neighbours. No assertion is lost.

Builds: design 47 H2 §819
2026-11-22 02:25:44 +01:00

287 lines
12 KiB
TypeScript

// A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224, issue 272): not on
// the first failure, which may be a restart; after failingAfterMs of failures with no success between;
// again every sayAgainEveryMs while it lasts; and recovered on the first success, or when the consumer
// goes. The same behaviour as the Go copy of the loop's standing_test.go, with the clock shortened.
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtemp, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
runProviderLoop, classOf, FAILING_EVENT, RECOVERED_EVENT, RETIREMENT_EVENT,
CLASS_CREDENTIALS, CLASS_UNREACHABLE, CLASS_SECRET, CLASS_REFUSED, type Adapter,
} from "../dist/provider-loop/index.js";
import { useBroker } from "../dist/messaging/index.js";
const EVERY = 10;
const FAILING_AFTER = 150;
const SAY_AGAIN = 400;
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
async function until(cond: () => boolean, ms = 4000): Promise<void> {
const start = Date.now();
while (!cond()) {
if (Date.now() - start > ms) throw new Error("timed out waiting");
await sleep(5);
}
}
interface Said { event: string; body: Record<string, unknown>; at: number }
interface World {
dir: string;
ask(list: { as: string; node?: string; values?: Record<string, unknown> }[]): Promise<void>;
failing: Error | null;
holdsErr: Error | null;
/** When set, holds fails only for these consumers (holdsErr's error). */
holdsErrFor: Set<string> | null;
created: string[];
/** How many times create was asked, failed or not. */
attempts: number;
/** Every event but the first success since start, which has its own test. */
said: Said[];
all: Said[];
stop(): void;
}
async function world(opts: { classify?: boolean; announce?: false; stableForMs?: number } = {}): Promise<World> {
const dir = await mkdtemp(join(tmpdir(), "standing-"));
const receives = join(dir, "postgres-database.json");
const w = { dir, failing: null, holdsErr: null, holdsErrFor: null, created: [], attempts: 0, said: [], all: [] } as unknown as World;
w.ask = async (list) => {
const given = [];
for (const g of list) {
await writeFile(join(dir, `${g.as}.secret`), `pw-${g.as}\n`);
given.push({ from: g.as, node: g.node ?? "anchor", as: g.as, secret: join(dir, `${g.as}.secret`), values: g.values });
}
await writeFile(receives, JSON.stringify({ contributions: 1, requirement: "postgres-database", given }));
};
const adapter: Adapter = {
async create(p) {
w.attempts++;
if (w.failing) throw w.failing;
w.created.push(p.as);
},
async remove() {},
async retire() {},
async holds(p) {
if (w.holdsErr && (!w.holdsErrFor || w.holdsErrFor.has(p.as))) throw w.holdsErr;
return true;
},
...(opts.classify ? { classify: () => "its-own" } : {}),
};
const was = console.error;
console.error = () => {};
const stop = runProviderLoop("postgres-database", adapter, {
receives, everyMs: EVERY, verifyEveryMs: 0, node: "home-server",
failingAfterMs: FAILING_AFTER, sayAgainEveryMs: SAY_AGAIN, stableForMs: opts.stableForMs ?? 0,
...(opts.announce === false ? {} : {
announce: (event: string, body: Record<string, unknown>) => {
const said = { event, body, at: Date.now() };
w.all.push(said);
if (body.why !== "first-success") w.said.push(said);
},
}),
});
w.stop = () => {
stop();
console.error = was;
void rm(dir, { recursive: true, force: true });
};
return w;
}
const events = (said: Said[]) => said.filter((s) => s.event !== RETIREMENT_EVENT).map((s) => s.event).join(",");
test("a consumer failed for minutes is announced, naming it, its machine and the class, again while it lasts, and recovered", async () => {
const w = await world();
try {
w.failing = new Error(`token request failed: 401 {"error":"invalid_grant"} for pw-a`);
// The loop stamps its own clock before it calls announce, so the test's stamp may read a few ms short: 10 ms of slack.
// "Not before failingAfterMs" and "not again too soon" are read from the clock at the moment each is said, not
// from a sleep: a sleep overruns under load and lets the very announcement it checks the absence of fall due.
const asked = Date.now();
await w.ask([{ as: "a", node: "anchor" }]);
await until(() => w.said.length === 1);
assert.ok(w.said[0].at - asked >= FAILING_AFTER - 10, `announced after ${w.said[0].at - asked} ms, before failingAfterMs`);
const b = w.said[0].body;
assert.equal(w.said[0].event, FAILING_EVENT);
assert.equal(b.consumer, "a");
assert.equal(b.node, "anchor");
assert.equal(b.class, CLASS_CREDENTIALS);
assert.equal(b.provider, "postgres-database");
assert.equal(b["provider-node"], "home-server");
assert.ok((b.attempts as number) > 1, `attempts ${b.attempts}`);
assert.match(String(b.since), /^\d{4}-\d{2}-\d{2}T/);
assert.doesNotMatch(String(b.error), /pw-a/, "the consumer's password travelled");
// Said again while it lasts, not every pass.
await until(() => w.said.length === 2);
assert.equal(events(w.said), `${FAILING_EVENT},${FAILING_EVENT}`);
assert.ok(w.said[1].at - w.said[0].at >= SAY_AGAIN - 10, `repeated after ${w.said[1].at - w.said[0].at} ms, before sayAgainEveryMs`);
// The first success takes it back.
w.failing = null;
await until(() => w.said.length === 3);
assert.equal(w.said[2].event, RECOVERED_EVENT);
assert.equal(w.said[2].body.consumer, "a");
assert.equal(w.said[2].body.node, "anchor");
} finally {
w.stop();
}
});
test("one success between failures starts the run again", async () => {
const w = await world();
try {
// Each failing run lasts two counted failed attempts and at least 0.6 of failingAfterMs of clock (long enough that a success which did not reset the run would be announced at the second run). A run the load
// stretched to failingAfterMs is a long run and may be announced, so only runs that stayed short are asserted.
const run = async (values?: Record<string, unknown>): Promise<number> => {
const start = Date.now();
const base = w.attempts;
w.failing = new Error("connection refused");
await w.ask([{ as: "a", values }]);
await until(() => w.attempts >= base + 2 && Date.now() - start >= FAILING_AFTER * 0.6);
w.failing = null;
return Date.now() - start;
};
const first = await run();
await until(() => w.created.length === 1);
const second = await run({ name: "changed" });
if (first < FAILING_AFTER && second < FAILING_AFTER) assert.equal(events(w.said), "", "two short runs are not one long one");
} finally {
w.stop();
}
});
// The check that failed for a day on 2026-10-05 (issue 179): consumers already made, every check
// refused. A check that cannot be asked is a failure too.
test("a check that keeps failing is a failure too", async () => {
const w = await world();
try {
await w.ask([{ as: "a" }]);
await until(() => w.created.length === 1);
w.holdsErr = new Error("401 invalid_grant");
await until(() => w.said.length === 1);
assert.equal(w.said[0].event, FAILING_EVENT);
assert.equal(w.said[0].body.class, CLASS_CREDENTIALS);
w.holdsErr = null;
await until(() => w.said.length === 2);
assert.equal(events(w.said), `${FAILING_EVENT},${RECOVERED_EVENT}`);
} finally {
w.stop();
}
});
test("an unreadable secret is announced as such", async () => {
const w = await world();
try {
await w.ask([{ as: "a" }]);
await rm(join(w.dir, "a.secret"));
await until(() => w.said.length === 1);
assert.equal(w.said[0].event, FAILING_EVENT);
assert.equal(w.said[0].body.class, CLASS_SECRET);
} finally {
w.stop();
}
});
test("a consumer no longer asked for is no longer failing", async () => {
const w = await world();
try {
w.failing = new Error("boom");
await w.ask([{ as: "a" }, { as: "b" }]);
await until(() => w.said.length === 2);
assert.equal(events(w.said), `${FAILING_EVENT},${FAILING_EVENT}`);
await w.ask([{ as: "a" }]);
await until(() => w.said.some((s) => s.event === RECOVERED_EVENT));
const last = w.said.find((s) => s.event === RECOVERED_EVENT)!;
assert.equal(last.body.consumer, "b");
assert.equal(last.body.why, "no longer asked for");
} finally {
w.stop();
}
});
test("a failing consumer the mesh stopped asking for is said recovered when it is retired", async () => {
const w = await world();
try {
await w.ask([{ as: "a" }, { as: "b" }, { as: "c" }]);
await until(() => w.created.length === 3);
w.holdsErr = new Error("boom");
await until(() => w.said.filter((s) => s.event === FAILING_EVENT).length >= 3);
// c is no longer asked for while it is still failing; the others then recover. c keeps failing, so its only way
// to be said recovered is its retirement: clearing every failure at once raced the loop's read of the new list
// under load (a pass before that read recovered c as an ordinary recovery, with no why).
await w.ask([{ as: "a" }, { as: "b" }]);
w.holdsErrFor = new Set(["c"]);
await until(() => w.said.some((s) => s.event === RECOVERED_EVENT && s.body.consumer === "c"));
const c = w.said.find((s) => s.event === RECOVERED_EVENT && s.body.consumer === "c")!;
assert.equal(c.body.why, "retired");
} finally {
w.stop();
}
});
test("an error is classed by its words, and an adapter that classes its own errors is believed", async () => {
const table: Record<string, string> = {
'Keycloak token request failed: 401 {"error":"invalid_grant"}': CLASS_CREDENTIALS,
'FATAL: password authentication failed for user "postgres"': CLASS_CREDENTIALS,
"dial tcp 127.0.0.1:5432: connect: connection refused": CLASS_UNREACHABLE,
"context deadline exceeded": CLASS_UNREACHABLE,
"Error: connect ECONNREFUSED 127.0.0.1:9000": CLASS_UNREACHABLE,
"getaddrinfo ENOTFOUND minio": CLASS_UNREACHABLE,
'extension "nope" is not available': CLASS_REFUSED,
};
for (const [text, want] of Object.entries(table)) assert.equal(classOf(text), want, text);
const w = await world({ classify: true });
try {
w.failing = new Error("anything");
await w.ask([{ as: "a" }]);
await until(() => w.said.length === 1);
assert.equal(w.said[0].body.class, "its-own");
} finally {
w.stop();
}
});
// A provider restarted after announcing a failure has forgotten it; its first success for each
// consumer is announced, so the controller clears what it kept rather than naming it for ever.
test("the first success since start is announced once", async () => {
const w = await world();
try {
await w.ask([{ as: "a" }]);
await until(() => w.all.length >= 1);
await sleep(EVERY * 10);
assert.equal(w.all.length, 1, JSON.stringify(w.all));
assert.equal(w.all[0].event, RECOVERED_EVENT);
assert.equal(w.all[0].body.why, "first-success");
assert.equal(w.all[0].body.consumer, "a");
} finally {
w.stop();
}
});
// No provider on main passed `announce`, so none said anything (issue 272): the loop says it through
// the SDK's own emit unless it is given another way.
test("without announce the standing is said through the SDK's emit", async () => {
const published: { key: string; body: Record<string, unknown> }[] = [];
useBroker(() => ({
async publish(env: { key: string; body: Record<string, unknown> }) { published.push({ key: env.key, body: env.body }); },
async subscribe() { return () => {}; },
}) as never);
const w = await world({ announce: false });
try {
w.failing = new Error("connection refused");
await w.ask([{ as: "a" }]);
await until(() => published.some((p) => p.key === FAILING_EVENT));
const p = published.find((x) => x.key === FAILING_EVENT)!;
assert.equal(p.body.consumer, "a");
assert.equal(p.body.class, CLASS_UNREACHABLE);
} finally {
w.stop();
}
});