Both caught a failed write and took the event, losing it silently; the SDK's rule is to throw when the work was not done. A failed write now throws so the bus offers the event again, and is spooled on disk at once; on its last delivery the spooled event is taken, and a background pass replays the spool once writing works. The runtime does not pass the delivery count, so the spool counts failed deliveries itself, across restarts. Over its bound (1000 events or 30 minutes) the last delivery is no longer taken, so the bus gives it up and the controller raises max-deliveries - the one existing condition that names a consumer which cannot keep up - while the spool still holds it. Writes are idempotent by event: the trail skips an id it already wrote; the usage upsert keeps the reading observed latest (migration 2), so a late replay never overwrites a newer one. Each module has a status tool for the spool, declared as valuable data (ADR 0233). model-usage moves to the bundle shape (ADR 0198) with its schema in a prepare step and numbered migrations; its old container shape had no image. Both on mesh-sdk 0.1.13. The log-only handlers of redis, mssql, mosquitto, mongodb, mesh-vault, showcase and the catalogue no longer throw a TypeError on an event without a body.
209 lines
8.7 KiB
TypeScript
209 lines
8.7 KiB
TypeScript
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtemp, readFile, readdir } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
import { useBroker } from "@novox/mesh-sdk/messaging";
|
|
import { emit, on, type Event } from "@novox/mesh-sdk/events";
|
|
import { Trail } from "../dist/audit.js";
|
|
import { Spool, takeOrSpool } from "../dist/spool.js";
|
|
import { getAuditTools } from "../dist/tools/index.js";
|
|
|
|
async function lines(path: string): Promise<Record<string, any>[]> {
|
|
const text = await readFile(path, "utf8").catch(() => "");
|
|
return text.trim() === "" ? [] : text.trim().split("\n").map((l) => JSON.parse(l));
|
|
}
|
|
|
|
function ev(id: string, body: unknown = { n: 1 }): Event {
|
|
return { type: "umami.site.created", id, source: "umami", node: "anchor", at: "2026-10-06T10:00:00.000Z", body };
|
|
}
|
|
|
|
async function setup() {
|
|
const dir = await mkdtemp(join(tmpdir(), "audit-"));
|
|
const trail = await Trail.open(join(dir, "audit.log"));
|
|
const spool = await Spool.open(join(dir, "spool"));
|
|
const store = { broken: false };
|
|
const write = async (e: Event) => {
|
|
if (store.broken) throw new Error("ENOSPC: no space left on device");
|
|
await trail.record(e);
|
|
};
|
|
const said: string[] = [];
|
|
const handle = (e: Event) => takeOrSpool(e, write, spool, (l) => said.push(l));
|
|
return { dir, trail, spool, store, write, handle, said, path: join(dir, "audit.log") };
|
|
}
|
|
|
|
test("audit-logger records every event to the trail as one line each", async () => {
|
|
const broker = memBroker();
|
|
useBroker(() => broker);
|
|
const { handle, path } = await setup();
|
|
|
|
// The audit-logger's whole behaviour: consume everything, record it.
|
|
await on("#", handle); // the pattern index.ts subscribes
|
|
|
|
process.env.MESH_MODULE = "umami";
|
|
process.env.MESH_NODE = "anchor";
|
|
await emit("site.created", { domain: "my-app" });
|
|
await emit("node.anchor.joined", { role: "worker" }); // a node event, not a module one
|
|
|
|
const got = await lines(path);
|
|
assert.equal(got.length, 2);
|
|
// A module names its events locally (design 29); the module is the `source`, which together with
|
|
// the type says whose event it was. This broker does no namespacing, so the type is as emitted.
|
|
assert.deepEqual(got.map((l) => l.type), ["site.created", "node.anchor.joined"]);
|
|
assert.equal(got[0].source, "umami");
|
|
assert.equal(got[0].node, "anchor");
|
|
assert.equal(got[0].body.domain, "my-app");
|
|
|
|
delete process.env.MESH_MODULE;
|
|
delete process.env.MESH_NODE;
|
|
});
|
|
|
|
test("a write that fails is thrown, so the bus offers the event again — and it is spooled at once", async () => {
|
|
const broker = memBroker();
|
|
useBroker(() => broker);
|
|
const { handle, store, spool, path } = await setup();
|
|
await on("#", handle);
|
|
store.broken = true;
|
|
await assert.rejects(emit("site.created", { domain: "x" }), /ENOSPC/);
|
|
assert.equal(spool.size, 1, "spooled on the first failure");
|
|
assert.equal(spool.list()[0].attempts, 1);
|
|
assert.equal(spool.list()[0].held, false);
|
|
assert.deepEqual(await lines(path), []);
|
|
});
|
|
|
|
test("a redelivered event is written once — also after a restart, and also when its earlier failure was spooled", async () => {
|
|
const { handle, store, spool, path, dir } = await setup();
|
|
await handle(ev("e1"));
|
|
await handle(ev("e1")); // the answer was lost; the bus offers it again
|
|
assert.equal((await lines(path)).length, 1);
|
|
|
|
// A restart between writing and answering: the trail remembers the ids at its end.
|
|
const again = await Trail.open(path);
|
|
await again.record(ev("e1"));
|
|
assert.equal((await lines(path)).length, 1);
|
|
|
|
// Failed once, then written: the spooled copy goes.
|
|
store.broken = true;
|
|
await assert.rejects(handle(ev("e2")));
|
|
assert.equal(spool.size, 1);
|
|
store.broken = false;
|
|
await handle(ev("e2"));
|
|
assert.equal(spool.size, 0);
|
|
assert.deepEqual(await readdir(join(dir, "spool")), []);
|
|
assert.deepEqual((await lines(path)).map((l) => l.id), ["e1", "e2"]);
|
|
});
|
|
|
|
test("on its last delivery a failed event is spooled and taken", async () => {
|
|
const { handle, store, spool, said } = await setup();
|
|
store.broken = true;
|
|
for (let i = 1; i < 5; i++) await assert.rejects(handle(ev("e3")), /ENOSPC/, `delivery ${i} asks again`);
|
|
await handle(ev("e3")); // the fifth — the controller's max-deliver — returns: taken
|
|
const [entry] = spool.list();
|
|
assert.equal(entry.attempts, 5);
|
|
assert.equal(entry.held, true);
|
|
assert.match(said.at(-1)!, /spooled and taken/);
|
|
});
|
|
|
|
test("the delivery count survives a restart, because it is the spool's", async () => {
|
|
const { handle, store, dir } = await setup();
|
|
store.broken = true;
|
|
for (let i = 0; i < 3; i++) await assert.rejects(handle(ev("e4")));
|
|
const reopened = await Spool.open(join(dir, "spool"));
|
|
assert.equal(reopened.list()[0].attempts, 3);
|
|
});
|
|
|
|
test("the spool replays into the trail once writing works, once each", async () => {
|
|
const { handle, store, spool, write, path } = await setup();
|
|
store.broken = true;
|
|
for (let i = 0; i < 5; i++) await handle(ev("e5")).catch(() => {});
|
|
for (let i = 0; i < 5; i++) await handle(ev("e6")).catch(() => {});
|
|
assert.equal(spool.size, 2);
|
|
|
|
// Still broken: the pass stops at the first failure and keeps everything.
|
|
let r = await spool.replay(write);
|
|
assert.equal(r.replayed, 0);
|
|
assert.equal(spool.size, 2);
|
|
|
|
store.broken = false;
|
|
r = await spool.replay(write);
|
|
assert.deepEqual(r, { replayed: 2, left: 0 });
|
|
await handle(ev("e5")); // a late redelivery after the replay
|
|
assert.deepEqual((await lines(path)).map((l) => l.id), ["e5", "e6"]);
|
|
});
|
|
|
|
test("over its bound the spool keeps the event but does not take it, so the bus gives it up and says so", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "audit-"));
|
|
const spool = await Spool.open(join(dir, "spool"), { boundCount: 1 });
|
|
const failing = async () => {
|
|
throw new Error("down");
|
|
};
|
|
for (let i = 0; i < 5; i++) await takeOrSpool(ev("a"), failing, spool, () => {}).catch(() => {});
|
|
assert.equal(spool.list()[0].held, true, "within the bound: taken");
|
|
for (let i = 1; i < 5; i++) await assert.rejects(takeOrSpool(ev("b"), failing, spool, () => {}));
|
|
await assert.rejects(takeOrSpool(ev("b"), failing, spool, () => {}), /down/, "over the bound: asked again on its last");
|
|
assert.equal(spool.size, 2, "and still spooled");
|
|
const st = spool.standing();
|
|
assert.equal(st.overBound, true);
|
|
assert.equal(st.spooled, 2);
|
|
assert.equal(st.held, 1);
|
|
assert.match(st.why!, /more than 1/);
|
|
});
|
|
|
|
test("an event without a body, or without an id, is recorded once", async () => {
|
|
const { handle, path } = await setup();
|
|
await handle(ev("e7", null));
|
|
await handle(ev("", undefined));
|
|
await handle(ev("", undefined));
|
|
const got = await lines(path);
|
|
assert.equal(got.length, 2);
|
|
assert.equal(got[0].body, null);
|
|
});
|
|
|
|
test("an event waiting longer than the bound puts the spool over it", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "audit-"));
|
|
let now = new Date("2026-10-06T10:00:00Z");
|
|
const spool = await Spool.open(join(dir, "spool"), { now: () => now });
|
|
await takeOrSpool(ev("old"), async () => { throw new Error("down"); }, spool, () => {}).catch(() => {});
|
|
assert.equal(spool.standing().overBound, false);
|
|
now = new Date("2026-10-06T10:31:00Z");
|
|
assert.match(spool.standing().why!, /longer than 30 minutes/);
|
|
});
|
|
|
|
test("audit_status says what waits in the spool, read from disk by the tool's own process", async () => {
|
|
const { handle, store, dir } = await setup();
|
|
store.broken = true;
|
|
for (let i = 0; i < 5; i++) await handle(ev("s1")).catch(() => {});
|
|
await handle(ev("s2")).catch(() => {});
|
|
const [tool] = getAuditTools({ AUDIT_LOG: join(dir, "audit.log"), AUDIT_SPOOL: join(dir, "spool") });
|
|
assert.equal(tool.name, "audit_status");
|
|
const got = (await tool.run({}, {} as never)) as { spool: { spooled: number; held: number; lastError: string } };
|
|
assert.equal(got.spool.spooled, 2);
|
|
assert.equal(got.spool.held, 1);
|
|
assert.match(got.spool.lastError, /ENOSPC/);
|
|
});
|
|
|
|
function memBroker() {
|
|
const subs: { pattern: string; handler: (env: { key: string; node: string; body: unknown }) => Promise<void> }[] = [];
|
|
return {
|
|
async request() {
|
|
throw new Error("unused");
|
|
},
|
|
async handle() {
|
|
return () => {};
|
|
},
|
|
async publish<T>(env: { key: string; node: string; body: T }): Promise<void> {
|
|
for (const s of subs) if (match(s.pattern, env.key)) await s.handler(env);
|
|
},
|
|
async subscribe<T>(pattern: string, handler: (env: { key: string; node: string; body: T }) => Promise<void>): Promise<() => void> {
|
|
subs.push({ pattern, handler: handler as (env: { key: string; node: string; body: unknown }) => Promise<void> });
|
|
return () => {};
|
|
},
|
|
async close() {},
|
|
};
|
|
}
|
|
|
|
function match(pattern: string, key: string): boolean {
|
|
return pattern === "#" || pattern === key;
|
|
}
|