audit-logger, model-usage: retry a failed write and never lose the event (hq issue 276)
Both caught a failed write and took the event, losing it silently; the SDK's rule is to throw when the work was not done. A failed write now throws so the bus offers the event again, and is spooled on disk at once; on its last delivery the spooled event is taken, and a background pass replays the spool once writing works. The runtime does not pass the delivery count, so the spool counts failed deliveries itself, across restarts. Over its bound (1000 events or 30 minutes) the last delivery is no longer taken, so the bus gives it up and the controller raises max-deliveries - the one existing condition that names a consumer which cannot keep up - while the spool still holds it. Writes are idempotent by event: the trail skips an id it already wrote; the usage upsert keeps the reading observed latest (migration 2), so a late replay never overwrites a newer one. Each module has a status tool for the spool, declared as valuable data (ADR 0233). model-usage moves to the bundle shape (ADR 0198) with its schema in a prepare step and numbered migrations; its old container shape had no image. Both on mesh-sdk 0.1.13. The log-only handlers of redis, mssql, mosquitto, mongodb, mesh-vault, showcase and the catalogue no longer throw a TypeError on an event without a body.
This commit is contained in:
jochen
@@ -3,8 +3,9 @@ import { readFileSync } from "node:fs";
|
||||
// session — which differ only in `consumer`; a reading is one row `(licence, consumer, period,
|
||||
// metric, value)` plus its `raw` vendor payload. The store keeps the LATEST reading per
|
||||
// `(licence, consumer, period, metric)`: an event carries a fresh total, and the upsert replaces the
|
||||
// previous one. Delivery is at-least-once, so an upsert is idempotent-latest by design — a duplicate
|
||||
// event writes the same row again, never a second.
|
||||
// previous one — unless the previous one was observed later. Delivery is at-least-once, so an upsert
|
||||
// is idempotent-latest by design: a duplicate event writes the same row again, never a second, and an
|
||||
// older event replayed late changes nothing.
|
||||
//
|
||||
// Usage is stored IN THE CLEAR (ADR 0054): the value and its raw payload are ordinary columns, not
|
||||
// sealed. This is not a credential; it is a reading a query answers.
|
||||
@@ -29,7 +30,20 @@ export interface UsageRow {
|
||||
raw?: unknown;
|
||||
}
|
||||
|
||||
const DDL = `
|
||||
/**
|
||||
* The store's schema, as numbered migrations applied in order and recorded in `usage_schema`. Each is
|
||||
* written to be safe on a store that already has it, because the first version created the table
|
||||
* without recording anything.
|
||||
*
|
||||
* 2 (novox/hq issue 276): a reading now carries the time its event was emitted, and an upsert replaces
|
||||
* a reading only with one emitted no earlier. A failed event is asked for again and, past its last
|
||||
* delivery, replayed from the spool — possibly after a newer reading arrived; it must not overwrite it.
|
||||
*/
|
||||
export const MIGRATIONS: { version: number; why: string; sql: string }[] = [
|
||||
{
|
||||
version: 1,
|
||||
why: "the one table, both grains",
|
||||
sql: `
|
||||
CREATE TABLE IF NOT EXISTS usage (
|
||||
licence text NOT NULL,
|
||||
consumer text NOT NULL,
|
||||
@@ -39,8 +53,32 @@ CREATE TABLE IF NOT EXISTS usage (
|
||||
raw jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (licence, consumer, period, metric)
|
||||
);
|
||||
`;
|
||||
);`,
|
||||
},
|
||||
{
|
||||
version: 2,
|
||||
why: "a reading knows when it was observed and which event said it, so an older one never replaces it",
|
||||
sql: `
|
||||
ALTER TABLE usage ADD COLUMN IF NOT EXISTS observed_at timestamptz;
|
||||
ALTER TABLE usage ADD COLUMN IF NOT EXISTS event_id text;`,
|
||||
},
|
||||
];
|
||||
|
||||
/** The upsert: the latest reading per key, where latest is the event's emit time — so a redelivered or
|
||||
* replayed event writes the row it already wrote, and an older one changes nothing. */
|
||||
export const UPSERT = `
|
||||
INSERT INTO usage (licence, consumer, period, metric, value, raw, observed_at, event_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::jsonb, COALESCE($7::timestamptz, now()), $8)
|
||||
ON CONFLICT (licence, consumer, period, metric)
|
||||
DO UPDATE SET value = excluded.value, raw = excluded.raw, observed_at = excluded.observed_at,
|
||||
event_id = excluded.event_id, updated_at = now()
|
||||
WHERE usage.observed_at IS NULL OR usage.observed_at <= excluded.observed_at`;
|
||||
|
||||
/** Who wrote a reading, and when it was true — the event it came from. */
|
||||
export interface Observed {
|
||||
at?: string;
|
||||
eventId?: string;
|
||||
}
|
||||
|
||||
export class UsageStore {
|
||||
private constructor(private readonly pool: PgPool) {}
|
||||
@@ -51,24 +89,42 @@ export class UsageStore {
|
||||
// As a file first (novox/hq ADR 0086): the connection string carries the password.
|
||||
const url = env["DATABASE_URL"] ?? readMaybe(env["DATABASE_URL_FILE"]);
|
||||
if (!url) throw new Error("DATABASE_URL_FILE (or DATABASE_URL) is not set — model-usage cannot reach its database");
|
||||
return new UsageStore(new Pool({ connectionString: url }));
|
||||
// Bounded, so a store that does not answer is a failed write the handler throws well within the
|
||||
// runtime's two-minute event timeout — not an event left hanging until the runtime gives up on it.
|
||||
return new UsageStore(new Pool({ connectionString: url, connectionTimeoutMillis: 10_000, query_timeout: 30_000 }));
|
||||
}
|
||||
|
||||
/** Create the one table if it is not there. Run once by the migrate entry before the consumer
|
||||
* starts; idempotent, so re-running is harmless. */
|
||||
async migrate(): Promise<void> {
|
||||
await this.pool.query(DDL);
|
||||
}
|
||||
|
||||
/** Upsert a reading, keeping the latest per `(licence, consumer, period, metric)`. */
|
||||
async upsert(row: UsageRow): Promise<void> {
|
||||
/** Bring the schema to the newest migration, recording each applied. Idempotent: run by the
|
||||
* prepare step before this version starts (novox/hq ADR 0135). */
|
||||
async migrate(): Promise<number[]> {
|
||||
await this.pool.query(
|
||||
`INSERT INTO usage (licence, consumer, period, metric, value, raw)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::jsonb)
|
||||
ON CONFLICT (licence, consumer, period, metric)
|
||||
DO UPDATE SET value = excluded.value, raw = excluded.raw, updated_at = now()`,
|
||||
[row.licence, row.consumer, row.period, row.metric, row.value, JSON.stringify(row.raw ?? {})],
|
||||
"CREATE TABLE IF NOT EXISTS usage_schema (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())",
|
||||
);
|
||||
const { rows } = await this.pool.query("SELECT version FROM usage_schema");
|
||||
const have = new Set(rows.map((r) => Number(r.version)));
|
||||
const applied: number[] = [];
|
||||
for (const m of MIGRATIONS) {
|
||||
if (have.has(m.version)) continue;
|
||||
await this.pool.query(m.sql);
|
||||
await this.pool.query("INSERT INTO usage_schema (version) VALUES ($1) ON CONFLICT DO NOTHING", [m.version]);
|
||||
applied.push(m.version);
|
||||
}
|
||||
return applied;
|
||||
}
|
||||
|
||||
/** Upsert a reading, keeping the latest per `(licence, consumer, period, metric)` by when it was
|
||||
* observed. Throws when the store did not take it. */
|
||||
async upsert(row: UsageRow, observed: Observed = {}): Promise<void> {
|
||||
await this.pool.query(UPSERT, [
|
||||
row.licence,
|
||||
row.consumer,
|
||||
row.period,
|
||||
row.metric,
|
||||
row.value,
|
||||
JSON.stringify(row.raw ?? {}),
|
||||
observed.at || null,
|
||||
observed.eventId || null,
|
||||
]);
|
||||
}
|
||||
|
||||
/** The current reading per key, whole or filtered to one licence — for the read-only tool. */
|
||||
|
||||
Reference in New Issue
Block a user