// Retirement (novox/hq ADR 0230): a consumer the mesh stops asking for is retired — access disabled, // data kept — only after the same result in five consecutive passes AND ten minutes (shortened here // unless a test is about it), and a set too large waits for a person. An adapter that cannot disable // is mark-only: remove is never called to retire. Deletion is a person's act, of a retired consumer only. import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdtemp, writeFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { runProvisioner, provisionerTools, RETIREMENT_EVENT, type Provisioner } from "../dist/provisioner/index.js"; import { resetTools, collectTools } from "../dist/tools/index.js"; const EVERY = 10; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); async function until(cond: () => boolean | Promise, ms = 3000): Promise { const start = Date.now(); while (!(await cond())) { if (Date.now() - start > ms) throw new Error("timed out waiting"); await sleep(5); } } interface World { receives: string; ask(names: string[]): Promise; created: string[]; retired: { as: string; why: string }[]; removed: string[]; deleted: string[]; reenabled: string[]; events: Record[]; said: string[]; p: Provisioner; done(): void; } async function world( names: string[], opts: { retire?: boolean; held?: string[]; del?: boolean; reenable?: boolean; stableForMs?: number } = {}, ): Promise { const dir = await mkdtemp(join(tmpdir(), "retire-")); const receives = join(dir, "postgres-database.json"); const ask = async (list: string[]) => { const given = []; for (const n of list) { await writeFile(join(dir, `${n}.secret`), `pw-${n}\n`); given.push({ from: n, node: "anchor", as: n, secret: join(dir, `${n}.secret`) }); } await writeFile(receives, JSON.stringify({ contributions: 1, requirement: "postgres-database", given })); }; await ask(names); const w = { receives, ask, created: [], retired: [], removed: [], deleted: [], reenabled: [], events: [], said: [] } as unknown as World; const was = console.error; console.error = (...a: unknown[]) => w.said.push(a.join(" ")); const adapter = { async create(p: { as: string }) { w.created.push(p.as); }, async remove(p: { as: string }) { w.removed.push(p.as); }, ...(opts.del === false ? {} : { async delete(p: { as: string }) { w.deleted.push(p.as); return { freedBytes: 42 }; } }), ...(opts.reenable ? { async reenable(p: { as: string }) { w.reenabled.push(p.as); } } : {}), ...(opts.retire === false ? {} : { async retire(p: { as: string; why: string }) { w.retired.push({ as: p.as, why: p.why }); } }), ...(opts.held ? { async held() { return opts.held!; } } : {}), }; w.p = runProvisioner("postgres-database", adapter, { receives, everyMs: EVERY, node: "anchor", stableForMs: opts.stableForMs ?? 0, announce: (event, body) => { assert.equal(event, RETIREMENT_EVENT); w.events.push(body); }, }); // A pass is counted by the file being read: wait wall time for n passes and some slack. w.done = () => { w.p(); console.error = was; }; await until(() => w.created.length >= names.length); return w; } const changes = (w: World) => w.events.map((e) => e.change); test("a transient empty list for four passes retires nothing; five stable passes retire, data kept", async () => { const w = await world(["a", "b", "c"], {}); try { // The unasked set is {a} — one of three, within the bound — seen for fewer than five passes. const p = w.p; // Run the loop by hand: stop the timer-driven one and drive passes directly is not exposed, so // the transient is made short in wall time instead: ask without a, then with a, quickly. await w.ask(["b", "c"]); await sleep(EVERY * 2); await w.ask(["a", "b", "c"]); await sleep(EVERY * 8); assert.deepEqual(w.retired, [], "retired on a transient result"); assert.deepEqual(changes(w), []); // Stable: retired, through retire, and not removed. await w.ask(["b", "c"]); await until(() => w.retired.length === 1); assert.equal(w.retired[0].as, "a"); assert.match(w.retired[0].why, /\d+ passes over/); assert.deepEqual(w.removed, []); assert.deepEqual(w.deleted, []); assert.deepEqual(changes(w), ["retired"]); const r = await p.retirement(); assert.deepEqual(r.retired.map((x) => x.consumer), ["a"]); assert.deepEqual(r.held, ["b", "c"]); assert.equal(r.stable_passes, 5); } finally { w.done(); } }); test("the fifth pass and not the fourth retires", async () => { // Driven by the clock: five passes 10 ms apart take at least 40 ms after the first sees the set. const w = await world(["a", "b", "c"]); try { await w.ask(["b", "c"]); const start = Date.now(); await until(() => w.retired.length === 1); assert.ok(Date.now() - start >= 4 * EVERY - 2, `retired after ${Date.now() - start} ms, before five passes`); } finally { w.done(); } }); test("an unreadable file resets the count", async () => { const w = await world(["a", "b", "c"]); try { await w.ask(["b", "c"]); // Interleave unreadable passes so five readable ones never run in a row. for (let i = 0; i < 6; i++) { await sleep(EVERY * 2); await rm(w.receives); await sleep(EVERY * 2); await w.ask(["b", "c"]); } assert.deepEqual(w.retired, [], "unreadable passes counted toward a stable result"); } finally { w.done(); } }); test("asked for again, a retired consumer is created again and announced re-enabled", async () => { const w = await world(["a", "b", "c"]); try { await w.ask(["b", "c"]); await until(() => w.retired.length === 1); const before = w.created.length; await w.ask(["a", "b", "c"]); await until(() => w.created.length > before); assert.equal(w.created.at(-1), "a"); await until(() => changes(w).includes("reenabled")); assert.deepEqual((await w.p.retirement()).retired, []); } finally { w.done(); } }); test("five passes are not enough: the same set must also hold for stableForMs", async () => { const HOLD = 400; const w = await world(["a", "b", "c"], { stableForMs: HOLD }); try { await w.ask(["b", "c"]); const start = Date.now(); await sleep(EVERY * 10); // well past five passes, well short of the hold assert.deepEqual(w.retired, [], "retired after five passes, before the set held long enough"); await until(() => w.retired.length === 1, 3000); assert.ok(Date.now() - start >= HOLD - 5, `retired after ${Date.now() - start} ms, before ${HOLD} ms`); assert.equal((await w.p.retirement()).stable_for_seconds, 0); // 400 ms rounds to 0 s } finally { w.done(); } }); test("the default hold is ten minutes", async () => { const { STABLE_FOR_MS } = await import("../dist/provisioner/index.js"); assert.equal(STABLE_FOR_MS, 10 * 60_000); }); test("an adapter without retire is MARK-ONLY: nothing changes in the backend, remove only on delete", async () => { const w = await world(["a", "b", "c"], { retire: false, del: false }); try { await w.ask(["b", "c"]); await until(() => changes(w).includes("retired")); await sleep(EVERY * 10); assert.deepEqual(w.removed, [], "remove was called to retire"); const ev = w.events.find((e) => e.change === "retired")!; assert.equal((ev.consumers as { access: string }[])[0].access, "kept"); assert.ok(w.said.some((l) => /MARK ONLY/.test(l) && /KEEPS ITS ACCESS/.test(l)), "mark-only was not said loudly"); const r = await w.p.retirement(); assert.equal(r.retires_by, "mark-only"); assert.deepEqual(r.retired.map((x) => [x.consumer, x.access]), [["a", "kept"]]); // Asked again: created again, nothing to undo. await w.ask(["a", "b", "c"]); await until(() => changes(w).includes("reenabled")); assert.deepEqual(w.removed, []); // Gone again, then a person's delete: the one path to remove. await w.ask(["b", "c"]); await until(async () => (await w.p.retirement()).retired.length === 1); await w.p.deleteRetired("a", { why: "gone", confirm: "a" }); assert.deepEqual(w.removed, ["a"]); } finally { w.done(); } }); test("an adapter with retire and reenable: disabled on retire, reenabled before create when asked again", async () => { const w = await world(["a", "b", "c"], { reenable: true }); try { await w.ask(["b", "c"]); await until(() => w.retired.length === 1); assert.equal((await w.p.retirement()).retires_by, "disable"); assert.equal((await w.p.retirement()).retired[0].access, "disabled"); const before = w.created.length; await w.ask(["a", "b", "c"]); await until(() => w.created.length > before); assert.deepEqual(w.reenabled, ["a"]); assert.deepEqual(w.removed, []); } finally { w.done(); } }); test("four of four waits for a person: approve the exact set retires, another set is refused", async () => { const w = await world(["a", "b", "c", "d"]); try { await w.ask([]); await until(async () => (await w.p.retirement()).waiting !== null); await sleep(EVERY * 10); assert.deepEqual(w.retired, [], "retired over the bound without a person"); assert.deepEqual(changes(w), ["waiting"], "waiting said more than once, or not"); const waiting = w.events[0]; assert.equal(waiting.held, 4); assert.match(String(waiting.bound), /more than 3, or more than 50% of the 4 held/); assert.deepEqual((waiting.consumers as { consumer: string }[]).map((c) => c.consumer), ["a", "b", "c", "d"]); await assert.rejects(w.p.approve(["a", "b", "c"], { why: "x" }), /exactly that set/); await assert.rejects(w.p.approve(["a", "b", "c", "d"], { why: "" }), /why is required/); const done = await w.p.approve(["d", "c", "b", "a"], { why: "all four unassigned", by: "operator", via: "mesh-controller" }); assert.deepEqual(done.retired, ["a", "b", "c", "d"]); assert.deepEqual(w.retired.map((r) => r.as), ["a", "b", "c", "d"]); assert.deepEqual(changes(w), ["waiting", "approved", "retired"]); assert.equal(w.events[1].via, "mesh-controller"); assert.equal(w.events[1].by, "operator"); } finally { w.done(); } }); test("rejected, the set is kept active and not asked again while the mesh says the same", async () => { const w = await world(["a", "b", "c", "d"]); try { await w.ask([]); await until(async () => (await w.p.retirement()).waiting !== null); await assert.rejects(w.p.reject(["a"], { why: "no" }), /exactly that set/); const kept = await w.p.reject(["a", "b", "c", "d"], { why: "the mesh is wrong", by: "operator" }); assert.deepEqual(kept.kept, ["a", "b", "c", "d"]); await sleep(EVERY * 12); assert.deepEqual(w.retired, []); assert.deepEqual(changes(w), ["waiting", "rejected"]); const r = await w.p.retirement(); assert.equal(r.waiting, null); assert.deepEqual(r.rejected?.consumers.map((c) => c.consumer), ["a", "b", "c", "d"]); // Approving a rejected set later still works. await w.p.approve(["a", "b", "c", "d"], { why: "on second thought" }); assert.equal(w.retired.length, 4); } finally { w.done(); } }); test("a waiting set the mesh asks for again settles, and nothing is retired", async () => { const w = await world(["a", "b", "c", "d"]); try { await w.ask([]); await until(async () => (await w.p.retirement()).waiting !== null); await w.ask(["a", "b", "c", "d"]); await until(() => changes(w).includes("settled")); assert.equal((await w.p.retirement()).waiting, null); assert.deepEqual(w.retired, []); } finally { w.done(); } }); test("delete refuses an active consumer and deletes only the named retired one", async () => { const w = await world(["a", "b", "c"]); try { await w.ask(["c"]); // Two of three is more than half: approve, then delete one. await until(async () => (await w.p.retirement()).waiting !== null); await w.p.approve(["a", "b"], { why: "gone" }); await assert.rejects(w.p.deleteRetired("c", { why: "x", confirm: "c" }), /active, not retired/); await assert.rejects(w.p.deleteRetired("zz", { why: "x", confirm: "zz" }), /not retired here/); await assert.rejects(w.p.deleteRetired("a", { why: "x", confirm: "b" }), /confirm must repeat/); await assert.rejects(w.p.deleteRetired("a", { why: "", confirm: "a" }), /why is required/); const r = await w.p.deleteRetired("a", { why: "older than a month", by: "operator", confirm: "a" }); assert.deepEqual(r, { deleted: "a", freed_bytes: 42 }); assert.deepEqual(w.deleted, ["a"]); assert.deepEqual((await w.p.retirement()).retired.map((x) => x.consumer), ["b"]); assert.equal(w.events.at(-1)?.change, "deleted"); } finally { w.done(); } }); test("an addition acts on the first pass", async () => { const w = await world(["a"]); try { const before = w.created.length; await w.ask(["a", "b"]); await until(() => w.created.length > before, EVERY * 3); assert.equal(w.created.at(-1), "b"); } finally { w.done(); } }); test("what the backend holds at start is retired by the same rules", async () => { // "x" was made by the mesh and the mesh stopped asking for it while the provider was down. const w = await world(["a", "b", "c"], { held: ["a", "b", "c", "x"] }); try { await until(() => w.retired.length === 1); assert.equal(w.retired[0].as, "x"); } finally { w.done(); } }); test("the four tools are registered for the module and answer", async () => { resetTools(); const dir = await mkdtemp(join(tmpdir(), "retire-tools-")); const receives = join(dir, "r.json"); await writeFile(receives, JSON.stringify({ requirement: "r", given: [] })); const p = runProvisioner("r", { async create() {}, async remove() {} }, { receives, everyMs: EVERY, module: "postgres" }); try { const mine = collectTools({}).find((c) => c.module === "postgres"); assert.deepEqual(mine?.tools.map((t) => t.name), [ "provisioner_retirement", "provisioner_retire_approve", "provisioner_retire_reject", "provisioner_delete", ]); const answer = (await mine!.tools[0].run({})) as { resource: string; waiting: unknown }; assert.equal(answer.resource, "r"); assert.equal(answer.waiting, null); assert.equal(provisionerTools(p).length, 4); } finally { p(); resetTools(); } });