The eight Go test functions H2 added become subtests of existing tests (catalogue's state test, the bundle's state test, the views test, the process state test, the testbus bus test, the seal byte-for-byte test), and four more are folded beside them (two seal, two process watch); the two TypeScript seal tests become one, and two more TypeScript tests are folded into their neighbours. No assertion is lost. Builds: design 47 H2 §819
347 lines
15 KiB
TypeScript
347 lines
15 KiB
TypeScript
// Retirement (novox/hq ADR 0230): a consumer the mesh stops asking for is retired — access disabled,
|
|
// data kept — only after the same result in five consecutive passes AND ten minutes (shortened here
|
|
// unless a test is about it), and a set too large waits for a person. An adapter that cannot disable
|
|
// is mark-only: remove is never called to retire. Deletion is a person's act, of a retired consumer only.
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
import { runProviderLoop, providerLoopTools, RETIREMENT_EVENT, type ProviderLoop } from "../dist/provider-loop/index.js";
|
|
import { resetTools, collectTools } from "../dist/tools/index.js";
|
|
|
|
const EVERY = 10;
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
async function until(cond: () => boolean | Promise<boolean>, ms = 3000): Promise<void> {
|
|
const start = Date.now();
|
|
while (!(await cond())) {
|
|
if (Date.now() - start > ms) throw new Error("timed out waiting");
|
|
await sleep(5);
|
|
}
|
|
}
|
|
|
|
interface World {
|
|
receives: string;
|
|
ask(names: string[]): Promise<void>;
|
|
created: string[];
|
|
retired: { as: string; why: string }[];
|
|
removed: string[];
|
|
deleted: string[];
|
|
reenabled: string[];
|
|
events: Record<string, unknown>[];
|
|
said: string[];
|
|
p: ProviderLoop;
|
|
done(): void;
|
|
}
|
|
|
|
async function world(
|
|
names: string[],
|
|
opts: { retire?: boolean; held?: string[]; del?: boolean; reenable?: boolean; stableForMs?: number; stablePasses?: number } = {},
|
|
): Promise<World> {
|
|
const dir = await mkdtemp(join(tmpdir(), "retire-"));
|
|
const receives = join(dir, "postgres-database.json");
|
|
const ask = async (list: string[]) => {
|
|
const given = [];
|
|
for (const n of list) {
|
|
await writeFile(join(dir, `${n}.secret`), `pw-${n}\n`);
|
|
given.push({ from: n, node: "anchor", as: n, secret: join(dir, `${n}.secret`) });
|
|
}
|
|
await writeFile(receives, JSON.stringify({ contributions: 1, requirement: "postgres-database", given }));
|
|
};
|
|
await ask(names);
|
|
const w = { receives, ask, created: [], retired: [], removed: [], deleted: [], reenabled: [], events: [], said: [] } as unknown as World;
|
|
const was = console.error;
|
|
console.error = (...a: unknown[]) => w.said.push(a.join(" "));
|
|
const adapter = {
|
|
async create(p: { as: string }) { w.created.push(p.as); },
|
|
async remove(p: { as: string }) { w.removed.push(p.as); },
|
|
...(opts.del === false ? {} : { async delete(p: { as: string }) { w.deleted.push(p.as); return { freedBytes: 42 }; } }),
|
|
...(opts.reenable ? { async reenable(p: { as: string }) { w.reenabled.push(p.as); } } : {}),
|
|
...(opts.retire === false ? {} : { async retire(p: { as: string; why: string }) { w.retired.push({ as: p.as, why: p.why }); } }),
|
|
...(opts.held ? { async held() { return opts.held!; } } : {}),
|
|
};
|
|
w.p = runProviderLoop("postgres-database", adapter, {
|
|
receives, everyMs: EVERY, node: "anchor", stableForMs: opts.stableForMs ?? 0,
|
|
...(opts.stablePasses ? { stablePasses: opts.stablePasses } : {}),
|
|
// Only retirement is these tests'; the standing (provisioner.failing, .recovered) is standing.test.ts's.
|
|
announce: (event, body) => { if (event === RETIREMENT_EVENT) w.events.push(body); },
|
|
});
|
|
// A pass is counted by the file being read: wait wall time for n passes and some slack.
|
|
w.done = () => { w.p(); console.error = was; };
|
|
await until(() => w.created.length >= names.length);
|
|
return w;
|
|
}
|
|
|
|
const changes = (w: World) => w.events.map((e) => e.change);
|
|
|
|
test("a transient empty list for four passes retires nothing; five stable passes retire, data kept", async () => {
|
|
const w = await world(["a", "b", "c"], {});
|
|
try {
|
|
// The unasked set is {a} — one of three, within the bound — seen for fewer than five passes.
|
|
const p = w.p;
|
|
// Run the loop by hand: stop the timer-driven one and drive passes directly is not exposed, so
|
|
// the transient is made short in wall time instead: ask without a, then with a, quickly.
|
|
await w.ask(["b", "c"]);
|
|
await sleep(EVERY * 2);
|
|
await w.ask(["a", "b", "c"]);
|
|
await sleep(EVERY * 8);
|
|
assert.deepEqual(w.retired, [], "retired on a transient result");
|
|
assert.deepEqual(changes(w), []);
|
|
// Stable: retired, through retire, and not removed.
|
|
await w.ask(["b", "c"]);
|
|
await until(() => w.retired.length === 1);
|
|
assert.equal(w.retired[0].as, "a");
|
|
assert.match(w.retired[0].why, /\d+ passes over/);
|
|
assert.deepEqual(w.removed, []);
|
|
assert.deepEqual(w.deleted, []);
|
|
assert.deepEqual(changes(w), ["retired"]);
|
|
const r = await p.retirement();
|
|
assert.deepEqual(r.retired.map((x) => x.consumer), ["a"]);
|
|
assert.deepEqual(r.held, ["b", "c"]);
|
|
assert.equal(r.stable_passes, 5);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("the fifth pass and not the fourth retires", async () => {
|
|
// Driven by the clock: five passes 10 ms apart take at least 40 ms after the first sees the set.
|
|
const w = await world(["a", "b", "c"]);
|
|
try {
|
|
// The clock starts before the file is written: a pass may read it the moment it lands, before any line after the
|
|
// write runs, and a start taken after would then count fewer than four intervals.
|
|
const start = Date.now();
|
|
await w.ask(["b", "c"]);
|
|
await until(() => w.retired.length === 1);
|
|
assert.ok(Date.now() - start >= 4 * EVERY - 2, `retired after ${Date.now() - start} ms, before five passes`);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("an unreadable file resets the count", async () => {
|
|
const w = await world(["a", "b", "c"]);
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
// Interleave unreadable passes so five readable ones never run in a row.
|
|
for (let i = 0; i < 6; i++) {
|
|
await sleep(EVERY * 2);
|
|
await rm(w.receives);
|
|
await sleep(EVERY * 2);
|
|
await w.ask(["b", "c"]);
|
|
}
|
|
assert.deepEqual(w.retired, [], "unreadable passes counted toward a stable result");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("asked for again, a retired consumer is created again and announced re-enabled", async () => {
|
|
const w = await world(["a", "b", "c"]);
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
await until(() => w.retired.length === 1);
|
|
const before = w.created.length;
|
|
await w.ask(["a", "b", "c"]);
|
|
await until(() => w.created.length > before);
|
|
assert.equal(w.created.at(-1), "a");
|
|
await until(() => changes(w).includes("reenabled"));
|
|
assert.deepEqual((await w.p.retirement()).retired, []);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("five passes are not enough: the same set must also hold for stableForMs, ten minutes by default", async () => {
|
|
const { STABLE_FOR_MS } = await import("../dist/provider-loop/index.js");
|
|
assert.equal(STABLE_FOR_MS, 10 * 60_000);
|
|
const HOLD = 400;
|
|
const w = await world(["a", "b", "c"], { stableForMs: HOLD });
|
|
try {
|
|
const start = Date.now(); // before the write, as in the fifth-pass test
|
|
await w.ask(["b", "c"]);
|
|
await sleep(EVERY * 10); // well past five passes, well short of the hold
|
|
if (Date.now() - start < HOLD) assert.deepEqual(w.retired, [], "retired after five passes, before the set held long enough");
|
|
await until(() => w.retired.length === 1, 3000);
|
|
assert.ok(Date.now() - start >= HOLD - 5, `retired after ${Date.now() - start} ms, before ${HOLD} ms`);
|
|
assert.equal((await w.p.retirement()).stable_for_seconds, 0); // 400 ms rounds to 0 s
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("an adapter without retire is MARK-ONLY: nothing changes in the backend, remove only on delete", async () => {
|
|
const w = await world(["a", "b", "c"], { retire: false, del: false });
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
await until(() => changes(w).includes("retired"));
|
|
await sleep(EVERY * 10);
|
|
assert.deepEqual(w.removed, [], "remove was called to retire");
|
|
const ev = w.events.find((e) => e.change === "retired")!;
|
|
assert.equal((ev.consumers as { access: string }[])[0].access, "kept");
|
|
assert.ok(w.said.some((l) => /MARK ONLY/.test(l) && /KEEPS ITS ACCESS/.test(l)), "mark-only was not said loudly");
|
|
const r = await w.p.retirement();
|
|
assert.equal(r.retires_by, "mark-only");
|
|
assert.deepEqual(r.retired.map((x) => [x.consumer, x.access]), [["a", "kept"]]);
|
|
// Asked again: created again, nothing to undo.
|
|
await w.ask(["a", "b", "c"]);
|
|
await until(() => changes(w).includes("reenabled"));
|
|
assert.deepEqual(w.removed, []);
|
|
// Gone again, then a person's delete: the one path to remove.
|
|
await w.ask(["b", "c"]);
|
|
await until(async () => (await w.p.retirement()).retired.length === 1);
|
|
await w.p.deleteRetired("a", { why: "gone", confirm: "a" });
|
|
assert.deepEqual(w.removed, ["a"]);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("an adapter with retire and reenable: disabled on retire, reenabled before create when asked again", async () => {
|
|
const w = await world(["a", "b", "c"], { reenable: true });
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
await until(() => w.retired.length === 1);
|
|
assert.equal((await w.p.retirement()).retires_by, "disable");
|
|
assert.equal((await w.p.retirement()).retired[0].access, "disabled");
|
|
const before = w.created.length;
|
|
await w.ask(["a", "b", "c"]);
|
|
await until(() => w.created.length > before);
|
|
assert.deepEqual(w.reenabled, ["a"]);
|
|
assert.deepEqual(w.removed, []);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("four of four waits for a person: approve the exact set retires, another set is refused", async () => {
|
|
const w = await world(["a", "b", "c", "d"]);
|
|
try {
|
|
await w.ask([]);
|
|
await until(async () => (await w.p.retirement()).waiting !== null);
|
|
await sleep(EVERY * 10);
|
|
assert.deepEqual(w.retired, [], "retired over the bound without a person");
|
|
assert.deepEqual(changes(w), ["waiting"], "waiting said more than once, or not");
|
|
const waiting = w.events[0];
|
|
assert.equal(waiting.held, 4);
|
|
assert.match(String(waiting.bound), /more than 3, or more than 50% of the 4 held/);
|
|
assert.deepEqual((waiting.consumers as { consumer: string }[]).map((c) => c.consumer), ["a", "b", "c", "d"]);
|
|
|
|
await assert.rejects(w.p.approve(["a", "b", "c"], { why: "x" }), /exactly that set/);
|
|
await assert.rejects(w.p.approve(["a", "b", "c", "d"], { why: "" }), /why is required/);
|
|
const done = await w.p.approve(["d", "c", "b", "a"], { why: "all four unassigned", by: "operator", via: "mesh-controller" });
|
|
assert.deepEqual(done.retired, ["a", "b", "c", "d"]);
|
|
assert.deepEqual(w.retired.map((r) => r.as), ["a", "b", "c", "d"]);
|
|
assert.deepEqual(changes(w), ["waiting", "approved", "retired"]);
|
|
assert.equal(w.events[1].via, "mesh-controller");
|
|
assert.equal(w.events[1].by, "operator");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("rejected, the set is kept active and not asked again while the mesh says the same", async () => {
|
|
const w = await world(["a", "b", "c", "d"]);
|
|
try {
|
|
await w.ask([]);
|
|
await until(async () => (await w.p.retirement()).waiting !== null);
|
|
await assert.rejects(w.p.reject(["a"], { why: "no" }), /exactly that set/);
|
|
const kept = await w.p.reject(["a", "b", "c", "d"], { why: "the mesh is wrong", by: "operator" });
|
|
assert.deepEqual(kept.kept, ["a", "b", "c", "d"]);
|
|
await sleep(EVERY * 12);
|
|
assert.deepEqual(w.retired, []);
|
|
assert.deepEqual(changes(w), ["waiting", "rejected"]);
|
|
const r = await w.p.retirement();
|
|
assert.equal(r.waiting, null);
|
|
assert.deepEqual(r.rejected?.consumers.map((c) => c.consumer), ["a", "b", "c", "d"]);
|
|
// Approving a rejected set later still works.
|
|
await w.p.approve(["a", "b", "c", "d"], { why: "on second thought" });
|
|
assert.equal(w.retired.length, 4);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("a waiting set the mesh asks for again settles, and nothing is retired", async () => {
|
|
const w = await world(["a", "b", "c", "d"]);
|
|
try {
|
|
await w.ask([]);
|
|
await until(async () => (await w.p.retirement()).waiting !== null);
|
|
await w.ask(["a", "b", "c", "d"]);
|
|
await until(() => changes(w).includes("settled"));
|
|
assert.equal((await w.p.retirement()).waiting, null);
|
|
assert.deepEqual(w.retired, []);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("delete refuses an active consumer and deletes only the named retired one", async () => {
|
|
const w = await world(["a", "b", "c"]);
|
|
try {
|
|
await w.ask(["c"]);
|
|
// Two of three is more than half: approve, then delete one.
|
|
await until(async () => (await w.p.retirement()).waiting !== null);
|
|
await w.p.approve(["a", "b"], { why: "gone" });
|
|
await assert.rejects(w.p.deleteRetired("c", { why: "x", confirm: "c" }), /active, not retired/);
|
|
await assert.rejects(w.p.deleteRetired("zz", { why: "x", confirm: "zz" }), /not retired here/);
|
|
await assert.rejects(w.p.deleteRetired("a", { why: "x", confirm: "b" }), /confirm must repeat/);
|
|
await assert.rejects(w.p.deleteRetired("a", { why: "", confirm: "a" }), /why is required/);
|
|
const r = await w.p.deleteRetired("a", { why: "older than a month", by: "operator", confirm: "a" });
|
|
assert.deepEqual(r, { deleted: "a", freed_bytes: 42 });
|
|
assert.deepEqual(w.deleted, ["a"]);
|
|
assert.deepEqual((await w.p.retirement()).retired.map((x) => x.consumer), ["b"]);
|
|
assert.equal(w.events.at(-1)?.change, "deleted");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("an addition acts on the first pass", async () => {
|
|
// Not gated by the stability a retirement waits for: with a hold of a minute and a thousand passes, only a pass that
|
|
// acts at once creates b. The deadline is a few passes, not three intervals of 10 ms: a scheduler stall must not fail it.
|
|
const w = await world(["a"], { stableForMs: 60_000, stablePasses: 1000 });
|
|
try {
|
|
const before = w.created.length;
|
|
await w.ask(["a", "b"]);
|
|
await until(() => w.created.length > before, 1000);
|
|
assert.equal(w.created.at(-1), "b");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("what the backend holds at start is retired by the same rules", async () => {
|
|
// "x" was made by the mesh and the mesh stopped asking for it while the provider was down.
|
|
const w = await world(["a", "b", "c"], { held: ["a", "b", "c", "x"] });
|
|
try {
|
|
await until(() => w.retired.length === 1);
|
|
assert.equal(w.retired[0].as, "x");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("the four tools are registered for the module and answer", async () => {
|
|
resetTools();
|
|
const dir = await mkdtemp(join(tmpdir(), "retire-tools-"));
|
|
const receives = join(dir, "r.json");
|
|
await writeFile(receives, JSON.stringify({ requirement: "r", given: [] }));
|
|
const p = runProviderLoop("r", { async create() {}, async remove() {} }, { receives, everyMs: EVERY, module: "postgres" });
|
|
try {
|
|
const mine = collectTools({}).find((c) => c.module === "postgres");
|
|
assert.deepEqual(mine?.tools.map((t) => t.name), [
|
|
"provisioner_retirement", "provisioner_retire_approve", "provisioner_retire_reject", "provisioner_delete",
|
|
]);
|
|
const answer = (await mine!.tools[0].run({})) as { resource: string; waiting: unknown };
|
|
assert.equal(answer.resource, "r");
|
|
assert.equal(answer.waiting, null);
|
|
assert.equal(providerLoopTools(p).length, 4);
|
|
} finally {
|
|
p();
|
|
resetTools();
|
|
}
|
|
});
|