jochen
0704908d0b
Five passes are 25 s, shorter than a real hiccup. And a provider's remove can destroy — a ledger file, a public DNS record — so an adapter without a retire of its own is mark-only: the consumer keeps its access, is marked and said, and remove is reached only by a person's delete (hq ADR 0230).
344 lines
14 KiB
TypeScript
344 lines
14 KiB
TypeScript
// Retirement (novox/hq ADR 0230): a consumer the mesh stops asking for is retired — access disabled,
|
|
// data kept — only after the same result in five consecutive passes AND ten minutes (shortened here
|
|
// unless a test is about it), and a set too large waits for a person. An adapter that cannot disable
|
|
// is mark-only: remove is never called to retire. Deletion is a person's act, of a retired consumer only.
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
import { runProvisioner, provisionerTools, RETIREMENT_EVENT, type Provisioner } from "../dist/provisioner/index.js";
|
|
import { resetTools, collectTools } from "../dist/tools/index.js";
|
|
|
|
const EVERY = 10;
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
async function until(cond: () => boolean | Promise<boolean>, ms = 3000): Promise<void> {
|
|
const start = Date.now();
|
|
while (!(await cond())) {
|
|
if (Date.now() - start > ms) throw new Error("timed out waiting");
|
|
await sleep(5);
|
|
}
|
|
}
|
|
|
|
interface World {
|
|
receives: string;
|
|
ask(names: string[]): Promise<void>;
|
|
created: string[];
|
|
retired: { as: string; why: string }[];
|
|
removed: string[];
|
|
deleted: string[];
|
|
reenabled: string[];
|
|
events: Record<string, unknown>[];
|
|
said: string[];
|
|
p: Provisioner;
|
|
done(): void;
|
|
}
|
|
|
|
async function world(
|
|
names: string[],
|
|
opts: { retire?: boolean; held?: string[]; del?: boolean; reenable?: boolean; stableForMs?: number } = {},
|
|
): Promise<World> {
|
|
const dir = await mkdtemp(join(tmpdir(), "retire-"));
|
|
const receives = join(dir, "postgres-database.json");
|
|
const ask = async (list: string[]) => {
|
|
const given = [];
|
|
for (const n of list) {
|
|
await writeFile(join(dir, `${n}.secret`), `pw-${n}\n`);
|
|
given.push({ from: n, node: "anchor", as: n, secret: join(dir, `${n}.secret`) });
|
|
}
|
|
await writeFile(receives, JSON.stringify({ contributions: 1, requirement: "postgres-database", given }));
|
|
};
|
|
await ask(names);
|
|
const w = { receives, ask, created: [], retired: [], removed: [], deleted: [], reenabled: [], events: [], said: [] } as unknown as World;
|
|
const was = console.error;
|
|
console.error = (...a: unknown[]) => w.said.push(a.join(" "));
|
|
const adapter = {
|
|
async create(p: { as: string }) { w.created.push(p.as); },
|
|
async remove(p: { as: string }) { w.removed.push(p.as); },
|
|
...(opts.del === false ? {} : { async delete(p: { as: string }) { w.deleted.push(p.as); return { freedBytes: 42 }; } }),
|
|
...(opts.reenable ? { async reenable(p: { as: string }) { w.reenabled.push(p.as); } } : {}),
|
|
...(opts.retire === false ? {} : { async retire(p: { as: string; why: string }) { w.retired.push({ as: p.as, why: p.why }); } }),
|
|
...(opts.held ? { async held() { return opts.held!; } } : {}),
|
|
};
|
|
w.p = runProvisioner("postgres-database", adapter, {
|
|
receives, everyMs: EVERY, node: "anchor", stableForMs: opts.stableForMs ?? 0,
|
|
announce: (event, body) => { assert.equal(event, RETIREMENT_EVENT); w.events.push(body); },
|
|
});
|
|
// A pass is counted by the file being read: wait wall time for n passes and some slack.
|
|
w.done = () => { w.p(); console.error = was; };
|
|
await until(() => w.created.length >= names.length);
|
|
return w;
|
|
}
|
|
|
|
const changes = (w: World) => w.events.map((e) => e.change);
|
|
|
|
test("a transient empty list for four passes retires nothing; five stable passes retire, data kept", async () => {
|
|
const w = await world(["a", "b", "c"], {});
|
|
try {
|
|
// The unasked set is {a} — one of three, within the bound — seen for fewer than five passes.
|
|
const p = w.p;
|
|
// Run the loop by hand: stop the timer-driven one and drive passes directly is not exposed, so
|
|
// the transient is made short in wall time instead: ask without a, then with a, quickly.
|
|
await w.ask(["b", "c"]);
|
|
await sleep(EVERY * 2);
|
|
await w.ask(["a", "b", "c"]);
|
|
await sleep(EVERY * 8);
|
|
assert.deepEqual(w.retired, [], "retired on a transient result");
|
|
assert.deepEqual(changes(w), []);
|
|
// Stable: retired, through retire, and not removed.
|
|
await w.ask(["b", "c"]);
|
|
await until(() => w.retired.length === 1);
|
|
assert.equal(w.retired[0].as, "a");
|
|
assert.match(w.retired[0].why, /\d+ passes over/);
|
|
assert.deepEqual(w.removed, []);
|
|
assert.deepEqual(w.deleted, []);
|
|
assert.deepEqual(changes(w), ["retired"]);
|
|
const r = await p.retirement();
|
|
assert.deepEqual(r.retired.map((x) => x.consumer), ["a"]);
|
|
assert.deepEqual(r.held, ["b", "c"]);
|
|
assert.equal(r.stable_passes, 5);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("the fifth pass and not the fourth retires", async () => {
|
|
// Driven by the clock: five passes 10 ms apart take at least 40 ms after the first sees the set.
|
|
const w = await world(["a", "b", "c"]);
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
const start = Date.now();
|
|
await until(() => w.retired.length === 1);
|
|
assert.ok(Date.now() - start >= 4 * EVERY - 2, `retired after ${Date.now() - start} ms, before five passes`);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("an unreadable file resets the count", async () => {
|
|
const w = await world(["a", "b", "c"]);
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
// Interleave unreadable passes so five readable ones never run in a row.
|
|
for (let i = 0; i < 6; i++) {
|
|
await sleep(EVERY * 2);
|
|
await rm(w.receives);
|
|
await sleep(EVERY * 2);
|
|
await w.ask(["b", "c"]);
|
|
}
|
|
assert.deepEqual(w.retired, [], "unreadable passes counted toward a stable result");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("asked for again, a retired consumer is created again and announced re-enabled", async () => {
|
|
const w = await world(["a", "b", "c"]);
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
await until(() => w.retired.length === 1);
|
|
const before = w.created.length;
|
|
await w.ask(["a", "b", "c"]);
|
|
await until(() => w.created.length > before);
|
|
assert.equal(w.created.at(-1), "a");
|
|
await until(() => changes(w).includes("reenabled"));
|
|
assert.deepEqual((await w.p.retirement()).retired, []);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("five passes are not enough: the same set must also hold for stableForMs", async () => {
|
|
const HOLD = 400;
|
|
const w = await world(["a", "b", "c"], { stableForMs: HOLD });
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
const start = Date.now();
|
|
await sleep(EVERY * 10); // well past five passes, well short of the hold
|
|
assert.deepEqual(w.retired, [], "retired after five passes, before the set held long enough");
|
|
await until(() => w.retired.length === 1, 3000);
|
|
assert.ok(Date.now() - start >= HOLD - 5, `retired after ${Date.now() - start} ms, before ${HOLD} ms`);
|
|
assert.equal((await w.p.retirement()).stable_for_seconds, 0); // 400 ms rounds to 0 s
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("the default hold is ten minutes", async () => {
|
|
const { STABLE_FOR_MS } = await import("../dist/provisioner/index.js");
|
|
assert.equal(STABLE_FOR_MS, 10 * 60_000);
|
|
});
|
|
|
|
test("an adapter without retire is MARK-ONLY: nothing changes in the backend, remove only on delete", async () => {
|
|
const w = await world(["a", "b", "c"], { retire: false, del: false });
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
await until(() => changes(w).includes("retired"));
|
|
await sleep(EVERY * 10);
|
|
assert.deepEqual(w.removed, [], "remove was called to retire");
|
|
const ev = w.events.find((e) => e.change === "retired")!;
|
|
assert.equal((ev.consumers as { access: string }[])[0].access, "kept");
|
|
assert.ok(w.said.some((l) => /MARK ONLY/.test(l) && /KEEPS ITS ACCESS/.test(l)), "mark-only was not said loudly");
|
|
const r = await w.p.retirement();
|
|
assert.equal(r.retires_by, "mark-only");
|
|
assert.deepEqual(r.retired.map((x) => [x.consumer, x.access]), [["a", "kept"]]);
|
|
// Asked again: created again, nothing to undo.
|
|
await w.ask(["a", "b", "c"]);
|
|
await until(() => changes(w).includes("reenabled"));
|
|
assert.deepEqual(w.removed, []);
|
|
// Gone again, then a person's delete: the one path to remove.
|
|
await w.ask(["b", "c"]);
|
|
await until(async () => (await w.p.retirement()).retired.length === 1);
|
|
await w.p.deleteRetired("a", { why: "gone", confirm: "a" });
|
|
assert.deepEqual(w.removed, ["a"]);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("an adapter with retire and reenable: disabled on retire, reenabled before create when asked again", async () => {
|
|
const w = await world(["a", "b", "c"], { reenable: true });
|
|
try {
|
|
await w.ask(["b", "c"]);
|
|
await until(() => w.retired.length === 1);
|
|
assert.equal((await w.p.retirement()).retires_by, "disable");
|
|
assert.equal((await w.p.retirement()).retired[0].access, "disabled");
|
|
const before = w.created.length;
|
|
await w.ask(["a", "b", "c"]);
|
|
await until(() => w.created.length > before);
|
|
assert.deepEqual(w.reenabled, ["a"]);
|
|
assert.deepEqual(w.removed, []);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("four of four waits for a person: approve the exact set retires, another set is refused", async () => {
|
|
const w = await world(["a", "b", "c", "d"]);
|
|
try {
|
|
await w.ask([]);
|
|
await until(async () => (await w.p.retirement()).waiting !== null);
|
|
await sleep(EVERY * 10);
|
|
assert.deepEqual(w.retired, [], "retired over the bound without a person");
|
|
assert.deepEqual(changes(w), ["waiting"], "waiting said more than once, or not");
|
|
const waiting = w.events[0];
|
|
assert.equal(waiting.held, 4);
|
|
assert.match(String(waiting.bound), /more than 3, or more than 50% of the 4 held/);
|
|
assert.deepEqual((waiting.consumers as { consumer: string }[]).map((c) => c.consumer), ["a", "b", "c", "d"]);
|
|
|
|
await assert.rejects(w.p.approve(["a", "b", "c"], { why: "x" }), /exactly that set/);
|
|
await assert.rejects(w.p.approve(["a", "b", "c", "d"], { why: "" }), /why is required/);
|
|
const done = await w.p.approve(["d", "c", "b", "a"], { why: "all four unassigned", by: "operator", via: "mesh-controller" });
|
|
assert.deepEqual(done.retired, ["a", "b", "c", "d"]);
|
|
assert.deepEqual(w.retired.map((r) => r.as), ["a", "b", "c", "d"]);
|
|
assert.deepEqual(changes(w), ["waiting", "approved", "retired"]);
|
|
assert.equal(w.events[1].via, "mesh-controller");
|
|
assert.equal(w.events[1].by, "operator");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("rejected, the set is kept active and not asked again while the mesh says the same", async () => {
|
|
const w = await world(["a", "b", "c", "d"]);
|
|
try {
|
|
await w.ask([]);
|
|
await until(async () => (await w.p.retirement()).waiting !== null);
|
|
await assert.rejects(w.p.reject(["a"], { why: "no" }), /exactly that set/);
|
|
const kept = await w.p.reject(["a", "b", "c", "d"], { why: "the mesh is wrong", by: "operator" });
|
|
assert.deepEqual(kept.kept, ["a", "b", "c", "d"]);
|
|
await sleep(EVERY * 12);
|
|
assert.deepEqual(w.retired, []);
|
|
assert.deepEqual(changes(w), ["waiting", "rejected"]);
|
|
const r = await w.p.retirement();
|
|
assert.equal(r.waiting, null);
|
|
assert.deepEqual(r.rejected?.consumers.map((c) => c.consumer), ["a", "b", "c", "d"]);
|
|
// Approving a rejected set later still works.
|
|
await w.p.approve(["a", "b", "c", "d"], { why: "on second thought" });
|
|
assert.equal(w.retired.length, 4);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("a waiting set the mesh asks for again settles, and nothing is retired", async () => {
|
|
const w = await world(["a", "b", "c", "d"]);
|
|
try {
|
|
await w.ask([]);
|
|
await until(async () => (await w.p.retirement()).waiting !== null);
|
|
await w.ask(["a", "b", "c", "d"]);
|
|
await until(() => changes(w).includes("settled"));
|
|
assert.equal((await w.p.retirement()).waiting, null);
|
|
assert.deepEqual(w.retired, []);
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("delete refuses an active consumer and deletes only the named retired one", async () => {
|
|
const w = await world(["a", "b", "c"]);
|
|
try {
|
|
await w.ask(["c"]);
|
|
// Two of three is more than half: approve, then delete one.
|
|
await until(async () => (await w.p.retirement()).waiting !== null);
|
|
await w.p.approve(["a", "b"], { why: "gone" });
|
|
await assert.rejects(w.p.deleteRetired("c", { why: "x", confirm: "c" }), /active, not retired/);
|
|
await assert.rejects(w.p.deleteRetired("zz", { why: "x", confirm: "zz" }), /not retired here/);
|
|
await assert.rejects(w.p.deleteRetired("a", { why: "x", confirm: "b" }), /confirm must repeat/);
|
|
await assert.rejects(w.p.deleteRetired("a", { why: "", confirm: "a" }), /why is required/);
|
|
const r = await w.p.deleteRetired("a", { why: "older than a month", by: "operator", confirm: "a" });
|
|
assert.deepEqual(r, { deleted: "a", freed_bytes: 42 });
|
|
assert.deepEqual(w.deleted, ["a"]);
|
|
assert.deepEqual((await w.p.retirement()).retired.map((x) => x.consumer), ["b"]);
|
|
assert.equal(w.events.at(-1)?.change, "deleted");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("an addition acts on the first pass", async () => {
|
|
const w = await world(["a"]);
|
|
try {
|
|
const before = w.created.length;
|
|
await w.ask(["a", "b"]);
|
|
await until(() => w.created.length > before, EVERY * 3);
|
|
assert.equal(w.created.at(-1), "b");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("what the backend holds at start is retired by the same rules", async () => {
|
|
// "x" was made by the mesh and the mesh stopped asking for it while the provider was down.
|
|
const w = await world(["a", "b", "c"], { held: ["a", "b", "c", "x"] });
|
|
try {
|
|
await until(() => w.retired.length === 1);
|
|
assert.equal(w.retired[0].as, "x");
|
|
} finally {
|
|
w.done();
|
|
}
|
|
});
|
|
|
|
test("the four tools are registered for the module and answer", async () => {
|
|
resetTools();
|
|
const dir = await mkdtemp(join(tmpdir(), "retire-tools-"));
|
|
const receives = join(dir, "r.json");
|
|
await writeFile(receives, JSON.stringify({ requirement: "r", given: [] }));
|
|
const p = runProvisioner("r", { async create() {}, async remove() {} }, { receives, everyMs: EVERY, module: "postgres" });
|
|
try {
|
|
const mine = collectTools({}).find((c) => c.module === "postgres");
|
|
assert.deepEqual(mine?.tools.map((t) => t.name), [
|
|
"provisioner_retirement", "provisioner_retire_approve", "provisioner_retire_reject", "provisioner_delete",
|
|
]);
|
|
const answer = (await mine!.tools[0].run({})) as { resource: string; waiting: unknown };
|
|
assert.equal(answer.resource, "r");
|
|
assert.equal(answer.waiting, null);
|
|
assert.equal(provisionerTools(p).length, 4);
|
|
} finally {
|
|
p();
|
|
resetTools();
|
|
}
|
|
});
|