0.1.12: retire a consumer the mesh stops asking for, after five stable passes, and wait for a person over the bound
The hourly release let a mistaken bulk unassignment drain one consumer an hour with nobody deciding. Per novox/hq ADR 0230 a consumer is retired (disabled, marked, data kept) only after the same unasked set in five consecutive passes; more than 3, or more than half of those held, waits for approve or reject. Deletion is a separate act of a person, of a retired consumer only.
This commit is contained in:
jochen
@@ -0,0 +1,283 @@
|
||||
// Retirement (novox/hq ADR 0230): a consumer the mesh stops asking for is retired — access disabled,
|
||||
// data kept — only after the same result in five consecutive passes, and a set too large waits for a
|
||||
// person. Deletion is a person's act, of a retired consumer only.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { runProvisioner, provisionerTools, RETIREMENT_EVENT, type Provisioner } from "../dist/provisioner/index.js";
|
||||
import { resetTools, collectTools } from "../dist/tools/index.js";
|
||||
|
||||
const EVERY = 10;
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
async function until(cond: () => boolean | Promise<boolean>, ms = 3000): Promise<void> {
|
||||
const start = Date.now();
|
||||
while (!(await cond())) {
|
||||
if (Date.now() - start > ms) throw new Error("timed out waiting");
|
||||
await sleep(5);
|
||||
}
|
||||
}
|
||||
|
||||
interface World {
|
||||
receives: string;
|
||||
ask(names: string[]): Promise<void>;
|
||||
created: string[];
|
||||
retired: { as: string; why: string }[];
|
||||
removed: string[];
|
||||
deleted: string[];
|
||||
events: Record<string, unknown>[];
|
||||
said: string[];
|
||||
p: Provisioner;
|
||||
done(): void;
|
||||
}
|
||||
|
||||
async function world(names: string[], opts: { retire?: boolean; held?: string[] } = {}): Promise<World> {
|
||||
const dir = await mkdtemp(join(tmpdir(), "retire-"));
|
||||
const receives = join(dir, "postgres-database.json");
|
||||
const ask = async (list: string[]) => {
|
||||
const given = [];
|
||||
for (const n of list) {
|
||||
await writeFile(join(dir, `${n}.secret`), `pw-${n}\n`);
|
||||
given.push({ from: n, node: "anchor", as: n, secret: join(dir, `${n}.secret`) });
|
||||
}
|
||||
await writeFile(receives, JSON.stringify({ contributions: 1, requirement: "postgres-database", given }));
|
||||
};
|
||||
await ask(names);
|
||||
const w = { receives, ask, created: [], retired: [], removed: [], deleted: [], events: [], said: [] } as unknown as World;
|
||||
const was = console.error;
|
||||
console.error = (...a: unknown[]) => w.said.push(a.join(" "));
|
||||
const adapter = {
|
||||
async create(p: { as: string }) { w.created.push(p.as); },
|
||||
async remove(p: { as: string }) { w.removed.push(p.as); },
|
||||
async delete(p: { as: string }) { w.deleted.push(p.as); return { freedBytes: 42 }; },
|
||||
...(opts.retire === false ? {} : { async retire(p: { as: string; why: string }) { w.retired.push({ as: p.as, why: p.why }); } }),
|
||||
...(opts.held ? { async held() { return opts.held!; } } : {}),
|
||||
};
|
||||
w.p = runProvisioner("postgres-database", adapter, {
|
||||
receives, everyMs: EVERY, node: "anchor",
|
||||
announce: (event, body) => { assert.equal(event, RETIREMENT_EVENT); w.events.push(body); },
|
||||
});
|
||||
// A pass is counted by the file being read: wait wall time for n passes and some slack.
|
||||
w.done = () => { w.p(); console.error = was; };
|
||||
await until(() => w.created.length >= names.length);
|
||||
return w;
|
||||
}
|
||||
|
||||
const changes = (w: World) => w.events.map((e) => e.change);
|
||||
|
||||
test("a transient empty list for four passes retires nothing; five stable passes retire, data kept", async () => {
|
||||
const w = await world(["a", "b", "c"], {});
|
||||
try {
|
||||
// The unasked set is {a} — one of three, within the bound — seen for fewer than five passes.
|
||||
const p = w.p;
|
||||
// Run the loop by hand: stop the timer-driven one and drive passes directly is not exposed, so
|
||||
// the transient is made short in wall time instead: ask without a, then with a, quickly.
|
||||
await w.ask(["b", "c"]);
|
||||
await sleep(EVERY * 2);
|
||||
await w.ask(["a", "b", "c"]);
|
||||
await sleep(EVERY * 8);
|
||||
assert.deepEqual(w.retired, [], "retired on a transient result");
|
||||
assert.deepEqual(changes(w), []);
|
||||
// Stable: retired, through retire, and not removed.
|
||||
await w.ask(["b", "c"]);
|
||||
await until(() => w.retired.length === 1);
|
||||
assert.equal(w.retired[0].as, "a");
|
||||
assert.match(w.retired[0].why, /5 passes/);
|
||||
assert.deepEqual(w.removed, []);
|
||||
assert.deepEqual(w.deleted, []);
|
||||
assert.deepEqual(changes(w), ["retired"]);
|
||||
const r = await p.retirement();
|
||||
assert.deepEqual(r.retired.map((x) => x.consumer), ["a"]);
|
||||
assert.deepEqual(r.held, ["b", "c"]);
|
||||
assert.equal(r.stable_passes, 5);
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("the fifth pass and not the fourth retires", async () => {
|
||||
// Driven by the clock: five passes 10 ms apart take at least 40 ms after the first sees the set.
|
||||
const w = await world(["a", "b", "c"]);
|
||||
try {
|
||||
await w.ask(["b", "c"]);
|
||||
const start = Date.now();
|
||||
await until(() => w.retired.length === 1);
|
||||
assert.ok(Date.now() - start >= 4 * EVERY - 2, `retired after ${Date.now() - start} ms, before five passes`);
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("an unreadable file resets the count", async () => {
|
||||
const w = await world(["a", "b", "c"]);
|
||||
try {
|
||||
await w.ask(["b", "c"]);
|
||||
// Interleave unreadable passes so five readable ones never run in a row.
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await sleep(EVERY * 2);
|
||||
await rm(w.receives);
|
||||
await sleep(EVERY * 2);
|
||||
await w.ask(["b", "c"]);
|
||||
}
|
||||
assert.deepEqual(w.retired, [], "unreadable passes counted toward a stable result");
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("asked for again, a retired consumer is created again and announced re-enabled", async () => {
|
||||
const w = await world(["a", "b", "c"]);
|
||||
try {
|
||||
await w.ask(["b", "c"]);
|
||||
await until(() => w.retired.length === 1);
|
||||
const before = w.created.length;
|
||||
await w.ask(["a", "b", "c"]);
|
||||
await until(() => w.created.length > before);
|
||||
assert.equal(w.created.at(-1), "a");
|
||||
await until(() => changes(w).includes("reenabled"));
|
||||
assert.deepEqual((await w.p.retirement()).retired, []);
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("an adapter without retire is retired through remove, as before", async () => {
|
||||
const w = await world(["a", "b", "c"], { retire: false });
|
||||
try {
|
||||
await w.ask(["b", "c"]);
|
||||
await until(() => w.removed.length === 1);
|
||||
assert.deepEqual(w.removed, ["a"]);
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("four of four waits for a person: approve the exact set retires, another set is refused", async () => {
|
||||
const w = await world(["a", "b", "c", "d"]);
|
||||
try {
|
||||
await w.ask([]);
|
||||
await until(async () => (await w.p.retirement()).waiting !== null);
|
||||
await sleep(EVERY * 10);
|
||||
assert.deepEqual(w.retired, [], "retired over the bound without a person");
|
||||
assert.deepEqual(changes(w), ["waiting"], "waiting said more than once, or not");
|
||||
const waiting = w.events[0];
|
||||
assert.equal(waiting.held, 4);
|
||||
assert.match(String(waiting.bound), /more than 3, or more than 50% of the 4 held/);
|
||||
assert.deepEqual((waiting.consumers as { consumer: string }[]).map((c) => c.consumer), ["a", "b", "c", "d"]);
|
||||
|
||||
await assert.rejects(w.p.approve(["a", "b", "c"], { why: "x" }), /exactly that set/);
|
||||
await assert.rejects(w.p.approve(["a", "b", "c", "d"], { why: "" }), /why is required/);
|
||||
const done = await w.p.approve(["d", "c", "b", "a"], { why: "all four unassigned", by: "operator", via: "mesh-controller" });
|
||||
assert.deepEqual(done.retired, ["a", "b", "c", "d"]);
|
||||
assert.deepEqual(w.retired.map((r) => r.as), ["a", "b", "c", "d"]);
|
||||
assert.deepEqual(changes(w), ["waiting", "approved", "retired"]);
|
||||
assert.equal(w.events[1].via, "mesh-controller");
|
||||
assert.equal(w.events[1].by, "operator");
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("rejected, the set is kept active and not asked again while the mesh says the same", async () => {
|
||||
const w = await world(["a", "b", "c", "d"]);
|
||||
try {
|
||||
await w.ask([]);
|
||||
await until(async () => (await w.p.retirement()).waiting !== null);
|
||||
await assert.rejects(w.p.reject(["a"], { why: "no" }), /exactly that set/);
|
||||
const kept = await w.p.reject(["a", "b", "c", "d"], { why: "the mesh is wrong", by: "operator" });
|
||||
assert.deepEqual(kept.kept, ["a", "b", "c", "d"]);
|
||||
await sleep(EVERY * 12);
|
||||
assert.deepEqual(w.retired, []);
|
||||
assert.deepEqual(changes(w), ["waiting", "rejected"]);
|
||||
const r = await w.p.retirement();
|
||||
assert.equal(r.waiting, null);
|
||||
assert.deepEqual(r.rejected?.consumers.map((c) => c.consumer), ["a", "b", "c", "d"]);
|
||||
// Approving a rejected set later still works.
|
||||
await w.p.approve(["a", "b", "c", "d"], { why: "on second thought" });
|
||||
assert.equal(w.retired.length, 4);
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("a waiting set the mesh asks for again settles, and nothing is retired", async () => {
|
||||
const w = await world(["a", "b", "c", "d"]);
|
||||
try {
|
||||
await w.ask([]);
|
||||
await until(async () => (await w.p.retirement()).waiting !== null);
|
||||
await w.ask(["a", "b", "c", "d"]);
|
||||
await until(() => changes(w).includes("settled"));
|
||||
assert.equal((await w.p.retirement()).waiting, null);
|
||||
assert.deepEqual(w.retired, []);
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("delete refuses an active consumer and deletes only the named retired one", async () => {
|
||||
const w = await world(["a", "b", "c"]);
|
||||
try {
|
||||
await w.ask(["c"]);
|
||||
// Two of three is more than half: approve, then delete one.
|
||||
await until(async () => (await w.p.retirement()).waiting !== null);
|
||||
await w.p.approve(["a", "b"], { why: "gone" });
|
||||
await assert.rejects(w.p.deleteRetired("c", { why: "x", confirm: "c" }), /active, not retired/);
|
||||
await assert.rejects(w.p.deleteRetired("zz", { why: "x", confirm: "zz" }), /not retired here/);
|
||||
await assert.rejects(w.p.deleteRetired("a", { why: "x", confirm: "b" }), /confirm must repeat/);
|
||||
await assert.rejects(w.p.deleteRetired("a", { why: "", confirm: "a" }), /why is required/);
|
||||
const r = await w.p.deleteRetired("a", { why: "older than a month", by: "operator", confirm: "a" });
|
||||
assert.deepEqual(r, { deleted: "a", freed_bytes: 42 });
|
||||
assert.deepEqual(w.deleted, ["a"]);
|
||||
assert.deepEqual((await w.p.retirement()).retired.map((x) => x.consumer), ["b"]);
|
||||
assert.equal(w.events.at(-1)?.change, "deleted");
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("an addition acts on the first pass", async () => {
|
||||
const w = await world(["a"]);
|
||||
try {
|
||||
const before = w.created.length;
|
||||
await w.ask(["a", "b"]);
|
||||
await until(() => w.created.length > before, EVERY * 3);
|
||||
assert.equal(w.created.at(-1), "b");
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("what the backend holds at start is retired by the same rules", async () => {
|
||||
// "x" was made by the mesh and the mesh stopped asking for it while the provider was down.
|
||||
const w = await world(["a", "b", "c"], { held: ["a", "b", "c", "x"] });
|
||||
try {
|
||||
await until(() => w.retired.length === 1);
|
||||
assert.equal(w.retired[0].as, "x");
|
||||
} finally {
|
||||
w.done();
|
||||
}
|
||||
});
|
||||
|
||||
test("the four tools are registered for the module and answer", async () => {
|
||||
resetTools();
|
||||
const dir = await mkdtemp(join(tmpdir(), "retire-tools-"));
|
||||
const receives = join(dir, "r.json");
|
||||
await writeFile(receives, JSON.stringify({ requirement: "r", given: [] }));
|
||||
const p = runProvisioner("r", { async create() {}, async remove() {} }, { receives, everyMs: EVERY, module: "postgres" });
|
||||
try {
|
||||
const mine = collectTools({}).find((c) => c.module === "postgres");
|
||||
assert.deepEqual(mine?.tools.map((t) => t.name), [
|
||||
"provisioner_retirement", "provisioner_retire_approve", "provisioner_retire_reject", "provisioner_delete",
|
||||
]);
|
||||
const answer = (await mine!.tools[0].run({})) as { resource: string; waiting: unknown };
|
||||
assert.equal(answer.resource, "r");
|
||||
assert.equal(answer.waiting, null);
|
||||
assert.equal(provisionerTools(p).length, 4);
|
||||
} finally {
|
||||
p();
|
||||
resetTools();
|
||||
}
|
||||
});
|
||||
+7
-55
@@ -185,7 +185,7 @@ test("provisioner removes nothing while its contributions file cannot be used (n
|
||||
removed.push(p.as);
|
||||
},
|
||||
},
|
||||
{ receives, everyMs: 20, releaseEveryMs: 300 },
|
||||
{ receives, everyMs: 20 },
|
||||
);
|
||||
await waitFor(() => created.length === 2, 2000);
|
||||
|
||||
@@ -203,65 +203,17 @@ test("provisioner removes nothing while its contributions file cannot be used (n
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
|
||||
// The file read and saying so still withdraws — within the brake (to-be 45 Phase 2): two at once is
|
||||
// over its bound, so nothing goes at first, then one at a time as the brake lets them go.
|
||||
// The file read and saying so is a result — but two of two held is over the bound (ADR 0230):
|
||||
// nothing is retired without a person, and approving that exact set retires both.
|
||||
await writeFile(receives, doc([]));
|
||||
await new Promise((r) => setTimeout(r, 150));
|
||||
assert.deepEqual(removed, [], "two consumers were withdrawn at once");
|
||||
await waitFor(() => removed.length === 2, 3000);
|
||||
for (let i = 0; i < 200 && (await stop.retirement()).waiting === null; i++) await new Promise((r) => setTimeout(r, 10));
|
||||
assert.notEqual((await stop.retirement()).waiting, null, "the retirement never waited for a person");
|
||||
assert.deepEqual(removed, [], "two of two consumers were retired without a person");
|
||||
await stop.approve(["mesh_anchor_mail", "mesh_anchor_forge"], { why: "both unassigned on purpose", by: "test" });
|
||||
assert.deepEqual([...removed].sort(), ["mesh_anchor_forge", "mesh_anchor_mail"]);
|
||||
stop();
|
||||
});
|
||||
|
||||
test("provisioner brakes a withdrawal of more than one consumer, and keeps one asked for again (to-be 45 Phase 2)", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "prov-"));
|
||||
const created: string[] = [];
|
||||
const removed: string[] = [];
|
||||
const receives = join(dir, "postgres-database.json");
|
||||
const doc = (given: unknown[]): string =>
|
||||
JSON.stringify({ contributions: 1, requirement: "postgres-database", given });
|
||||
const seven: unknown[] = [];
|
||||
for (let i = 1; i <= 7; i++) {
|
||||
await writeFile(join(dir, `c${i}.secret`), `pw-${i}\n`);
|
||||
seven.push({ from: `c${i}`, node: "anchor", as: `consumer${i}`, secret: join(dir, `c${i}.secret`) });
|
||||
}
|
||||
await writeFile(receives, doc(seven));
|
||||
const said: string[] = [];
|
||||
const was = console.error;
|
||||
console.error = (...args: unknown[]) => said.push(args.join(" "));
|
||||
const stop = runProvisioner(
|
||||
"postgres-database",
|
||||
{
|
||||
async create(p) {
|
||||
created.push(p.as);
|
||||
},
|
||||
async remove(p) {
|
||||
removed.push(p.as);
|
||||
},
|
||||
},
|
||||
{ receives, everyMs: 10, releaseEveryMs: 60_000 },
|
||||
);
|
||||
try {
|
||||
await waitFor(() => created.length === 7, 2000);
|
||||
// A file read whole that names nobody — the shape of 241 its first fix does not catch.
|
||||
await writeFile(receives, doc([]));
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
assert.deepEqual(removed, [], "a withdrawal of seven at once was not braked");
|
||||
assert.ok(said.some((l) => l.includes("WITHDRAWAL BRAKED")), "the brake was not said");
|
||||
// The file is right again: nothing was withdrawn, and nothing will be.
|
||||
await writeFile(receives, doc(seven));
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
assert.deepEqual(removed, []);
|
||||
// One of seven goes as asked: within the bound.
|
||||
await writeFile(receives, doc(seven.slice(1)));
|
||||
await waitFor(() => removed.length === 1, 2000);
|
||||
assert.deepEqual(removed, ["consumer1"]);
|
||||
} finally {
|
||||
stop();
|
||||
console.error = was;
|
||||
}
|
||||
});
|
||||
|
||||
test("provisioner applies again what the backend no longer holds, and trusts memory without holds", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "prov-holds-"));
|
||||
await writeFile(join(dir, "webapp.secret"), "minted-pw\n");
|
||||
|
||||
Reference in New Issue
Block a user