mesh-sdk/test/retire.test.ts

// Retirement (novox/hq ADR 0230): a consumer the mesh stops asking for is retired — access disabled,
// data kept — only after the same result in five consecutive passes AND ten minutes (shortened here
// unless a test is about it), and a set too large waits for a person. An adapter that cannot disable
// is mark-only: remove is never called to retire. Deletion is a person's act, of a retired consumer only.
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtemp, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { runProvisioner, provisionerTools, RETIREMENT_EVENT, type Provisioner } from "../dist/provisioner/index.js";
import { resetTools, collectTools } from "../dist/tools/index.js";

const EVERY = 10;
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

async function until(cond: () => boolean | Promise<boolean>, ms = 3000): Promise<void> {
  const start = Date.now();
  while (!(await cond())) {
    if (Date.now() - start > ms) throw new Error("timed out waiting");
    await sleep(5);
  }
}

interface World {
  receives: string;
  ask(names: string[]): Promise<void>;
  created: string[];
  retired: { as: string; why: string }[];
  removed: string[];
  deleted: string[];
  reenabled: string[];
  events: Record<string, unknown>[];
  said: string[];
  p: Provisioner;
  done(): void;
}

async function world(
  names: string[],
  opts: { retire?: boolean; held?: string[]; del?: boolean; reenable?: boolean; stableForMs?: number } = {},
): Promise<World> {
  const dir = await mkdtemp(join(tmpdir(), "retire-"));
  const receives = join(dir, "postgres-database.json");
  const ask = async (list: string[]) => {
    const given = [];
    for (const n of list) {
      await writeFile(join(dir, `${n}.secret`), `pw-${n}\n`);
      given.push({ from: n, node: "anchor", as: n, secret: join(dir, `${n}.secret`) });
    }
    await writeFile(receives, JSON.stringify({ contributions: 1, requirement: "postgres-database", given }));
  };
  await ask(names);
  const w = { receives, ask, created: [], retired: [], removed: [], deleted: [], reenabled: [], events: [], said: [] } as unknown as World;
  const was = console.error;
  console.error = (...a: unknown[]) => w.said.push(a.join(" "));
  const adapter = {
    async create(p: { as: string }) { w.created.push(p.as); },
    async remove(p: { as: string }) { w.removed.push(p.as); },
    ...(opts.del === false ? {} : { async delete(p: { as: string }) { w.deleted.push(p.as); return { freedBytes: 42 }; } }),
    ...(opts.reenable ? { async reenable(p: { as: string }) { w.reenabled.push(p.as); } } : {}),
    ...(opts.retire === false ? {} : { async retire(p: { as: string; why: string }) { w.retired.push({ as: p.as, why: p.why }); } }),
    ...(opts.held ? { async held() { return opts.held!; } } : {}),
  };
  w.p = runProvisioner("postgres-database", adapter, {
    receives, everyMs: EVERY, node: "anchor", stableForMs: opts.stableForMs ?? 0,
    // Only retirement is these tests'; the standing (provisioner.failing, .recovered) is standing.test.ts's.
    announce: (event, body) => { if (event === RETIREMENT_EVENT) w.events.push(body); },
  });
  // A pass is counted by the file being read: wait wall time for n passes and some slack.
  w.done = () => { w.p(); console.error = was; };
  await until(() => w.created.length >= names.length);
  return w;
}

const changes = (w: World) => w.events.map((e) => e.change);

test("a transient empty list for four passes retires nothing; five stable passes retire, data kept", async () => {
  const w = await world(["a", "b", "c"], {});
  try {
    // The unasked set is {a} — one of three, within the bound — seen for fewer than five passes.
    const p = w.p;
    // Run the loop by hand: stop the timer-driven one and drive passes directly is not exposed, so
    // the transient is made short in wall time instead: ask without a, then with a, quickly.
    await w.ask(["b", "c"]);
    await sleep(EVERY * 2);
    await w.ask(["a", "b", "c"]);
    await sleep(EVERY * 8);
    assert.deepEqual(w.retired, [], "retired on a transient result");
    assert.deepEqual(changes(w), []);
    // Stable: retired, through retire, and not removed.
    await w.ask(["b", "c"]);
    await until(() => w.retired.length === 1);
    assert.equal(w.retired[0].as, "a");
    assert.match(w.retired[0].why, /\d+ passes over/);
    assert.deepEqual(w.removed, []);
    assert.deepEqual(w.deleted, []);
    assert.deepEqual(changes(w), ["retired"]);
    const r = await p.retirement();
    assert.deepEqual(r.retired.map((x) => x.consumer), ["a"]);
    assert.deepEqual(r.held, ["b", "c"]);
    assert.equal(r.stable_passes, 5);
  } finally {
    w.done();
  }
});

test("the fifth pass and not the fourth retires", async () => {
  // Driven by the clock: five passes 10 ms apart take at least 40 ms after the first sees the set.
  const w = await world(["a", "b", "c"]);
  try {
    await w.ask(["b", "c"]);
    const start = Date.now();
    await until(() => w.retired.length === 1);
    assert.ok(Date.now() - start >= 4 * EVERY - 2, `retired after ${Date.now() - start} ms, before five passes`);
  } finally {
    w.done();
  }
});

test("an unreadable file resets the count", async () => {
  const w = await world(["a", "b", "c"]);
  try {
    await w.ask(["b", "c"]);
    // Interleave unreadable passes so five readable ones never run in a row.
    for (let i = 0; i < 6; i++) {
      await sleep(EVERY * 2);
      await rm(w.receives);
      await sleep(EVERY * 2);
      await w.ask(["b", "c"]);
    }
    assert.deepEqual(w.retired, [], "unreadable passes counted toward a stable result");
  } finally {
    w.done();
  }
});

test("asked for again, a retired consumer is created again and announced re-enabled", async () => {
  const w = await world(["a", "b", "c"]);
  try {
    await w.ask(["b", "c"]);
    await until(() => w.retired.length === 1);
    const before = w.created.length;
    await w.ask(["a", "b", "c"]);
    await until(() => w.created.length > before);
    assert.equal(w.created.at(-1), "a");
    await until(() => changes(w).includes("reenabled"));
    assert.deepEqual((await w.p.retirement()).retired, []);
  } finally {
    w.done();
  }
});

test("five passes are not enough: the same set must also hold for stableForMs", async () => {
  const HOLD = 400;
  const w = await world(["a", "b", "c"], { stableForMs: HOLD });
  try {
    await w.ask(["b", "c"]);
    const start = Date.now();
    await sleep(EVERY * 10); // well past five passes, well short of the hold
    assert.deepEqual(w.retired, [], "retired after five passes, before the set held long enough");
    await until(() => w.retired.length === 1, 3000);
    assert.ok(Date.now() - start >= HOLD - 5, `retired after ${Date.now() - start} ms, before ${HOLD} ms`);
    assert.equal((await w.p.retirement()).stable_for_seconds, 0); // 400 ms rounds to 0 s
  } finally {
    w.done();
  }
});

test("the default hold is ten minutes", async () => {
  const { STABLE_FOR_MS } = await import("../dist/provisioner/index.js");
  assert.equal(STABLE_FOR_MS, 10 * 60_000);
});

test("an adapter without retire is MARK-ONLY: nothing changes in the backend, remove only on delete", async () => {
  const w = await world(["a", "b", "c"], { retire: false, del: false });
  try {
    await w.ask(["b", "c"]);
    await until(() => changes(w).includes("retired"));
    await sleep(EVERY * 10);
    assert.deepEqual(w.removed, [], "remove was called to retire");
    const ev = w.events.find((e) => e.change === "retired")!;
    assert.equal((ev.consumers as { access: string }[])[0].access, "kept");
    assert.ok(w.said.some((l) => /MARK ONLY/.test(l) && /KEEPS ITS ACCESS/.test(l)), "mark-only was not said loudly");
    const r = await w.p.retirement();
    assert.equal(r.retires_by, "mark-only");
    assert.deepEqual(r.retired.map((x) => [x.consumer, x.access]), [["a", "kept"]]);
    // Asked again: created again, nothing to undo.
    await w.ask(["a", "b", "c"]);
    await until(() => changes(w).includes("reenabled"));
    assert.deepEqual(w.removed, []);
    // Gone again, then a person's delete: the one path to remove.
    await w.ask(["b", "c"]);
    await until(async () => (await w.p.retirement()).retired.length === 1);
    await w.p.deleteRetired("a", { why: "gone", confirm: "a" });
    assert.deepEqual(w.removed, ["a"]);
  } finally {
    w.done();
  }
});

test("an adapter with retire and reenable: disabled on retire, reenabled before create when asked again", async () => {
  const w = await world(["a", "b", "c"], { reenable: true });
  try {
    await w.ask(["b", "c"]);
    await until(() => w.retired.length === 1);
    assert.equal((await w.p.retirement()).retires_by, "disable");
    assert.equal((await w.p.retirement()).retired[0].access, "disabled");
    const before = w.created.length;
    await w.ask(["a", "b", "c"]);
    await until(() => w.created.length > before);
    assert.deepEqual(w.reenabled, ["a"]);
    assert.deepEqual(w.removed, []);
  } finally {
    w.done();
  }
});

test("four of four waits for a person: approve the exact set retires, another set is refused", async () => {
  const w = await world(["a", "b", "c", "d"]);
  try {
    await w.ask([]);
    await until(async () => (await w.p.retirement()).waiting !== null);
    await sleep(EVERY * 10);
    assert.deepEqual(w.retired, [], "retired over the bound without a person");
    assert.deepEqual(changes(w), ["waiting"], "waiting said more than once, or not");
    const waiting = w.events[0];
    assert.equal(waiting.held, 4);
    assert.match(String(waiting.bound), /more than 3, or more than 50% of the 4 held/);
    assert.deepEqual((waiting.consumers as { consumer: string }[]).map((c) => c.consumer), ["a", "b", "c", "d"]);

    await assert.rejects(w.p.approve(["a", "b", "c"], { why: "x" }), /exactly that set/);
    await assert.rejects(w.p.approve(["a", "b", "c", "d"], { why: "" }), /why is required/);
    const done = await w.p.approve(["d", "c", "b", "a"], { why: "all four unassigned", by: "operator", via: "mesh-controller" });
    assert.deepEqual(done.retired, ["a", "b", "c", "d"]);
    assert.deepEqual(w.retired.map((r) => r.as), ["a", "b", "c", "d"]);
    assert.deepEqual(changes(w), ["waiting", "approved", "retired"]);
    assert.equal(w.events[1].via, "mesh-controller");
    assert.equal(w.events[1].by, "operator");
  } finally {
    w.done();
  }
});

test("rejected, the set is kept active and not asked again while the mesh says the same", async () => {
  const w = await world(["a", "b", "c", "d"]);
  try {
    await w.ask([]);
    await until(async () => (await w.p.retirement()).waiting !== null);
    await assert.rejects(w.p.reject(["a"], { why: "no" }), /exactly that set/);
    const kept = await w.p.reject(["a", "b", "c", "d"], { why: "the mesh is wrong", by: "operator" });
    assert.deepEqual(kept.kept, ["a", "b", "c", "d"]);
    await sleep(EVERY * 12);
    assert.deepEqual(w.retired, []);
    assert.deepEqual(changes(w), ["waiting", "rejected"]);
    const r = await w.p.retirement();
    assert.equal(r.waiting, null);
    assert.deepEqual(r.rejected?.consumers.map((c) => c.consumer), ["a", "b", "c", "d"]);
    // Approving a rejected set later still works.
    await w.p.approve(["a", "b", "c", "d"], { why: "on second thought" });
    assert.equal(w.retired.length, 4);
  } finally {
    w.done();
  }
});

test("a waiting set the mesh asks for again settles, and nothing is retired", async () => {
  const w = await world(["a", "b", "c", "d"]);
  try {
    await w.ask([]);
    await until(async () => (await w.p.retirement()).waiting !== null);
    await w.ask(["a", "b", "c", "d"]);
    await until(() => changes(w).includes("settled"));
    assert.equal((await w.p.retirement()).waiting, null);
    assert.deepEqual(w.retired, []);
  } finally {
    w.done();
  }
});

test("delete refuses an active consumer and deletes only the named retired one", async () => {
  const w = await world(["a", "b", "c"]);
  try {
    await w.ask(["c"]);
    // Two of three is more than half: approve, then delete one.
    await until(async () => (await w.p.retirement()).waiting !== null);
    await w.p.approve(["a", "b"], { why: "gone" });
    await assert.rejects(w.p.deleteRetired("c", { why: "x", confirm: "c" }), /active, not retired/);
    await assert.rejects(w.p.deleteRetired("zz", { why: "x", confirm: "zz" }), /not retired here/);
    await assert.rejects(w.p.deleteRetired("a", { why: "x", confirm: "b" }), /confirm must repeat/);
    await assert.rejects(w.p.deleteRetired("a", { why: "", confirm: "a" }), /why is required/);
    const r = await w.p.deleteRetired("a", { why: "older than a month", by: "operator", confirm: "a" });
    assert.deepEqual(r, { deleted: "a", freed_bytes: 42 });
    assert.deepEqual(w.deleted, ["a"]);
    assert.deepEqual((await w.p.retirement()).retired.map((x) => x.consumer), ["b"]);
    assert.equal(w.events.at(-1)?.change, "deleted");
  } finally {
    w.done();
  }
});

test("an addition acts on the first pass", async () => {
  const w = await world(["a"]);
  try {
    const before = w.created.length;
    await w.ask(["a", "b"]);
    await until(() => w.created.length > before, EVERY * 3);
    assert.equal(w.created.at(-1), "b");
  } finally {
    w.done();
  }
});

test("what the backend holds at start is retired by the same rules", async () => {
  // "x" was made by the mesh and the mesh stopped asking for it while the provider was down.
  const w = await world(["a", "b", "c"], { held: ["a", "b", "c", "x"] });
  try {
    await until(() => w.retired.length === 1);
    assert.equal(w.retired[0].as, "x");
  } finally {
    w.done();
  }
});

test("the four tools are registered for the module and answer", async () => {
  resetTools();
  const dir = await mkdtemp(join(tmpdir(), "retire-tools-"));
  const receives = join(dir, "r.json");
  await writeFile(receives, JSON.stringify({ requirement: "r", given: [] }));
  const p = runProvisioner("r", { async create() {}, async remove() {} }, { receives, everyMs: EVERY, module: "postgres" });
  try {
    const mine = collectTools({}).find((c) => c.module === "postgres");
    assert.deepEqual(mine?.tools.map((t) => t.name), [
      "provisioner_retirement", "provisioner_retire_approve", "provisioner_retire_reject", "provisioner_delete",
    ]);
    const answer = (await mine!.tools[0].run({})) as { resource: string; waiting: unknown };
    assert.equal(answer.resource, "r");
    assert.equal(answer.waiting, null);
    assert.equal(provisionerTools(p).length, 4);
  } finally {
    p();
    resetTools();
  }
});