Merge pull request '0.1.11: a provisioner brakes a withdrawal larger than its bound (hq to-be 45 Phase 2)' (#8) from feat/a-core-that-cannot-fail-silently-phase-2 into main

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@novox/mesh-sdk",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@novox/mesh-sdk",
-      "version": "0.1.10",
+      "version": "0.1.11",
       "devDependencies": {
         "@types/node": "^22.0.0",
         "typescript": "^5.6.0"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@novox/mesh-sdk",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "description": "The stable spine a Novox Mesh module's own code builds against.",
   "type": "module",
   "exports": {

src/provisioner/index.ts

@@ -72,6 +72,17 @@ export interface ProvisionerOptions {
   /** How long one `holds` may take before it counts as "could not ask". Defaults to 30000: the
    *  loop is sequential, so a check that hangs would otherwise stop every consumer's provisioning. */
   holdsTimeoutMs?: number;
+  /** The most consumers one pass may withdraw (novox/hq to-be 45 Phase 2, ADR 0227 rule 4). Defaults
+   *  to 1: a pass that would withdraw more withdraws nothing — the shape of issue 241, where one
+   *  misread file withdrew seven at once — and says so. */
+  withdrawAtOnce?: number;
+  /** The largest share of the consumers held one pass may withdraw, where it holds more than one.
+   *  Defaults to 0.5. */
+  withdrawFraction?: number;
+  /** While a braked withdrawal goes on being asked for, one consumer is let go this often, in ms.
+   *  Defaults to an hour: an intended unassignment of many completes without a hand, and a mistaken
+   *  one costs at most one consumer an hour while it is said. */
+  releaseEveryMs?: number;
 }
 
 /** The longest a consumer whose `create` keeps failing to satisfy `holds` waits between checks. */
@@ -106,6 +117,14 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi
   const everyMs = opts.everyMs ?? 5000;
   const verifyEveryMs = opts.verifyEveryMs ?? 60_000;
   const holdsTimeoutMs = opts.holdsTimeoutMs ?? 30_000;
+  const withdrawAtOnce = opts.withdrawAtOnce ?? 1;
+  const withdrawFraction = opts.withdrawFraction ?? 0.5;
+  const releaseEveryMs = opts.releaseEveryMs ?? 60 * 60_000;
+  // Consumers a braked pass kept, by when first kept; when the brake last let one go; and the set it
+  // last said, so a pass repeats nothing.
+  const braked = new Map<string, number>();
+  let releasedAt = 0;
+  let brakeSaid = "";
   let verifiedAt = 0;
   // Consumers the backend reported lost, and how many times in a row. A `create` that succeeded and
   // is still not held on the next check will never be: something in the adapter disagrees with
@@ -237,14 +256,54 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi
     // Withdraw every login the provider made that the mesh no longer asks for. The mesh drops a
     // consumer from the file when it goes away; that is how a login is withdrawn rather than kept
     // working for ever. Only logins this harness created are touched.
-    for (const [as, was] of [...applied.entries()]) {
-      if (wantByAs.has(as)) continue;
+    //
+    // **Within a bound** (novox/hq to-be 45 Phase 2, ADR 0227 rule 4): a pass that would withdraw more
+    // than `withdrawAtOnce` consumers, or more than `withdrawFraction` of those held, withdraws
+    // nothing and says so; one is let go every `releaseEveryMs` while the mesh goes on not asking.
+    const withdrawing = [...applied.keys()].filter((as) => !wantByAs.has(as)).sort();
+    for (const as of [...braked.keys()]) if (wantByAs.has(as)) braked.delete(as);
+    let releasing = withdrawing;
+    const over =
+      withdrawing.length > 0 &&
+      (withdrawing.length > withdrawAtOnce ||
+        (applied.size > 1 && withdrawing.length > withdrawFraction * applied.size));
+    if (over) {
+      const set = withdrawing.join(", ");
+      if (set !== brakeSaid) {
+        console.error(
+          `[provisioner:${resource}] WITHDRAWAL BRAKED: this pass would withdraw ${withdrawing.length} of the ` +
+            `${applied.size} consumer(s) this provider holds (${set}), more than ${withdrawAtOnce} at once or ` +
+            `${Math.round(withdrawFraction * 100)}% of them. Nothing is withdrawn; one is let go every ` +
+            `${Math.round(releaseEveryMs / 1000)}s while the mesh goes on not asking for them (novox/hq ADR 0227 rule 4)`,
+        );
+        brakeSaid = set;
+      }
+      if (braked.size === 0) releasedAt = Date.now();
+      for (const as of withdrawing) if (!braked.has(as)) braked.set(as, Date.now());
+      releasing = [];
+      if (Date.now() - releasedAt >= releaseEveryMs) {
+        releasedAt = Date.now();
+        releasing = [withdrawing[0]];
+        console.error(
+          `[provisioner:${resource}] ${withdrawing[0]}: released by the withdrawal brake; ` +
+            `${withdrawing.length - 1} more kept`,
+        );
+      }
+    } else if (braked.size > 0) {
+      console.error(`[provisioner:${resource}] the withdrawal is within its bound again`);
+      braked.clear();
+      brakeSaid = "";
+    }
+    for (const as of releasing) {
+      const was = applied.get(as);
+      if (was === undefined) continue;
       // Said before it is done: a removal is the one act here that destroys what a consumer kept.
       console.error(`[provisioner:${resource}] ${as}: no longer in ${receives}; removing it from the backend`);
       try {
         await adapter.remove({ as, derived: was.derived });
         applied.delete(as);
         lost.delete(as);
+        braked.delete(as);
       } catch (err) {
         console.error(`[provisioner:${resource}] ${as}: remove failed, will retry: ${err}`);
       }

test/sdk.test.ts

@@ -185,7 +185,7 @@ test("provisioner removes nothing while its contributions file cannot be used (n
         removed.push(p.as);
       },
     },
-    { receives, everyMs: 20 },
+    { receives, everyMs: 20, releaseEveryMs: 300 },
   );
   await waitFor(() => created.length === 2, 2000);
 
@@ -203,13 +203,65 @@ test("provisioner removes nothing while its contributions file cannot be used (n
     await new Promise((r) => setTimeout(r, 100));
   }
 
-  // The file read and saying so still withdraws: an explicit empty list removes both.
+  // The file read and saying so still withdraws — within the brake (to-be 45 Phase 2): two at once is
+  // over its bound, so nothing goes at first, then one at a time as the brake lets them go.
   await writeFile(receives, doc([]));
-  await waitFor(() => removed.length === 2, 2000);
+  await new Promise((r) => setTimeout(r, 150));
+  assert.deepEqual(removed, [], "two consumers were withdrawn at once");
+  await waitFor(() => removed.length === 2, 3000);
   assert.deepEqual([...removed].sort(), ["mesh_anchor_forge", "mesh_anchor_mail"]);
   stop();
 });
 
+test("provisioner brakes a withdrawal of more than one consumer, and keeps one asked for again (to-be 45 Phase 2)", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "prov-"));
+  const created: string[] = [];
+  const removed: string[] = [];
+  const receives = join(dir, "postgres-database.json");
+  const doc = (given: unknown[]): string =>
+    JSON.stringify({ contributions: 1, requirement: "postgres-database", given });
+  const seven: unknown[] = [];
+  for (let i = 1; i <= 7; i++) {
+    await writeFile(join(dir, `c${i}.secret`), `pw-${i}\n`);
+    seven.push({ from: `c${i}`, node: "anchor", as: `consumer${i}`, secret: join(dir, `c${i}.secret`) });
+  }
+  await writeFile(receives, doc(seven));
+  const said: string[] = [];
+  const was = console.error;
+  console.error = (...args: unknown[]) => said.push(args.join(" "));
+  const stop = runProvisioner(
+    "postgres-database",
+    {
+      async create(p) {
+        created.push(p.as);
+      },
+      async remove(p) {
+        removed.push(p.as);
+      },
+    },
+    { receives, everyMs: 10, releaseEveryMs: 60_000 },
+  );
+  try {
+    await waitFor(() => created.length === 7, 2000);
+    // A file read whole that names nobody — the shape of 241 its first fix does not catch.
+    await writeFile(receives, doc([]));
+    await new Promise((r) => setTimeout(r, 200));
+    assert.deepEqual(removed, [], "a withdrawal of seven at once was not braked");
+    assert.ok(said.some((l) => l.includes("WITHDRAWAL BRAKED")), "the brake was not said");
+    // The file is right again: nothing was withdrawn, and nothing will be.
+    await writeFile(receives, doc(seven));
+    await new Promise((r) => setTimeout(r, 100));
+    assert.deepEqual(removed, []);
+    // One of seven goes as asked: within the bound.
+    await writeFile(receives, doc(seven.slice(1)));
+    await waitFor(() => removed.length === 1, 2000);
+    assert.deepEqual(removed, ["consumer1"]);
+  } finally {
+    stop();
+    console.error = was;
+  }
+});
+
 test("provisioner applies again what the backend no longer holds, and trusts memory without holds", async () => {
   const dir = await mkdtemp(join(tmpdir(), "prov-holds-"));
   await writeFile(join(dir, "webapp.secret"), "minted-pw\n");