0.1.11: a provisioner brakes a withdrawal larger than its bound (hq to-be 45 Phase 2)

Issue 241 withdrew seven consumers in one pass; 0.1.10 refuses a file it
cannot read, and a file read whole that names nobody still withdrew
everybody. A pass that would withdraw more than withdrawAtOnce consumers
(1), or more than withdrawFraction (0.5) of those held, now withdraws
nothing and says so; while the mesh goes on not asking, one is let go every
releaseEveryMs (an hour). A consumer asked for again is kept.

src/provisioner/index.ts

@@ -72,6 +72,17 @@ export interface ProvisionerOptions {
   /** How long one `holds` may take before it counts as "could not ask". Defaults to 30000: the
    *  loop is sequential, so a check that hangs would otherwise stop every consumer's provisioning. */
   holdsTimeoutMs?: number;
+  /** The most consumers one pass may withdraw (novox/hq to-be 45 Phase 2, ADR 0227 rule 4). Defaults
+   *  to 1: a pass that would withdraw more withdraws nothing — the shape of issue 241, where one
+   *  misread file withdrew seven at once — and says so. */
+  withdrawAtOnce?: number;
+  /** The largest share of the consumers held one pass may withdraw, where it holds more than one.
+   *  Defaults to 0.5. */
+  withdrawFraction?: number;
+  /** While a braked withdrawal goes on being asked for, one consumer is let go this often, in ms.
+   *  Defaults to an hour: an intended unassignment of many completes without a hand, and a mistaken
+   *  one costs at most one consumer an hour while it is said. */
+  releaseEveryMs?: number;
 }
 
 /** The longest a consumer whose `create` keeps failing to satisfy `holds` waits between checks. */
@@ -106,6 +117,14 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi
   const everyMs = opts.everyMs ?? 5000;
   const verifyEveryMs = opts.verifyEveryMs ?? 60_000;
   const holdsTimeoutMs = opts.holdsTimeoutMs ?? 30_000;
+  const withdrawAtOnce = opts.withdrawAtOnce ?? 1;
+  const withdrawFraction = opts.withdrawFraction ?? 0.5;
+  const releaseEveryMs = opts.releaseEveryMs ?? 60 * 60_000;
+  // Consumers a braked pass kept, by when first kept; when the brake last let one go; and the set it
+  // last said, so a pass repeats nothing.
+  const braked = new Map<string, number>();
+  let releasedAt = 0;
+  let brakeSaid = "";
   let verifiedAt = 0;
   // Consumers the backend reported lost, and how many times in a row. A `create` that succeeded and
   // is still not held on the next check will never be: something in the adapter disagrees with
@@ -237,14 +256,54 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi
     // Withdraw every login the provider made that the mesh no longer asks for. The mesh drops a
     // consumer from the file when it goes away; that is how a login is withdrawn rather than kept
     // working for ever. Only logins this harness created are touched.
-    for (const [as, was] of [...applied.entries()]) {
-      if (wantByAs.has(as)) continue;
+    //
+    // **Within a bound** (novox/hq to-be 45 Phase 2, ADR 0227 rule 4): a pass that would withdraw more
+    // than `withdrawAtOnce` consumers, or more than `withdrawFraction` of those held, withdraws
+    // nothing and says so; one is let go every `releaseEveryMs` while the mesh goes on not asking.
+    const withdrawing = [...applied.keys()].filter((as) => !wantByAs.has(as)).sort();
+    for (const as of [...braked.keys()]) if (wantByAs.has(as)) braked.delete(as);
+    let releasing = withdrawing;
+    const over =
+      withdrawing.length > 0 &&
+      (withdrawing.length > withdrawAtOnce ||
+        (applied.size > 1 && withdrawing.length > withdrawFraction * applied.size));
+    if (over) {
+      const set = withdrawing.join(", ");
+      if (set !== brakeSaid) {
+        console.error(
+          `[provisioner:${resource}] WITHDRAWAL BRAKED: this pass would withdraw ${withdrawing.length} of the ` +
+            `${applied.size} consumer(s) this provider holds (${set}), more than ${withdrawAtOnce} at once or ` +
+            `${Math.round(withdrawFraction * 100)}% of them. Nothing is withdrawn; one is let go every ` +
+            `${Math.round(releaseEveryMs / 1000)}s while the mesh goes on not asking for them (novox/hq ADR 0227 rule 4)`,
+        );
+        brakeSaid = set;
+      }
+      if (braked.size === 0) releasedAt = Date.now();
+      for (const as of withdrawing) if (!braked.has(as)) braked.set(as, Date.now());
+      releasing = [];
+      if (Date.now() - releasedAt >= releaseEveryMs) {
+        releasedAt = Date.now();
+        releasing = [withdrawing[0]];
+        console.error(
+          `[provisioner:${resource}] ${withdrawing[0]}: released by the withdrawal brake; ` +
+            `${withdrawing.length - 1} more kept`,
+        );
+      }
+    } else if (braked.size > 0) {
+      console.error(`[provisioner:${resource}] the withdrawal is within its bound again`);
+      braked.clear();
+      brakeSaid = "";
+    }
+    for (const as of releasing) {
+      const was = applied.get(as);
+      if (was === undefined) continue;
       // Said before it is done: a removal is the one act here that destroys what a consumer kept.
       console.error(`[provisioner:${resource}] ${as}: no longer in ${receives}; removing it from the backend`);
       try {
         await adapter.remove({ as, derived: was.derived });
         applied.delete(as);
         lost.delete(as);
+        braked.delete(as);
       } catch (err) {
         console.error(`[provisioner:${resource}] ${as}: remove failed, will retry: ${err}`);
       }