diff --git a/package-lock.json b/package-lock.json index edc358a..1eb9bcc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@novox/mesh-sdk", - "version": "0.1.10", + "version": "0.1.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@novox/mesh-sdk", - "version": "0.1.10", + "version": "0.1.11", "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" diff --git a/package.json b/package.json index 17160ec..a76a4c5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@novox/mesh-sdk", - "version": "0.1.10", + "version": "0.1.11", "description": "The stable spine a Novox Mesh module's own code builds against.", "type": "module", "exports": { diff --git a/src/provisioner/index.ts b/src/provisioner/index.ts index 9b1978b..e0a8b13 100644 --- a/src/provisioner/index.ts +++ b/src/provisioner/index.ts @@ -72,6 +72,17 @@ export interface ProvisionerOptions { /** How long one `holds` may take before it counts as "could not ask". Defaults to 30000: the * loop is sequential, so a check that hangs would otherwise stop every consumer's provisioning. */ holdsTimeoutMs?: number; + /** The most consumers one pass may withdraw (novox/hq to-be 45 Phase 2, ADR 0227 rule 4). Defaults + * to 1: a pass that would withdraw more withdraws nothing — the shape of issue 241, where one + * misread file withdrew seven at once — and says so. */ + withdrawAtOnce?: number; + /** The largest share of the consumers held one pass may withdraw, where it holds more than one. + * Defaults to 0.5. */ + withdrawFraction?: number; + /** While a braked withdrawal goes on being asked for, one consumer is let go this often, in ms. + * Defaults to an hour: an intended unassignment of many completes without a hand, and a mistaken + * one costs at most one consumer an hour while it is said. */ + releaseEveryMs?: number; } /** The longest a consumer whose `create` keeps failing to satisfy `holds` waits between checks. */ @@ -106,6 +117,14 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi const everyMs = opts.everyMs ?? 5000; const verifyEveryMs = opts.verifyEveryMs ?? 60_000; const holdsTimeoutMs = opts.holdsTimeoutMs ?? 30_000; + const withdrawAtOnce = opts.withdrawAtOnce ?? 1; + const withdrawFraction = opts.withdrawFraction ?? 0.5; + const releaseEveryMs = opts.releaseEveryMs ?? 60 * 60_000; + // Consumers a braked pass kept, by when first kept; when the brake last let one go; and the set it + // last said, so a pass repeats nothing. + const braked = new Map(); + let releasedAt = 0; + let brakeSaid = ""; let verifiedAt = 0; // Consumers the backend reported lost, and how many times in a row. A `create` that succeeded and // is still not held on the next check will never be: something in the adapter disagrees with @@ -237,14 +256,54 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi // Withdraw every login the provider made that the mesh no longer asks for. The mesh drops a // consumer from the file when it goes away; that is how a login is withdrawn rather than kept // working for ever. Only logins this harness created are touched. - for (const [as, was] of [...applied.entries()]) { - if (wantByAs.has(as)) continue; + // + // **Within a bound** (novox/hq to-be 45 Phase 2, ADR 0227 rule 4): a pass that would withdraw more + // than `withdrawAtOnce` consumers, or more than `withdrawFraction` of those held, withdraws + // nothing and says so; one is let go every `releaseEveryMs` while the mesh goes on not asking. + const withdrawing = [...applied.keys()].filter((as) => !wantByAs.has(as)).sort(); + for (const as of [...braked.keys()]) if (wantByAs.has(as)) braked.delete(as); + let releasing = withdrawing; + const over = + withdrawing.length > 0 && + (withdrawing.length > withdrawAtOnce || + (applied.size > 1 && withdrawing.length > withdrawFraction * applied.size)); + if (over) { + const set = withdrawing.join(", "); + if (set !== brakeSaid) { + console.error( + `[provisioner:${resource}] WITHDRAWAL BRAKED: this pass would withdraw ${withdrawing.length} of the ` + + `${applied.size} consumer(s) this provider holds (${set}), more than ${withdrawAtOnce} at once or ` + + `${Math.round(withdrawFraction * 100)}% of them. Nothing is withdrawn; one is let go every ` + + `${Math.round(releaseEveryMs / 1000)}s while the mesh goes on not asking for them (novox/hq ADR 0227 rule 4)`, + ); + brakeSaid = set; + } + if (braked.size === 0) releasedAt = Date.now(); + for (const as of withdrawing) if (!braked.has(as)) braked.set(as, Date.now()); + releasing = []; + if (Date.now() - releasedAt >= releaseEveryMs) { + releasedAt = Date.now(); + releasing = [withdrawing[0]]; + console.error( + `[provisioner:${resource}] ${withdrawing[0]}: released by the withdrawal brake; ` + + `${withdrawing.length - 1} more kept`, + ); + } + } else if (braked.size > 0) { + console.error(`[provisioner:${resource}] the withdrawal is within its bound again`); + braked.clear(); + brakeSaid = ""; + } + for (const as of releasing) { + const was = applied.get(as); + if (was === undefined) continue; // Said before it is done: a removal is the one act here that destroys what a consumer kept. console.error(`[provisioner:${resource}] ${as}: no longer in ${receives}; removing it from the backend`); try { await adapter.remove({ as, derived: was.derived }); applied.delete(as); lost.delete(as); + braked.delete(as); } catch (err) { console.error(`[provisioner:${resource}] ${as}: remove failed, will retry: ${err}`); } diff --git a/test/sdk.test.ts b/test/sdk.test.ts index 129f06c..2e779cd 100644 --- a/test/sdk.test.ts +++ b/test/sdk.test.ts @@ -185,7 +185,7 @@ test("provisioner removes nothing while its contributions file cannot be used (n removed.push(p.as); }, }, - { receives, everyMs: 20 }, + { receives, everyMs: 20, releaseEveryMs: 300 }, ); await waitFor(() => created.length === 2, 2000); @@ -203,13 +203,65 @@ test("provisioner removes nothing while its contributions file cannot be used (n await new Promise((r) => setTimeout(r, 100)); } - // The file read and saying so still withdraws: an explicit empty list removes both. + // The file read and saying so still withdraws — within the brake (to-be 45 Phase 2): two at once is + // over its bound, so nothing goes at first, then one at a time as the brake lets them go. await writeFile(receives, doc([])); - await waitFor(() => removed.length === 2, 2000); + await new Promise((r) => setTimeout(r, 150)); + assert.deepEqual(removed, [], "two consumers were withdrawn at once"); + await waitFor(() => removed.length === 2, 3000); assert.deepEqual([...removed].sort(), ["mesh_anchor_forge", "mesh_anchor_mail"]); stop(); }); +test("provisioner brakes a withdrawal of more than one consumer, and keeps one asked for again (to-be 45 Phase 2)", async () => { + const dir = await mkdtemp(join(tmpdir(), "prov-")); + const created: string[] = []; + const removed: string[] = []; + const receives = join(dir, "postgres-database.json"); + const doc = (given: unknown[]): string => + JSON.stringify({ contributions: 1, requirement: "postgres-database", given }); + const seven: unknown[] = []; + for (let i = 1; i <= 7; i++) { + await writeFile(join(dir, `c${i}.secret`), `pw-${i}\n`); + seven.push({ from: `c${i}`, node: "anchor", as: `consumer${i}`, secret: join(dir, `c${i}.secret`) }); + } + await writeFile(receives, doc(seven)); + const said: string[] = []; + const was = console.error; + console.error = (...args: unknown[]) => said.push(args.join(" ")); + const stop = runProvisioner( + "postgres-database", + { + async create(p) { + created.push(p.as); + }, + async remove(p) { + removed.push(p.as); + }, + }, + { receives, everyMs: 10, releaseEveryMs: 60_000 }, + ); + try { + await waitFor(() => created.length === 7, 2000); + // A file read whole that names nobody — the shape of 241 its first fix does not catch. + await writeFile(receives, doc([])); + await new Promise((r) => setTimeout(r, 200)); + assert.deepEqual(removed, [], "a withdrawal of seven at once was not braked"); + assert.ok(said.some((l) => l.includes("WITHDRAWAL BRAKED")), "the brake was not said"); + // The file is right again: nothing was withdrawn, and nothing will be. + await writeFile(receives, doc(seven)); + await new Promise((r) => setTimeout(r, 100)); + assert.deepEqual(removed, []); + // One of seven goes as asked: within the bound. + await writeFile(receives, doc(seven.slice(1))); + await waitFor(() => removed.length === 1, 2000); + assert.deepEqual(removed, ["consumer1"]); + } finally { + stop(); + console.error = was; + } +}); + test("provisioner applies again what the backend no longer holds, and trusts memory without holds", async () => { const dir = await mkdtemp(join(tmpdir(), "prov-holds-")); await writeFile(join(dir, "webapp.secret"), "minted-pw\n");