Feature 57 (config and provisioning): release mesh-sdk, the reviewed release branch on top of main

src/provider-loop/index.ts

@@ -0,0 +1,921 @@
+// The provider loop: the reconcile harness every provider on the controller's grant files shares.
+// Called `provisioner` until design 57 (novox/hq issue 678): that word is mesh-provisioner's, which
+// places every value a module needs (ADR 0349), so the loop is named for what it is. The old path
+// `@novox/mesh-sdk/provisioner` and the names `runProvisioner`, `provisionerTools`, `Provisioner` and
+// `ProvisionerOptions` stay as aliases of these until the providers that import them move to the Go
+// helper that answers consumers over the bus (mesh-sdk go/provider, design 57 B2 and B3) and this loop
+// retires with the grant files. The events (`provisioner.failing`, `.recovered`, `.retirement`) and the
+// tool names (`provisioner_retirement`, …) are what the controller and the operator read on the bus,
+// and keep their names.
+//
+// The reconcile harness every provider shares. The loop below is identical for postgres, redis,
+// minio and umami: read the contributions the mesh delivered, bring each consumer's resource into
+// being through the provider's adapter under the login and password the mesh minted, and retire
+// what the mesh no longer asks for. A module writes ONLY the adapter — the per-service half — which
+// is why this lives in the sdk and the adapter lives in the module (novox/hq ADR 0039).
+//
+// **A provider creates the credential the mesh minted, and seals nothing (novox/hq ADR 0048).** The
+// control plane mints one password per consumer and seals it to this node; the host unseals it into
+// the file the contribution names. The provider does not generate a password, does not seal one, and
+// does not hand one back — the consumer already receives its copy through the mesh's own channel.
+// This is why the old $MESH_SEAL_KEY, the symmetric seal, and the *.grant.json / *.credential files
+// are gone: they described a second, contradictory credential path the mesh does not have.
+//
+// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer
+// whose create, check or secret has failed without one success in between for `failingAfterMs` is
+// announced as `provisioner.failing` — naming the consumer, its machine and the class of error — and
+// again every `sayAgainEveryMs` while it lasts; the first success after that is
+// `provisioner.recovered`, as is a consumer the mesh stopped asking for, and the first success for
+// each consumer since this process started. The controller keeps the newest per provider and consumer
+// and `status` names it. Until novox/hq issue 272 only the Go copy of this loop (postgres, keycloak)
+// said it; every provider on this one failed its consumers in its journal alone.
+
+import { readFile } from "node:fs/promises";
+
+import type { ToolDefinition } from "../contracts/index.js";
+import { emit } from "../events/index.js";
+import { registerModuleTools } from "../tools/index.js";
+
+/** One consumer's resource to bring into being — everything the mesh derived and delivered. */
+export interface Provision {
+  /** The login the mesh derived and gave the consumer to present. The provider creates exactly
+   *  this name — a name the consumer cannot learn is a name it cannot authenticate with. */
+  readonly as: string;
+  /** The password the mesh minted for this consumer, read from the file the mesh sealed to this
+   *  node and the host unsealed. The provider sets it; it never invents one. */
+  readonly password: string;
+  /** What the consumer contributed, per the interface's spec keys (e.g. `{ name: "umami" }`). */
+  readonly values: Readonly<Record<string, unknown>>;
+  /** What this provider's own definition said it derives for this consumer, already derived by
+   *  the mesh (novox/hq ADR 0201).
+   *
+   *  **The rule is declared, not computed here.** A provider that names the resource it gives each
+   *  consumer — a bucket, a database, a vhost — writes that name in its `serves` block with
+   *  `${consumer:as}` or `${consumer:as:dns}` in it. The mesh fills it for this consumer and hands
+   *  the same filled value to the consumer's own configuration, so the two ends cannot disagree.
+   *  An adapter reads it from here; an adapter that recomputed it would be the second rule this
+   *  removed. Empty for a provider that derives nothing. */
+  readonly derived: Readonly<Record<string, unknown>>;
+  /** Where the consumer is, for a provider that must reach back to it. Usually unused. */
+  readonly at?: string;
+  /** The consumer node, for logging and lifecycle events. */
+  readonly consumer?: string;
+}
+
+/** What a provider implements — the only per-service code. It is handed the login and password the
+ *  mesh made and brings the resource into being under them, or withdraws it. It returns nothing:
+ *  the credential is the mesh's, already delivered to the consumer. */
+export interface Adapter {
+  create(p: Provision): Promise<void>;
+  /** Remove what `create` made for this consumer. **Called only to DELETE a retired consumer, on a
+   *  person's approved `cleanup delete`, and only when the adapter has no `delete`** (novox/hq ADR
+   *  0230): many providers' remove destroys something — a ledger file, a public DNS record, a user —
+   *  and nothing is destroyed without a person. Retiring never calls it. `derived` is what the mesh
+   *  last derived for it, remembered by the harness: the mesh has already dropped the consumer from
+   *  the contributions file by then, so it cannot be read again, and a provider that recomputed it
+   *  would be keeping the second copy of a rule this removed (novox/hq ADR 0201). */
+  remove(p: { readonly as: string; readonly derived: Readonly<Record<string, unknown>> }): Promise<void>;
+  /** Optional: whether the backend still holds this consumer's credential exactly as `p` says.
+   *  Asked of every consumer already applied, every `verifyEveryMs`. `false` makes the harness apply
+   *  it again on the same pass, so a backend that lost what it was given (a server restarted
+   *  without persisting its users, a restore, a login removed by hand) is provisioned again instead
+   *  of being trusted from memory (novox/hq issue 120). An adapter without it is trusted from memory,
+   *  as before. It must only read: it is asked often, and must never change the backend. */
+  holds?(p: Provision): Promise<boolean>;
+  /** Optional: RETIRE a consumer the mesh stopped asking for (novox/hq ADR 0230) — DISABLE its
+   *  access, reversibly, and mark it in the backend "to delete" with when and why; delete nothing.
+   *  Asked for again, `reenable` (when given) and then `create` bring it back as it was.
+   *
+   *  **An adapter without it is MARK-ONLY**: retiring a consumer then changes nothing in the backend —
+   *  its access stays as it was — and it is only marked, in this harness, "retired, access kept, needs
+   *  deletion", said loudly and announced. Only a person's `cleanup delete` then removes it. */
+  retire?(p: {
+    readonly as: string;
+    readonly derived: Readonly<Record<string, unknown>>;
+    readonly why: string;
+    readonly at: Date;
+  }): Promise<void>;
+  /** Optional, with `retire`: undo what `retire` disabled, before `create` runs for a consumer asked
+   *  for again. An adapter whose `create` already re-enables needs none. */
+  reenable?(p: { readonly as: string; readonly derived: Readonly<Record<string, unknown>> }): Promise<void>;
+  /** Optional: every consumer the backend holds retired — what `cleanup list` shows. */
+  retired?(): Promise<RetiredConsumer[]>;
+  /** Optional: DELETE one retired consumer and its data. Only ever called for a consumer this
+   *  harness or `retired()` knows retired, and only through `provisioner_delete`, a person's act. */
+  delete?(p: { readonly as: string; readonly why: string }): Promise<{ freedBytes?: number } | void>;
+  /** Optional: every consumer the backend holds ACTIVE that the mesh made, read at start, so one the
+   *  mesh stopped asking for while this process was down is still retired (by the same rules). */
+  held?(): Promise<string[]>;
+  /** Optional: the class of one of the adapter's own errors (novox/hq ADR 0224), when its code knows
+   *  better than the error's words; "" or undefined falls back to `classOf` of the text. */
+  classify?(err: unknown): string | undefined;
+}
+
+/** One retired consumer, as `provisioner_retirement` answers it. */
+export interface RetiredConsumer {
+  readonly consumer: string;
+  readonly node?: string;
+  /** When it was retired, RFC 3339, or "" when the backend does not know. */
+  readonly retired_at: string;
+  readonly why: string;
+  /** Its size in bytes, or -1 when not knowable. */
+  readonly size_bytes: number;
+  /** "consumer", or a backend's own word for something else it keeps retired. */
+  readonly kind: string;
+  /** "disabled" when the adapter retired it; "kept" for a mark-only provider, whose consumer keeps its
+   *  access until a person deletes it (novox/hq ADR 0230). */
+  readonly access?: "disabled" | "kept";
+}
+
+/** What `provisioner_retirement` answers: the provider's retirement state (novox/hq ADR 0230). */
+export interface Retirement {
+  readonly resource: string;
+  readonly node: string;
+  readonly held: string[];
+  readonly stable_passes: number;
+  readonly stable_for_seconds: number;
+  /** "disable" when the adapter can retire; "mark-only" when it cannot (novox/hq ADR 0230). */
+  readonly retires_by: "disable" | "mark-only";
+  readonly bound: string;
+  readonly waiting: { consumers: { consumer: string; node?: string }[]; since: string; held: number } | null;
+  readonly rejected: { consumers: { consumer: string; node?: string }[]; by: string; why: string; at: string } | null;
+  readonly retired: RetiredConsumer[];
+}
+
+/** Who acts on a waiting retirement or a deletion, and why. `via` is "mesh-controller" when asked
+ *  through the controller's verbs. */
+export interface Act {
+  readonly why: string;
+  readonly by?: string;
+  readonly via?: string;
+}
+
+/** The running provider loop: a stop function, as before, that also answers and takes the person's
+ *  acts on retirement (novox/hq ADR 0230). */
+export type ProviderLoop = (() => void) & {
+  retirement(): Promise<Retirement>;
+  approve(consumers: readonly string[], act: Act): Promise<{ retired: string[] }>;
+  reject(consumers: readonly string[], act: Act): Promise<{ kept: string[] }>;
+  deleteRetired(consumer: string, act: Act & { readonly confirm?: string }): Promise<{ deleted: string; freed_bytes: number }>;
+};
+
+/** The one event a provider says about retirement (novox/hq ADR 0230). */
+export const RETIREMENT_EVENT = "provisioner.retirement";
+
+/** The events a provider's standing toward one consumer is announced as (novox/hq ADR 0224). The
+ *  controller derives the permission to emit them for every module that receives contributions. */
+export const FAILING_EVENT = "provisioner.failing";
+export const RECOVERED_EVENT = "provisioner.recovered";
+
+/** The classes a failing consumer is announced with: what a person reading `status` needs to know
+ *  before reading the journal (novox/hq ADR 0224). An adapter may say better (`classify`). */
+export const CLASS_CREDENTIALS = "credentials-rejected";
+export const CLASS_UNREACHABLE = "unreachable";
+export const CLASS_SECRET = "secret-unreadable";
+export const CLASS_REFUSED = "refused";
+
+/** How long a consumer fails without one success before it is announced failing (ADR 0224). */
+export const FAILING_AFTER_MS = 5 * 60_000;
+
+export interface ProviderLoopOptions {
+  /** The contributions file the mesh writes for this resource (the module's `receives` path).
+   *  Defaults to $MESH_RECEIVES. */
+  receives?: string;
+  /** Reconcile interval in ms. Defaults to 5000. */
+  everyMs?: number;
+  /** How often, in ms, an adapter with `holds` is asked whether the backend still holds each
+   *  applied consumer. Defaults to 60000: slower than reconciling, because it reads the backend for
+   *  every consumer, and fast enough that a lost login is back within a minute. */
+  verifyEveryMs?: number;
+  /** How long one `holds` may take before it counts as "could not ask". Defaults to 30000: the
+   *  loop is sequential, so a check that hangs would otherwise stop every consumer's provisioning. */
+  holdsTimeoutMs?: number;
+  /** How many consecutive passes that read the file must name the same set of consumers no longer
+   *  asked for before they are retired (novox/hq ADR 0230). Defaults to 5. Additions and changes are
+   *  never delayed. */
+  stablePasses?: number;
+  /** And how long, in ms, that same set must have held since the first pass that saw it. Defaults to
+   *  STABLE_FOR_MS, ten minutes: five passes alone are 25 s, shorter than a controller restart, a store
+   *  reconnecting or a file half written. Both must hold. */
+  stableForMs?: number;
+  /** A stable set larger than this is not retired without a person. Defaults to 3. */
+  retireAtOnce?: number;
+  /** Nor a set larger than this share of the consumers held, where more than one is held. Defaults
+   *  to 0.5. */
+  retireFraction?: number;
+  /** How often a waiting retirement, and a consumer still failing, is said again, in ms. Defaults to
+   *  15 minutes. */
+  sayAgainEveryMs?: number;
+  /** How long, in ms, a consumer fails without one success before it is announced
+   *  `provisioner.failing` (novox/hq ADR 0224). Defaults to FAILING_AFTER_MS, five minutes. */
+  failingAfterMs?: number;
+  /** Publishes one of the provider's events (`provisioner.failing`, `provisioner.recovered`,
+   *  `provisioner.retirement`). Defaults to the SDK's `emit`, so every provider says them on the bus
+   *  without asking (novox/hq issue 272); a failed publish is said in the journal and never fails
+   *  the act it reports. */
+  announce?: (event: string, body: Record<string, unknown>) => Promise<void> | void;
+  /** The machine this provider runs on, said in each event. Defaults to $MESH_NODE. */
+  node?: string;
+  /** The module's name: when given, the four retirement tools are registered for it through
+   *  registerModuleTools, so the controller's `retire` and `cleanup` verbs can reach them. */
+  module?: string;
+}
+
+/** The least time the same unasked set must hold before it is retired (novox/hq ADR 0230). */
+export const STABLE_FOR_MS = 10 * 60_000;
+
+/** The longest a consumer whose `create` keeps failing to satisfy `holds` waits between checks. */
+const MAX_BACKOFF_MS = 60 * 60_000;
+
+/** How many passes a secret may be unreadable before the harness stops calling it a race. At the
+ *  default cadence this is about a minute, which is longer than the host takes to write a file
+ *  and far shorter than the three hours issue 225 went unread. */
+const patiently = 12;
+
+/** Once it is saying so loudly, how often to repeat it. Rare enough to stay readable, often
+ *  enough that somebody arriving at the log finds it. */
+const loudlyEvery = 240;
+
+/** One entry in the mesh's contributions file: a consumer the provider must serve. */
+interface Contribution {
+  readonly as: string;
+  readonly secret: string;
+  readonly node?: string;
+  readonly at?: string;
+  readonly values?: Readonly<Record<string, unknown>>;
+  readonly derived?: Readonly<Record<string, unknown>>;
+}
+
+/**
+ * Run the reconcile loop for one provided resource. Returns a stop function, which also carries the
+ * provider's retirement state and a person's acts on it. Never throws for a single bad contribution —
+ * it logs and keeps converging, because one consumer's failure must not stop the others'
+ * provisioning.
+ *
+ * **A consumer the mesh stops asking for is retired, never withdrawn whole (novox/hq ADR 0230).**
+ * ACTIVE → RETIRED once the same set of consumers has gone unasked in `stablePasses` consecutive
+ * passes that read the file AND for `stableForMs` (ten minutes); RETIRED is access disabled and
+ * marked — or, for an adapter with no `retire`, only marked here with its access kept — nothing
+ * deleted, and asked for again it is re-enabled at once. `remove` is never called to retire. A stable set of more than `retireAtOnce`, or more than
+ * `retireFraction` of those held where more than one is held, is not retired without a person: the
+ * provider waits for approve or reject. RETIRED → DELETED only through `provisioner_delete`.
+ */
+export function runProviderLoop(resource: string, adapter: Adapter, opts: ProviderLoopOptions = {}): ProviderLoop {
+  const receives = opts.receives ?? envOrThrow("MESH_RECEIVES");
+  const everyMs = opts.everyMs ?? 5000;
+  const verifyEveryMs = opts.verifyEveryMs ?? 60_000;
+  const holdsTimeoutMs = opts.holdsTimeoutMs ?? 30_000;
+  const stablePasses = opts.stablePasses ?? 5;
+  const stableForMs = opts.stableForMs ?? STABLE_FOR_MS;
+  const retiresBy: "disable" | "mark-only" = adapter.retire ? "disable" : "mark-only";
+  const retireAtOnce = opts.retireAtOnce ?? 3;
+  const retireFraction = opts.retireFraction ?? 0.5;
+  const sayAgainEveryMs = opts.sayAgainEveryMs ?? 15 * 60_000;
+  const failingAfterMs = opts.failingAfterMs ?? FAILING_AFTER_MS;
+  const announce = opts.announce ?? ((event: string, body: Record<string, unknown>) => emit(event, body));
+  const node = opts.node ?? process.env.MESH_NODE ?? "";
+  const say = (line: string): void => console.error(`[provisioner:${resource}] ${line}`);
+  // Retirement (novox/hq ADR 0230). The unasked set last seen and in how many consecutive readable
+  // passes; a set too large, waiting for a person; a set a person rejected; and every consumer known
+  // retired, by name.
+  let stable = { set: "", count: 0, since: 0 };
+  let pending: { consumers: string[]; since: number; held: number; saidAt: number } | null = null;
+  let rejected: { consumers: string[]; by: string; why: string; at: number } | null = null;
+  const retired = new Map<string, {
+    node?: string; at: number; why: string; derived?: Readonly<Record<string, unknown>>; access: "disabled" | "kept";
+  }>();
+  let seeded = false;
+  let verifiedAt = 0;
+  // Consumers the backend reported lost, and how many times in a row. A `create` that succeeded and
+  // is still not held on the next check will never be: something in the adapter disagrees with
+  // itself. Each repeat doubles the wait before asking again, so such a bug costs one re-apply an
+  // hour rather than one a minute, and it is said loudly rather than quietly repeated.
+  const lost = new Map<string, { times: number; nextAt: number }>();
+  // Each consumer's unbroken run of failures — since when, how often, the last error, and when it was
+  // last announced (0: not yet) — and every consumer that has succeeded since this process started
+  // (novox/hq ADR 0224).
+  const trouble = new Map<string, {
+    node?: string; since: number; attempts: number; cls: string; text: string; saidAt: number;
+  }>();
+  const cleared = new Set<string>();
+
+  // login (`as`) -> what was last applied: the hash that says whether it changed, and the
+  // derivation to hand back on withdrawal, which by then is no longer in the file.
+  const applied = new Map<string, { hash: string; derived: Readonly<Record<string, unknown>>; node?: string }>();
+  // How many passes each consumer's secret has been unreadable for, so the wait can stop sounding
+  // like a wait.
+  const waiting = new Map<string, number>();
+  let stopped = false;
+  // A reason the contributions file could not be used, said once until it changes.
+  let lastWarning = "";
+  const warn = (why: string): void => {
+    if (why === lastWarning) return;
+    if (why) console.error(`[provisioner:${resource}] ${why}; nothing applied or removed until it can be read`);
+    else console.error(`[provisioner:${resource}] contributions file readable again`);
+    lastWarning = why;
+  };
+
+  async function reconcile(): Promise<void> {
+    const given = await readContributions(receives, resource, warn);
+    // **Nothing read is not nobody asking** (novox/hq issue 241). A file that could not be read,
+    // parsed or recognised says nothing about which consumers the mesh wants, and treating it as
+    // "none" withdrew every consumer at once: on 2026-10-04 one failed read made the postgres provider
+    // drop every database it held. Only a file that was read can withdraw anything, so the whole pass
+    // waits for the next one.
+    if (given === null) {
+      // Not a result: the passes that make a retirement stable must each have read the file.
+      stable = { set: "", count: 0, since: 0 };
+      return;
+    }
+    if (!seeded) await seed();
+    const wantByAs = new Map(given.map((g) => [g.as, g]));
+    // On this pass, ask the backend rather than memory whether each applied consumer is still there.
+    let verifying = adapter.holds !== undefined && Date.now() - verifiedAt >= verifyEveryMs;
+    if (verifying) verifiedAt = Date.now();
+
+    // Create or update every consumer whose login, password or values changed.
+    for (const g of given) {
+      let password: string;
+      try {
+        // The file the mesh sealed to this node, unsealed by the host into plaintext. Trailing
+        // newline trimmed: a sealed value is exactly the secret, and a host writing a file may add
+        // one.
+        password = (await readFile(g.secret, "utf8")).replace(/\n$/, "");
+      } catch (err) {
+        // The contribution names a secret the host has not written yet — a normal race on the
+        // first pass. Skipped, retried on the next tick, never fatal.
+        //
+        // **"Yet" stops being true, and then the word is the fault** (novox/hq issue 225). A
+        // permission the host will never grant reads exactly like a file that has not arrived,
+        // and the two want opposite responses: one is waiting, the other is a person. Said the
+        // patient way while it is plausibly a race, and then said plainly, once, and from then
+        // on only occasionally — because a line repeated every five seconds is a line nobody
+        // reads. Three hours of "not readable yet" is how a provider that served nobody looked
+        // healthy.
+        const n = (waiting.get(g.as) ?? 0) + 1;
+        waiting.set(g.as, n);
+        if (n <= patiently) {
+          console.error(`[provisioner:${resource}] ${g.as}: secret not readable yet (${g.secret}): ${err}`);
+        } else if (n === patiently + 1 || n % loudlyEvery === 0) {
+          console.error(
+            `[provisioner:${resource}] ${g.as}: CANNOT READ the secret after ${n} attempts ` +
+              `(${g.secret}): ${err}. This is not a race any more — nothing has been provisioned ` +
+              `for this consumer and nothing will be until somebody looks. Check who owns the ` +
+              `file and who this process runs as (novox/hq issue 225)`,
+          );
+        }
+        await failed(g.as, g.node, CLASS_SECRET, `secret not readable (${g.secret}): ${err}`);
+        continue;
+      }
+      waiting.delete(g.as);
+      const h = hash(g.as, password, g.values ?? {}, g.derived ?? {});
+      const p: Provision = {
+        as: g.as, password, values: g.values ?? {}, derived: g.derived ?? {}, at: g.at, consumer: g.node,
+      };
+      // Set when the backend reported this consumer lost: how many successful re-applies in a row
+      // it has now needed, counting this one.
+      let reapplying: number | undefined;
+      if (applied.get(g.as)?.hash === h) {
+        if (!verifying) continue;
+        const brake = lost.get(g.as);
+        if (brake && Date.now() < brake.nextAt) continue;
+        try {
+          if (await withTimeout(adapter.holds!(p), holdsTimeoutMs)) {
+            lost.delete(g.as);
+            await succeeded(g.as);
+            continue;
+          }
+        } catch (err) {
+          // Unable to ask is not evidence of loss. Kept as applied, asked again next time — but a
+          // check that keeps failing is a consumer failed all the same (ADR 0224, issue 179).
+          const text = scrub(err, password);
+          console.error(`[provisioner:${resource}] ${g.as}: could not check the backend, will ask again: ${text}`);
+          await failed(g.as, g.node, classed(err, text), text);
+          // A backend that did not answer in time will not answer for the next consumer either, and
+          // each would cost a timeout, so the rest of this pass is not asked. Any other failure may
+          // be this consumer's alone, and the others are still asked.
+          if (err instanceof HoldsTimeout) verifying = false;
+          continue;
+        }
+        reapplying = (brake?.times ?? 0) + 1;
+        if (reapplying === 1) {
+          // Said, because it means the backend lost something while nothing was looking.
+          console.error(`[provisioner:${resource}] ${g.as}: the backend no longer holds it; applying again`);
+        } else {
+          console.error(
+            `[provisioner:${resource}] ${g.as}: still not held after being applied again (${reapplying} times in a row) — ` +
+              `create does not produce what holds checks; applying again`,
+          );
+        }
+      }
+      try {
+        const was = retired.get(g.as);
+        if (was && was.access === "disabled" && adapter.reenable) {
+          await adapter.reenable({ as: g.as, derived: was.derived ?? p.derived });
+        }
+        await adapter.create(p);
+        await succeeded(g.as);
+        applied.set(g.as, { hash: h, derived: p.derived, node: g.node });
+        if (retired.has(g.as)) {
+          // Asked for again: the ordinary create re-enabled it and cleared its mark (ADR 0230).
+          retired.delete(g.as);
+          say(`${g.as}: asked for again; re-enabled as it was`);
+          await tell("reenabled", { consumers: [{ consumer: g.as, node: g.node }] });
+        }
+        if (reapplying === undefined) {
+          // Applied for a new login, password or values: whatever was counted before does not carry over.
+          lost.delete(g.as);
+        } else {
+          // Only a create that succeeded counts toward the brake: if the backend still does not hold
+          // it at the next check, create and holds disagree, and each repeat waits twice as long.
+          const waitMs = Math.min(MAX_BACKOFF_MS, verifyEveryMs * 2 ** (reapplying - 1));
+          lost.set(g.as, { times: reapplying, nextAt: Date.now() + waitMs });
+          if (reapplying > 1) {
+            console.error(`[provisioner:${resource}] ${g.as}: next check in ${Math.round(waitMs / 1000)}s`);
+          }
+        }
+      } catch (err) {
+        const text = scrub(err, password);
+        console.error(`[provisioner:${resource}] ${g.as}: create failed, will retry: ${text}`);
+        await failed(g.as, g.node, classed(err, text), text);
+        // A failed create is not a disagreement: asked again at the next check, the count unchanged.
+        if (reapplying !== undefined) lost.set(g.as, { times: reapplying - 1, nextAt: 0 });
+      }
+    }
+
+    // Retire what the mesh no longer asks for — once the same set has gone unasked for
+    // `stablePasses` passes, and within the bound, or with a person's word (novox/hq ADR 0230).
+    // Only consumers this harness applied, or the backend says the mesh made, are touched.
+    const unasked = [...applied.keys()].filter((as) => !wantByAs.has(as)).sort();
+    // A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed
+    // by anyone: said, so the standing the controller keeps for it is cleared rather than left naming a
+    // consumer that is gone. One still held is said recovered when it is retired (ADR 0224, 0230).
+    for (const as of [...trouble.keys()]) {
+      if (!wantByAs.has(as) && !applied.has(as)) await recovered(as, "no longer asked for");
+    }
+    const key = unasked.join(",");
+    if (pending && pending.consumers.join(",") !== key) {
+      say(`the retirement waiting for a person (${pending.consumers.join(", ")}) is no longer what the mesh says; settled, nothing retired`);
+      await tell("settled", { consumers: named(pending.consumers) });
+      pending = null;
+    }
+    if (rejected && rejected.consumers.join(",") !== key) {
+      say(`the rejected retirement (${rejected.consumers.join(", ")}) is no longer what the mesh says`);
+      await tell("settled", { consumers: named(rejected.consumers) });
+      rejected = null;
+    }
+    if (unasked.length === 0) {
+      stable = { set: "", count: 0, since: 0 };
+      return;
+    }
+    stable = stable.set === key
+      ? { set: key, count: stable.count + 1, since: stable.since }
+      : { set: key, count: 1, since: Date.now() };
+    if (stable.count < stablePasses || Date.now() - stable.since < stableForMs) return;
+    if (rejected) return; // a person said keep them, and the mesh still says the same
+    const held = applied.size;
+    if (overTheBound(unasked.length, held)) {
+      const now = Date.now();
+      if (pending === null) {
+        pending = { consumers: unasked, since: now, held, saidAt: now };
+        say(
+          `RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for ${unasked.length} of the ${held} consumer(s) ` +
+            `this provider holds (${unasked.join(", ")}), ${bound(held)}. Nothing is retired until ` +
+            `\`retire approve\` or \`retire reject\` (novox/hq ADR 0230)`,
+        );
+        await tell("waiting", { consumers: named(unasked), held, since: iso(now) });
+      } else if (now - pending.saidAt >= sayAgainEveryMs) {
+        pending.saidAt = now;
+        await tell("waiting", { consumers: named(unasked), held: pending.held, since: iso(pending.since) });
+      }
+      return;
+    }
+    const why = `the mesh stopped asking for it: the same answer in ${stable.count} passes over ` +
+      `${Math.round((Date.now() - stable.since) / 1000)}s, from ${iso(stable.since)}`;
+    const done = await retireAll(unasked, why);
+    if (done.length > 0) await tell("retired", { consumers: done.map((as) => retiredBody(as)), held, why });
+  }
+
+  function retiresByAccess(): "disabled" | "kept" {
+    return adapter.retire ? "disabled" : "kept";
+  }
+
+  function overTheBound(n: number, held: number): boolean {
+    return n > retireAtOnce || (held > 1 && n > retireFraction * held);
+  }
+
+  function bound(held: number): string {
+    return `more than ${retireAtOnce}, or more than ${Math.round(retireFraction * 100)}% of the ${held} held`;
+  }
+
+  // Retire each, as it is said: disabled and marked by the adapter — or, where the adapter cannot
+  // disable, only marked here, its access kept. Nothing is deleted, and `remove` is never called: it
+  // is the delete of a provider without one. Answers those that went; one that failed stays active and
+  // is asked again once its set is stable again.
+  async function retireAll(names: readonly string[], why: string): Promise<string[]> {
+    const done: string[] = [];
+    for (const as of names) {
+      const was = applied.get(as);
+      if (was === undefined) continue;
+      try {
+        const at = new Date();
+        if (adapter.retire) {
+          say(`${as}: no longer in ${receives}; RETIRED — access disabled, nothing deleted: ${why}`);
+          await adapter.retire({ as, derived: was.derived, why, at });
+        } else {
+          say(
+            `${as}: no longer in ${receives}; RETIRED, MARK ONLY — this provider cannot disable a consumer ` +
+              `(its adapter has no retire), so ${as} KEEPS ITS ACCESS and its data until a person deletes it ` +
+              `with \`cleanup delete\`. Nothing was changed in the backend (novox/hq ADR 0230): ${why}`,
+          );
+        }
+        applied.delete(as);
+        lost.delete(as);
+        await recovered(as, "retired");
+        retired.set(as, { node: was.node, at: at.getTime(), why, derived: was.derived, access: adapter.retire ? "disabled" : "kept" });
+        done.push(as);
+      } catch (err) {
+        say(`${as}: retiring failed, will retry: ${err}`);
+      }
+    }
+    stable = { set: "", count: 0, since: 0 };
+    return done;
+  }
+
+  function named(names: readonly string[]): { consumer: string; node?: string }[] {
+    return names.map((as) => ({ consumer: as, node: applied.get(as)?.node ?? retired.get(as)?.node }));
+  }
+
+  function retiredBody(as: string): Record<string, unknown> {
+    const r = retired.get(as);
+    return { consumer: as, node: r?.node, retired_at: r ? iso(r.at) : "", why: r?.why ?? "", access: r?.access ?? retiresByAccess() };
+  }
+
+  // Said on the bus, never failing the act it reports (the journal still has it).
+  async function tell(change: string, body: Record<string, unknown>): Promise<void> {
+    await said(RETIREMENT_EVENT, {
+      provider: resource, "provider-node": node, change, bound: bound(applied.size), at: iso(Date.now()), ...body,
+    }, change);
+  }
+
+  async function said(event: string, body: Record<string, unknown>, what = ""): Promise<void> {
+    try {
+      await announce(event, body);
+    } catch (err) {
+      say(`could not announce ${event}${what ? ` ${what}` : ""}: ${err}`);
+    }
+  }
+
+  // An error's class: the adapter's word when it has one, else read from the text (ADR 0224).
+  function classed(err: unknown, text: string): string {
+    try {
+      const own = adapter.classify?.(err);
+      if (own) return own;
+    } catch {
+      // An adapter that cannot class its own error is read by the words.
+    }
+    // A check that did not answer in time is a backend out of reach, whatever its words.
+    if (err instanceof HoldsTimeout) return CLASS_UNREACHABLE;
+    return classOf(text);
+  }
+
+  // One more failure in a consumer's unbroken run, announced once the run has lasted failingAfterMs,
+  // then again every sayAgainEveryMs while it lasts (novox/hq ADR 0224).
+  async function failed(as: string, consumerNode: string | undefined, cls: string, text: string): Promise<void> {
+    const now = Date.now();
+    let s = trouble.get(as);
+    if (!s) {
+      s = { since: now, attempts: 0, cls, text, saidAt: 0 };
+      trouble.set(as, s);
+    }
+    s.node = consumerNode;
+    s.cls = cls;
+    s.text = text;
+    s.attempts++;
+    if (now - s.since < failingAfterMs) return;
+    if (s.saidAt !== 0 && now - s.saidAt < sayAgainEveryMs) return;
+    if (s.saidAt === 0) {
+      say(
+        `${as}: FAILING for ${Math.round((now - s.since) / 1000)}s (${s.attempts} attempts, ${cls}): ${text}. ` +
+          `Announced as ${FAILING_EVENT}; \`status\` names it until it succeeds (novox/hq ADR 0224)`,
+      );
+    }
+    s.saidAt = now;
+    await said(FAILING_EVENT, {
+      provider: resource, "provider-node": node, consumer: as, node: consumerNode ?? "",
+      class: cls, error: clip(text), since: iso(s.since), attempts: s.attempts,
+    });
+  }
+
+  // A success ends a consumer's run of failures; one that was announced is announced recovered. And
+  // the first success for a consumer since this process started is announced too, failing or not: a
+  // provider that announced a failure and was restarted has forgotten it, and without this the
+  // controller would name the consumer failing for ever after it recovered unheard.
+  async function succeeded(as: string): Promise<void> {
+    if (!trouble.has(as) && !cleared.has(as)) {
+      cleared.add(as);
+      await said(RECOVERED_EVENT, { provider: resource, "provider-node": node, consumer: as, why: "first-success" });
+      return;
+    }
+    cleared.add(as);
+    await recovered(as, "");
+  }
+
+  async function recovered(as: string, why: string): Promise<void> {
+    const s = trouble.get(as);
+    if (!s) return;
+    trouble.delete(as);
+    if (s.saidAt === 0) return; // never announced, so there is nothing to take back
+    if (!why) say(`${as}: recovered after ${Math.round((Date.now() - s.since) / 1000)}s and ${s.attempts} failed attempt(s)`);
+    await said(RECOVERED_EVENT, {
+      provider: resource, "provider-node": node, consumer: as, node: s.node ?? "",
+      since: iso(s.since), attempts: s.attempts, ...(why ? { why } : {}),
+    });
+  }
+
+  // What the backend holds that the mesh made: active ones become held, so one the mesh stopped asking
+  // for while this process was down is still retired, by the same rules; retired ones are known.
+  async function seed(): Promise<void> {
+    try {
+      for (const as of (await adapter.held?.()) ?? []) {
+        if (!applied.has(as)) applied.set(as, { hash: "", derived: {} });
+      }
+      for (const r of (await adapter.retired?.()) ?? []) {
+        if (r.kind === "consumer" && !retired.has(r.consumer)) {
+          retired.set(r.consumer, {
+            node: r.node, at: r.retired_at ? Date.parse(r.retired_at) : 0, why: r.why, access: r.access ?? retiresByAccess(),
+          });
+        }
+      }
+      seeded = true;
+    } catch (err) {
+      say(`cannot read what the backend holds yet, will ask again: ${err}`);
+    }
+  }
+
+  // One act at a time: a person's approval and a pass never interleave.
+  let queue: Promise<unknown> = Promise.resolve();
+  function serially<T>(f: () => Promise<T>): Promise<T> {
+    const run = queue.then(f, f);
+    queue = run.catch(() => undefined);
+    return run;
+  }
+
+  const sameSet = (a: readonly string[], b: readonly string[]): boolean =>
+    [...a].sort().join(",") === [...b].sort().join(",");
+
+  async function retirement(): Promise<Retirement> {
+    const fromBackend = (await adapter.retired?.()) ?? [];
+    const list: RetiredConsumer[] = [...fromBackend];
+    for (const [as, r] of retired) {
+      if (!list.some((x) => x.consumer === as && x.kind === "consumer")) {
+        list.push({
+          consumer: as, node: r.node, retired_at: r.at ? iso(r.at) : "", why: r.why, size_bytes: -1, kind: "consumer",
+          access: r.access,
+        });
+      }
+    }
+    return {
+      resource, node, held: [...applied.keys()].sort(), stable_passes: stablePasses,
+      stable_for_seconds: Math.round(stableForMs / 1000), retires_by: retiresBy, bound: bound(applied.size),
+      waiting: pending && { consumers: named(pending.consumers), since: iso(pending.since), held: pending.held },
+      rejected: rejected && { consumers: named(rejected.consumers), by: rejected.by, why: rejected.why, at: iso(rejected.at) },
+      retired: list,
+    };
+  }
+
+  function needWhy(act: Act, verb: string): void {
+    if (!act.why || !act.why.trim()) throw new Error(`${verb}: why is required`);
+  }
+
+  async function approve(consumers: readonly string[], act: Act): Promise<{ retired: string[] }> {
+    needWhy(act, "approve");
+    return serially(async () => {
+      const set = pending?.consumers ?? rejected?.consumers;
+      if (!set || !sameSet(set, consumers)) {
+        throw new Error(
+          `approve: nothing waits with exactly that set (${[...consumers].sort().join(", ")}); ` +
+            `waiting: ${pending?.consumers.join(", ") || "none"}; rejected: ${rejected?.consumers.join(", ") || "none"}. Look again`,
+        );
+      }
+      const held = applied.size;
+      say(`retirement of ${set.join(", ")} approved by ${act.by || "someone"}: ${act.why}`);
+      await tell("approved", { consumers: named(set), held, why: act.why, by: act.by ?? "", via: act.via ?? "" });
+      pending = null;
+      rejected = null;
+      const why = `approved by ${act.by || "someone"}: ${act.why}`;
+      const done = await retireAll(set, why);
+      if (done.length > 0) await tell("retired", { consumers: done.map((as) => retiredBody(as)), held, why });
+      return { retired: done };
+    });
+  }
+
+  async function reject(consumers: readonly string[], act: Act): Promise<{ kept: string[] }> {
+    needWhy(act, "reject");
+    return serially(async () => {
+      if (!pending || !sameSet(pending.consumers, consumers)) {
+        throw new Error(
+          `reject: nothing waits with exactly that set (${[...consumers].sort().join(", ")}); ` +
+            `waiting: ${pending?.consumers.join(", ") || "none"}. Look again`,
+        );
+      }
+      const kept = pending.consumers;
+      rejected = { consumers: kept, by: act.by ?? "", why: act.why, at: Date.now() };
+      pending = null;
+      say(`retirement of ${kept.join(", ")} rejected by ${act.by || "someone"}: ${act.why}; kept active while the mesh says the same`);
+      await tell("rejected", { consumers: named(kept), why: act.why, by: act.by ?? "", via: act.via ?? "" });
+      return { kept };
+    });
+  }
+
+  async function deleteRetired(consumer: string, act: Act & { readonly confirm?: string }): Promise<{ deleted: string; freed_bytes: number }> {
+    needWhy(act, "delete");
+    if (act.confirm !== consumer) throw new Error("delete: confirm must repeat the consumer's name");
+    return serially(async () => {
+      if (applied.has(consumer)) throw new Error(`delete: ${consumer} is active, not retired — only a retired consumer is deleted`);
+      const known = retired.has(consumer) || ((await adapter.retired?.()) ?? []).some((r) => r.consumer === consumer);
+      if (!known) throw new Error(`delete: ${consumer} is not retired here`);
+      const was = retired.get(consumer);
+      say(`${consumer}: DELETING, asked by ${act.by || "someone"}: ${act.why}`);
+      let freed = -1;
+      if (adapter.delete) {
+        const r = await adapter.delete({ as: consumer, why: act.why });
+        if (r && typeof r.freedBytes === "number") freed = r.freedBytes;
+      } else {
+        // The one path to `remove`: a person's approved deletion (novox/hq ADR 0230).
+        await adapter.remove({ as: consumer, derived: was?.derived ?? {} });
+      }
+      retired.delete(consumer);
+      await tell("deleted", {
+        consumers: [{ consumer, node: was?.node, size_bytes: freed }], why: act.why, by: act.by ?? "", via: act.via ?? "",
+      });
+      return { deleted: consumer, freed_bytes: freed };
+    });
+  }
+
+  const tick = async (): Promise<void> => {
+    if (stopped) return;
+    try {
+      await serially(reconcile);
+    } catch (err) {
+      console.error(`[provisioner:${resource}] reconcile error: ${err}`);
+    }
+    if (!stopped) setTimeout(() => void tick(), everyMs);
+  };
+  void tick();
+
+  const stop = () => {
+    stopped = true;
+  };
+  const handle: ProviderLoop = Object.assign(stop, { retirement, approve, reject, deleteRetired });
+  if (opts.module) registerModuleTools(opts.module, () => providerLoopTools(handle));
+  return handle;
+}
+
+const strings = (v: unknown): string[] => (Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []);
+const text = (v: unknown): string => (typeof v === "string" ? v : "");
+
+/** The four tools every provider serves for retirement (novox/hq ADR 0230), the controller's `retire`
+ *  and `cleanup` verbs asking them per machine. */
+export function providerLoopTools(p: ProviderLoop): ToolDefinition[] {
+  const act = { type: "string" };
+  return [
+    {
+      name: "provisioner_retirement",
+      description: "This provider's retirement: the consumers it holds, a retirement waiting for a person, one a person rejected, and every consumer retired (access disabled, data kept) with when, why and size.",
+      input: {},
+      run: async () => p.retirement(),
+    },
+    {
+      name: "provisioner_retire_approve",
+      description: "Retire exactly the set waiting for a person (or rejected earlier). Repeat the set in consumers; why is required.",
+      input: { consumers: { type: "array", items: { type: "string" } }, why: act, by: act, via: act },
+      run: async (a) => p.approve(strings(a.consumers), { why: text(a.why), by: text(a.by), via: text(a.via) }),
+    },
+    {
+      name: "provisioner_retire_reject",
+      description: "Keep the set waiting for a person active, though the mesh no longer asks for it. Repeat the set in consumers; why is required.",
+      input: { consumers: { type: "array", items: { type: "string" } }, why: act, by: act, via: act },
+      run: async (a) => p.reject(strings(a.consumers), { why: text(a.why), by: text(a.by), via: text(a.via) }),
+    },
+    {
+      name: "provisioner_delete",
+      description: "Delete one RETIRED consumer and its data from this provider's backend — never an active one. Repeat its name in confirm; why is required.",
+      input: { consumer: act, confirm: act, why: act, by: act, via: act },
+      run: async (a) =>
+        p.deleteRetired(text(a.consumer), { why: text(a.why), by: text(a.by), via: text(a.via), confirm: text(a.confirm) }),
+    },
+  ];
+}
+
+function iso(ms: number): string {
+  return new Date(ms).toISOString();
+}
+
+/**
+ * Read the mesh's contributions file for one resource: the consumers it asks for, or `null` when the
+ * file says nothing usable — absent, unreadable, not JSON, for another requirement, or without a
+ * `given` list. `null` is not "no consumer": the mesh always writes the file, and writes `given: []`
+ * when nobody asks, so only that makes every consumer unasked — retired by the rules of ADR 0230, never at once without a person.
+ */
+async function readContributions(
+  path: string,
+  resource: string,
+  warn: (why: string) => void,
+): Promise<Contribution[] | null> {
+  let raw: string;
+  try {
+    raw = await readFile(path, "utf8");
+  } catch (err) {
+    warn(`contributions file unreadable (${path}): ${(err as Error).message}`);
+    return null;
+  }
+  let doc: { requirement?: string; given?: unknown };
+  try {
+    doc = JSON.parse(raw) as { requirement?: string; given?: unknown };
+  } catch (err) {
+    warn(`contributions file is not JSON (${path}): ${err}`);
+    return null;
+  }
+  if (doc.requirement && doc.requirement !== resource) {
+    warn(`${path} is for ${doc.requirement}, not ${resource}`);
+    return null;
+  }
+  if (!Array.isArray(doc.given)) {
+    warn(`${path} has no given list`);
+    return null;
+  }
+  warn("");
+  // A contribution with no `as` is not a credential grant (a module offering something on its own
+  // machine) — the provisioner has nothing to create for it.
+  return (doc.given as Contribution[]).filter((g) => g.as && g.secret);
+}
+
+/** An error's class read from its text — the words the backends the mesh runs use (novox/hq ADR
+ *  0224): the same words, in the same order, as the Go copy of this loop reads. */
+export function classOf(text: string): string {
+  const t = text.toLowerCase();
+  const credentials = ["invalid_grant", "invalid user credentials", "password authentication failed",
+    "authentication failed", "unauthorized", " 401"];
+  if (credentials.some((w) => t.includes(w))) return CLASS_CREDENTIALS;
+  const unreachable = ["connection refused", "no such host", "i/o timeout", "deadline exceeded",
+    "connection reset", "network is unreachable", "no route to host", "eof",
+    // Node's own words for the same faults.
+    "econnrefused", "enotfound", "etimedout", "econnreset", "ehostunreach", "enetunreach", "eai_again"];
+  if (unreachable.some((w) => t.includes(w))) return CLASS_UNREACHABLE;
+  return CLASS_REFUSED;
+}
+
+/** An announced error kept to what belongs in a status line. */
+function clip(text: string): string {
+  const most = 300;
+  return text.length <= most ? text : `${text.slice(0, most)}…`;
+}
+
+/** A check that did not answer in time: the backend, not the consumer, is the likely cause. */
+class HoldsTimeout extends Error {}
+
+/** Reject with a HoldsTimeout if `p` has not settled within `ms`. */
+function withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
+  let timer: NodeJS.Timeout | undefined;
+  const timeout = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => reject(new HoldsTimeout(`no answer within ${ms}ms`)), ms);
+  });
+  return Promise.race([p, timeout]).finally(() => clearTimeout(timer));
+}
+
+/** An error as text with the consumer's password removed, raw and URL-encoded: a failed command's
+ *  message can carry its arguments, and this log is not a place a password may appear. */
+function scrub(err: unknown, password: string): string {
+  let text = String(err);
+  if (!password) return text;
+  for (const form of new Set([password, encodeURIComponent(password)])) text = text.split(form).join("***");
+  return text;
+}
+
+function hash(
+  as: string,
+  password: string,
+  values: Readonly<Record<string, unknown>>,
+  derived: Readonly<Record<string, unknown>>,
+): string {
+  // The derivation is in the hash because it can change without the login or the password
+  // changing: a provider that renames what it derives has given this consumer a different
+  // resource, and an adapter trusted from memory would never be told (novox/hq ADR 0201).
+  return JSON.stringify([as, password, values, derived]);
+}
+
+function envOrThrow(name: string): string {
+  const v = process.env[name];
+  if (!v) throw new Error(`${name} is not set — the provisioner cannot run without it`);
+  return v;
+}
+
+/** @deprecated The provider loop's name before design 57: {@link runProviderLoop}. */
+export const runProvisioner = runProviderLoop;
+/** @deprecated {@link providerLoopTools}. */
+export const provisionerTools = providerLoopTools;
+/** @deprecated {@link ProviderLoop}. */
+export type Provisioner = ProviderLoop;
+/** @deprecated {@link ProviderLoopOptions}. */
+export type ProvisionerOptions = ProviderLoopOptions;