// The provider loop: the reconcile harness every provider on the controller's grant files shares.
// Called `provisioner` until design 57 (novox/hq issue 678): that word is mesh-provisioner's, which
// places every value a module needs (ADR 0349), so the loop is named for what it is. The old path
// `@novox/mesh-sdk/provisioner` and the names `runProvisioner`, `provisionerTools`, `Provisioner` and
// `ProvisionerOptions` stay as aliases of these until the providers that import them move to the Go
// helper that answers consumers over the bus (mesh-sdk go/provider, design 57 B2 and B3) and this loop
// retires with the grant files. The events (`provisioner.failing`, `.recovered`, `.retirement`) and the
// tool names (`provisioner_retirement`, …) are what the controller and the operator read on the bus,
// and keep their names.
//
// The reconcile harness every provider shares. The loop below is identical for postgres, redis,
// minio and umami: read the contributions the mesh delivered, bring each consumer's resource into
// being through the provider's adapter under the login and password the mesh minted, and retire
// what the mesh no longer asks for. A module writes ONLY the adapter — the per-service half — which
// is why this lives in the sdk and the adapter lives in the module (novox/hq ADR 0039).
//
// **A provider creates the credential the mesh minted, and seals nothing (novox/hq ADR 0048).** The
// control plane mints one password per consumer and seals it to this node; the host unseals it into
// the file the contribution names. The provider does not generate a password, does not seal one, and
// does not hand one back — the consumer already receives its copy through the mesh's own channel.
// This is why the old $MESH_SEAL_KEY, the symmetric seal, and the *.grant.json / *.credential files
// are gone: they described a second, contradictory credential path the mesh does not have.
//
// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer
// whose create, check or secret has failed without one success in between for `failingAfterMs` is
// announced as `provisioner.failing` — naming the consumer, its machine and the class of error — and
// again every `sayAgainEveryMs` while it lasts; the first success after that is
// `provisioner.recovered`, as is a consumer the mesh stopped asking for, and the first success for
// each consumer since this process started. The controller keeps the newest per provider and consumer
// and `status` names it. Until novox/hq issue 272 only the Go copy of this loop (postgres, keycloak)
// said it; every provider on this one failed its consumers in its journal alone.

import { readFile } from "node:fs/promises";

import type { ToolDefinition } from "../contracts/index.js";
import { emit } from "../events/index.js";
import { registerModuleTools } from "../tools/index.js";

/** One consumer's resource to bring into being — everything the mesh derived and delivered. */
export interface Provision {
  /** The login the mesh derived and gave the consumer to present. The provider creates exactly
   *  this name — a name the consumer cannot learn is a name it cannot authenticate with. */
  readonly as: string;
  /** The password the mesh minted for this consumer, read from the file the mesh sealed to this
   *  node and the host unsealed. The provider sets it; it never invents one. */
  readonly password: string;
  /** What the consumer contributed, per the interface's spec keys (e.g. `{ name: "umami" }`). */
  readonly values: Readonly<Record<string, unknown>>;
  /** What this provider's own definition said it derives for this consumer, already derived by
   *  the mesh (novox/hq ADR 0201).
   *
   *  **The rule is declared, not computed here.** A provider that names the resource it gives each
   *  consumer — a bucket, a database, a vhost — writes that name in its `serves` block with
   *  `${consumer:as}` or `${consumer:as:dns}` in it. The mesh fills it for this consumer and hands
   *  the same filled value to the consumer's own configuration, so the two ends cannot disagree.
   *  An adapter reads it from here; an adapter that recomputed it would be the second rule this
   *  removed. Empty for a provider that derives nothing. */
  readonly derived: Readonly<Record<string, unknown>>;
  /** Where the consumer is, for a provider that must reach back to it. Usually unused. */
  readonly at?: string;
  /** The consumer node, for logging and lifecycle events. */
  readonly consumer?: string;
}

/** What a provider implements — the only per-service code. It is handed the login and password the
 *  mesh made and brings the resource into being under them, or withdraws it. It returns nothing:
 *  the credential is the mesh's, already delivered to the consumer. */
export interface Adapter {
  create(p: Provision): Promise<void>;
  /** Remove what `create` made for this consumer. **Called only to DELETE a retired consumer, on a
   *  person's approved `cleanup delete`, and only when the adapter has no `delete`** (novox/hq ADR
   *  0230): many providers' remove destroys something — a ledger file, a public DNS record, a user —
   *  and nothing is destroyed without a person. Retiring never calls it. `derived` is what the mesh
   *  last derived for it, remembered by the harness: the mesh has already dropped the consumer from
   *  the contributions file by then, so it cannot be read again, and a provider that recomputed it
   *  would be keeping the second copy of a rule this removed (novox/hq ADR 0201). */
  remove(p: { readonly as: string; readonly derived: Readonly<Record<string, unknown>> }): Promise<void>;
  /** Optional: whether the backend still holds this consumer's credential exactly as `p` says.
   *  Asked of every consumer already applied, every `verifyEveryMs`. `false` makes the harness apply
   *  it again on the same pass, so a backend that lost what it was given (a server restarted
   *  without persisting its users, a restore, a login removed by hand) is provisioned again instead
   *  of being trusted from memory (novox/hq issue 120). An adapter without it is trusted from memory,
   *  as before. It must only read: it is asked often, and must never change the backend. */
  holds?(p: Provision): Promise<boolean>;
  /** Optional: RETIRE a consumer the mesh stopped asking for (novox/hq ADR 0230) — DISABLE its
   *  access, reversibly, and mark it in the backend "to delete" with when and why; delete nothing.
   *  Asked for again, `reenable` (when given) and then `create` bring it back as it was.
   *
   *  **An adapter without it is MARK-ONLY**: retiring a consumer then changes nothing in the backend —
   *  its access stays as it was — and it is only marked, in this harness, "retired, access kept, needs
   *  deletion", said loudly and announced. Only a person's `cleanup delete` then removes it. */
  retire?(p: {
    readonly as: string;
    readonly derived: Readonly<Record<string, unknown>>;
    readonly why: string;
    readonly at: Date;
  }): Promise<void>;
  /** Optional, with `retire`: undo what `retire` disabled, before `create` runs for a consumer asked
   *  for again. An adapter whose `create` already re-enables needs none. */
  reenable?(p: { readonly as: string; readonly derived: Readonly<Record<string, unknown>> }): Promise<void>;
  /** Optional: every consumer the backend holds retired — what `cleanup list` shows. */
  retired?(): Promise<RetiredConsumer[]>;
  /** Optional: DELETE one retired consumer and its data. Only ever called for a consumer this
   *  harness or `retired()` knows retired, and only through `provisioner_delete`, a person's act. */
  delete?(p: { readonly as: string; readonly why: string }): Promise<{ freedBytes?: number } | void>;
  /** Optional: every consumer the backend holds ACTIVE that the mesh made, read at start, so one the
   *  mesh stopped asking for while this process was down is still retired (by the same rules). */
  held?(): Promise<string[]>;
  /** Optional: the class of one of the adapter's own errors (novox/hq ADR 0224), when its code knows
   *  better than the error's words; "" or undefined falls back to `classOf` of the text. */
  classify?(err: unknown): string | undefined;
}

/** One retired consumer, as `provisioner_retirement` answers it. */
export interface RetiredConsumer {
  readonly consumer: string;
  readonly node?: string;
  /** When it was retired, RFC 3339, or "" when the backend does not know. */
  readonly retired_at: string;
  readonly why: string;
  /** Its size in bytes, or -1 when not knowable. */
  readonly size_bytes: number;
  /** "consumer", or a backend's own word for something else it keeps retired. */
  readonly kind: string;
  /** "disabled" when the adapter retired it; "kept" for a mark-only provider, whose consumer keeps its
   *  access until a person deletes it (novox/hq ADR 0230). */
  readonly access?: "disabled" | "kept";
}

/** What `provisioner_retirement` answers: the provider's retirement state (novox/hq ADR 0230). */
export interface Retirement {
  readonly resource: string;
  readonly node: string;
  readonly held: string[];
  readonly stable_passes: number;
  readonly stable_for_seconds: number;
  /** "disable" when the adapter can retire; "mark-only" when it cannot (novox/hq ADR 0230). */
  readonly retires_by: "disable" | "mark-only";
  readonly bound: string;
  readonly waiting: { consumers: { consumer: string; node?: string }[]; since: string; held: number } | null;
  readonly rejected: { consumers: { consumer: string; node?: string }[]; by: string; why: string; at: string } | null;
  readonly retired: RetiredConsumer[];
}

/** Who acts on a waiting retirement or a deletion, and why. `via` is "mesh-controller" when asked
 *  through the controller's verbs. */
export interface Act {
  readonly why: string;
  readonly by?: string;
  readonly via?: string;
}

/** The running provider loop: a stop function, as before, that also answers and takes the person's
 *  acts on retirement (novox/hq ADR 0230). */
export type ProviderLoop = (() => void) & {
  retirement(): Promise<Retirement>;
  approve(consumers: readonly string[], act: Act): Promise<{ retired: string[] }>;
  reject(consumers: readonly string[], act: Act): Promise<{ kept: string[] }>;
  deleteRetired(consumer: string, act: Act & { readonly confirm?: string }): Promise<{ deleted: string; freed_bytes: number }>;
};

/** The one event a provider says about retirement (novox/hq ADR 0230). */
export const RETIREMENT_EVENT = "provisioner.retirement";

/** The events a provider's standing toward one consumer is announced as (novox/hq ADR 0224). The
 *  controller derives the permission to emit them for every module that receives contributions. */
export const FAILING_EVENT = "provisioner.failing";
export const RECOVERED_EVENT = "provisioner.recovered";

/** The classes a failing consumer is announced with: what a person reading `status` needs to know
 *  before reading the journal (novox/hq ADR 0224). An adapter may say better (`classify`). */
export const CLASS_CREDENTIALS = "credentials-rejected";
export const CLASS_UNREACHABLE = "unreachable";
export const CLASS_SECRET = "secret-unreadable";
export const CLASS_REFUSED = "refused";

/** How long a consumer fails without one success before it is announced failing (ADR 0224). */
export const FAILING_AFTER_MS = 5 * 60_000;

export interface ProviderLoopOptions {
  /** The contributions file the mesh writes for this resource (the module's `receives` path).
   *  Defaults to $MESH_RECEIVES. */
  receives?: string;
  /** Reconcile interval in ms. Defaults to 5000. */
  everyMs?: number;
  /** How often, in ms, an adapter with `holds` is asked whether the backend still holds each
   *  applied consumer. Defaults to 60000: slower than reconciling, because it reads the backend for
   *  every consumer, and fast enough that a lost login is back within a minute. */
  verifyEveryMs?: number;
  /** How long one `holds` may take before it counts as "could not ask". Defaults to 30000: the
   *  loop is sequential, so a check that hangs would otherwise stop every consumer's provisioning. */
  holdsTimeoutMs?: number;
  /** How many consecutive passes that read the file must name the same set of consumers no longer
   *  asked for before they are retired (novox/hq ADR 0230). Defaults to 5. Additions and changes are
   *  never delayed. */
  stablePasses?: number;
  /** And how long, in ms, that same set must have held since the first pass that saw it. Defaults to
   *  STABLE_FOR_MS, ten minutes: five passes alone are 25 s, shorter than a controller restart, a store
   *  reconnecting or a file half written. Both must hold. */
  stableForMs?: number;
  /** A stable set larger than this is not retired without a person. Defaults to 3. */
  retireAtOnce?: number;
  /** Nor a set larger than this share of the consumers held, where more than one is held. Defaults
   *  to 0.5. */
  retireFraction?: number;
  /** How often a waiting retirement, and a consumer still failing, is said again, in ms. Defaults to
   *  15 minutes. */
  sayAgainEveryMs?: number;
  /** How long, in ms, a consumer fails without one success before it is announced
   *  `provisioner.failing` (novox/hq ADR 0224). Defaults to FAILING_AFTER_MS, five minutes. */
  failingAfterMs?: number;
  /** Publishes one of the provider's events (`provisioner.failing`, `provisioner.recovered`,
   *  `provisioner.retirement`). Defaults to the SDK's `emit`, so every provider says them on the bus
   *  without asking (novox/hq issue 272); a failed publish is said in the journal and never fails
   *  the act it reports. */
  announce?: (event: string, body: Record<string, unknown>) => Promise<void> | void;
  /** The machine this provider runs on, said in each event. Defaults to $MESH_NODE. */
  node?: string;
  /** The module's name: when given, the four retirement tools are registered for it through
   *  registerModuleTools, so the controller's `retire` and `cleanup` verbs can reach them. */
  module?: string;
}

/** The least time the same unasked set must hold before it is retired (novox/hq ADR 0230). */
export const STABLE_FOR_MS = 10 * 60_000;

/** The longest a consumer whose `create` keeps failing to satisfy `holds` waits between checks. */
const MAX_BACKOFF_MS = 60 * 60_000;

/** How many passes a secret may be unreadable before the harness stops calling it a race. At the
 *  default cadence this is about a minute, which is longer than the host takes to write a file
 *  and far shorter than the three hours issue 225 went unread. */
const patiently = 12;

/** Once it is saying so loudly, how often to repeat it. Rare enough to stay readable, often
 *  enough that somebody arriving at the log finds it. */
const loudlyEvery = 240;

/** One entry in the mesh's contributions file: a consumer the provider must serve. */
interface Contribution {
  readonly as: string;
  readonly secret: string;
  readonly node?: string;
  readonly at?: string;
  readonly values?: Readonly<Record<string, unknown>>;
  readonly derived?: Readonly<Record<string, unknown>>;
}

/**
 * Run the reconcile loop for one provided resource. Returns a stop function, which also carries the
 * provider's retirement state and a person's acts on it. Never throws for a single bad contribution —
 * it logs and keeps converging, because one consumer's failure must not stop the others'
 * provisioning.
 *
 * **A consumer the mesh stops asking for is retired, never withdrawn whole (novox/hq ADR 0230).**
 * ACTIVE → RETIRED once the same set of consumers has gone unasked in `stablePasses` consecutive
 * passes that read the file AND for `stableForMs` (ten minutes); RETIRED is access disabled and
 * marked — or, for an adapter with no `retire`, only marked here with its access kept — nothing
 * deleted, and asked for again it is re-enabled at once. `remove` is never called to retire. A stable set of more than `retireAtOnce`, or more than
 * `retireFraction` of those held where more than one is held, is not retired without a person: the
 * provider waits for approve or reject. RETIRED → DELETED only through `provisioner_delete`.
 */
export function runProviderLoop(resource: string, adapter: Adapter, opts: ProviderLoopOptions = {}): ProviderLoop {
  const receives = opts.receives ?? envOrThrow("MESH_RECEIVES");
  const everyMs = opts.everyMs ?? 5000;
  const verifyEveryMs = opts.verifyEveryMs ?? 60_000;
  const holdsTimeoutMs = opts.holdsTimeoutMs ?? 30_000;
  const stablePasses = opts.stablePasses ?? 5;
  const stableForMs = opts.stableForMs ?? STABLE_FOR_MS;
  const retiresBy: "disable" | "mark-only" = adapter.retire ? "disable" : "mark-only";
  const retireAtOnce = opts.retireAtOnce ?? 3;
  const retireFraction = opts.retireFraction ?? 0.5;
  const sayAgainEveryMs = opts.sayAgainEveryMs ?? 15 * 60_000;
  const failingAfterMs = opts.failingAfterMs ?? FAILING_AFTER_MS;
  const announce = opts.announce ?? ((event: string, body: Record<string, unknown>) => emit(event, body));
  const node = opts.node ?? process.env.MESH_NODE ?? "";
  const say = (line: string): void => console.error(`[provisioner:${resource}] ${line}`);
  // Retirement (novox/hq ADR 0230). The unasked set last seen and in how many consecutive readable
  // passes; a set too large, waiting for a person; a set a person rejected; and every consumer known
  // retired, by name.
  let stable = { set: "", count: 0, since: 0 };
  let pending: { consumers: string[]; since: number; held: number; saidAt: number } | null = null;
  let rejected: { consumers: string[]; by: string; why: string; at: number } | null = null;
  const retired = new Map<string, {
    node?: string; at: number; why: string; derived?: Readonly<Record<string, unknown>>; access: "disabled" | "kept";
  }>();
  let seeded = false;
  let verifiedAt = 0;
  // Consumers the backend reported lost, and how many times in a row. A `create` that succeeded and
  // is still not held on the next check will never be: something in the adapter disagrees with
  // itself. Each repeat doubles the wait before asking again, so such a bug costs one re-apply an
  // hour rather than one a minute, and it is said loudly rather than quietly repeated.
  const lost = new Map<string, { times: number; nextAt: number }>();
  // Each consumer's unbroken run of failures — since when, how often, the last error, and when it was
  // last announced (0: not yet) — and every consumer that has succeeded since this process started
  // (novox/hq ADR 0224).
  const trouble = new Map<string, {
    node?: string; since: number; attempts: number; cls: string; text: string; saidAt: number;
  }>();
  const cleared = new Set<string>();

  // login (`as`) -> what was last applied: the hash that says whether it changed, and the
  // derivation to hand back on withdrawal, which by then is no longer in the file.
  const applied = new Map<string, { hash: string; derived: Readonly<Record<string, unknown>>; node?: string }>();
  // How many passes each consumer's secret has been unreadable for, so the wait can stop sounding
  // like a wait.
  const waiting = new Map<string, number>();
  let stopped = false;
  // A reason the contributions file could not be used, said once until it changes.
  let lastWarning = "";
  const warn = (why: string): void => {
    if (why === lastWarning) return;
    if (why) console.error(`[provisioner:${resource}] ${why}; nothing applied or removed until it can be read`);
    else console.error(`[provisioner:${resource}] contributions file readable again`);
    lastWarning = why;
  };

  async function reconcile(): Promise<void> {
    const given = await readContributions(receives, resource, warn);
    // **Nothing read is not nobody asking** (novox/hq issue 241). A file that could not be read,
    // parsed or recognised says nothing about which consumers the mesh wants, and treating it as
    // "none" withdrew every consumer at once: on 2026-10-04 one failed read made the postgres provider
    // drop every database it held. Only a file that was read can withdraw anything, so the whole pass
    // waits for the next one.
    if (given === null) {
      // Not a result: the passes that make a retirement stable must each have read the file.
      stable = { set: "", count: 0, since: 0 };
      return;
    }
    if (!seeded) await seed();
    const wantByAs = new Map(given.map((g) => [g.as, g]));
    // On this pass, ask the backend rather than memory whether each applied consumer is still there.
    let verifying = adapter.holds !== undefined && Date.now() - verifiedAt >= verifyEveryMs;
    if (verifying) verifiedAt = Date.now();

    // Create or update every consumer whose login, password or values changed.
    for (const g of given) {
      let password: string;
      try {
        // The file the mesh sealed to this node, unsealed by the host into plaintext. Trailing
        // newline trimmed: a sealed value is exactly the secret, and a host writing a file may add
        // one.
        password = (await readFile(g.secret, "utf8")).replace(/\n$/, "");
      } catch (err) {
        // The contribution names a secret the host has not written yet — a normal race on the
        // first pass. Skipped, retried on the next tick, never fatal.
        //
        // **"Yet" stops being true, and then the word is the fault** (novox/hq issue 225). A
        // permission the host will never grant reads exactly like a file that has not arrived,
        // and the two want opposite responses: one is waiting, the other is a person. Said the
        // patient way while it is plausibly a race, and then said plainly, once, and from then
        // on only occasionally — because a line repeated every five seconds is a line nobody
        // reads. Three hours of "not readable yet" is how a provider that served nobody looked
        // healthy.
        const n = (waiting.get(g.as) ?? 0) + 1;
        waiting.set(g.as, n);
        if (n <= patiently) {
          console.error(`[provisioner:${resource}] ${g.as}: secret not readable yet (${g.secret}): ${err}`);
        } else if (n === patiently + 1 || n % loudlyEvery === 0) {
          console.error(
            `[provisioner:${resource}] ${g.as}: CANNOT READ the secret after ${n} attempts ` +
              `(${g.secret}): ${err}. This is not a race any more — nothing has been provisioned ` +
              `for this consumer and nothing will be until somebody looks. Check who owns the ` +
              `file and who this process runs as (novox/hq issue 225)`,
          );
        }
        await failed(g.as, g.node, CLASS_SECRET, `secret not readable (${g.secret}): ${err}`);
        continue;
      }
      waiting.delete(g.as);
      const h = hash(g.as, password, g.values ?? {}, g.derived ?? {});
      const p: Provision = {
        as: g.as, password, values: g.values ?? {}, derived: g.derived ?? {}, at: g.at, consumer: g.node,
      };
      // Set when the backend reported this consumer lost: how many successful re-applies in a row
      // it has now needed, counting this one.
      let reapplying: number | undefined;
      if (applied.get(g.as)?.hash === h) {
        if (!verifying) continue;
        const brake = lost.get(g.as);
        if (brake && Date.now() < brake.nextAt) continue;
        try {
          if (await withTimeout(adapter.holds!(p), holdsTimeoutMs)) {
            lost.delete(g.as);
            await succeeded(g.as);
            continue;
          }
        } catch (err) {
          // Unable to ask is not evidence of loss. Kept as applied, asked again next time — but a
          // check that keeps failing is a consumer failed all the same (ADR 0224, issue 179).
          const text = scrub(err, password);
          console.error(`[provisioner:${resource}] ${g.as}: could not check the backend, will ask again: ${text}`);
          await failed(g.as, g.node, classed(err, text), text);
          // A backend that did not answer in time will not answer for the next consumer either, and
          // each would cost a timeout, so the rest of this pass is not asked. Any other failure may
          // be this consumer's alone, and the others are still asked.
          if (err instanceof HoldsTimeout) verifying = false;
          continue;
        }
        reapplying = (brake?.times ?? 0) + 1;
        if (reapplying === 1) {
          // Said, because it means the backend lost something while nothing was looking.
          console.error(`[provisioner:${resource}] ${g.as}: the backend no longer holds it; applying again`);
        } else {
          console.error(
            `[provisioner:${resource}] ${g.as}: still not held after being applied again (${reapplying} times in a row) — ` +
              `create does not produce what holds checks; applying again`,
          );
        }
      }
      try {
        const was = retired.get(g.as);
        if (was && was.access === "disabled" && adapter.reenable) {
          await adapter.reenable({ as: g.as, derived: was.derived ?? p.derived });
        }
        await adapter.create(p);
        await succeeded(g.as);
        applied.set(g.as, { hash: h, derived: p.derived, node: g.node });
        if (retired.has(g.as)) {
          // Asked for again: the ordinary create re-enabled it and cleared its mark (ADR 0230).
          retired.delete(g.as);
          say(`${g.as}: asked for again; re-enabled as it was`);
          await tell("reenabled", { consumers: [{ consumer: g.as, node: g.node }] });
        }
        if (reapplying === undefined) {
          // Applied for a new login, password or values: whatever was counted before does not carry over.
          lost.delete(g.as);
        } else {
          // Only a create that succeeded counts toward the brake: if the backend still does not hold
          // it at the next check, create and holds disagree, and each repeat waits twice as long.
          const waitMs = Math.min(MAX_BACKOFF_MS, verifyEveryMs * 2 ** (reapplying - 1));
          lost.set(g.as, { times: reapplying, nextAt: Date.now() + waitMs });
          if (reapplying > 1) {
            console.error(`[provisioner:${resource}] ${g.as}: next check in ${Math.round(waitMs / 1000)}s`);
          }
        }
      } catch (err) {
        const text = scrub(err, password);
        console.error(`[provisioner:${resource}] ${g.as}: create failed, will retry: ${text}`);
        await failed(g.as, g.node, classed(err, text), text);
        // A failed create is not a disagreement: asked again at the next check, the count unchanged.
        if (reapplying !== undefined) lost.set(g.as, { times: reapplying - 1, nextAt: 0 });
      }
    }

    // Retire what the mesh no longer asks for — once the same set has gone unasked for
    // `stablePasses` passes, and within the bound, or with a person's word (novox/hq ADR 0230).
    // Only consumers this harness applied, or the backend says the mesh made, are touched.
    const unasked = [...applied.keys()].filter((as) => !wantByAs.has(as)).sort();
    // A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed
    // by anyone: said, so the standing the controller keeps for it is cleared rather than left naming a
    // consumer that is gone. One still held is said recovered when it is retired (ADR 0224, 0230).
    for (const as of [...trouble.keys()]) {
      if (!wantByAs.has(as) && !applied.has(as)) await recovered(as, "no longer asked for");
    }
    const key = unasked.join(",");
    if (pending && pending.consumers.join(",") !== key) {
      say(`the retirement waiting for a person (${pending.consumers.join(", ")}) is no longer what the mesh says; settled, nothing retired`);
      await tell("settled", { consumers: named(pending.consumers) });
      pending = null;
    }
    if (rejected && rejected.consumers.join(",") !== key) {
      say(`the rejected retirement (${rejected.consumers.join(", ")}) is no longer what the mesh says`);
      await tell("settled", { consumers: named(rejected.consumers) });
      rejected = null;
    }
    if (unasked.length === 0) {
      stable = { set: "", count: 0, since: 0 };
      return;
    }
    stable = stable.set === key
      ? { set: key, count: stable.count + 1, since: stable.since }
      : { set: key, count: 1, since: Date.now() };
    if (stable.count < stablePasses || Date.now() - stable.since < stableForMs) return;
    if (rejected) return; // a person said keep them, and the mesh still says the same
    const held = applied.size;
    if (overTheBound(unasked.length, held)) {
      const now = Date.now();
      if (pending === null) {
        pending = { consumers: unasked, since: now, held, saidAt: now };
        say(
          `RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for ${unasked.length} of the ${held} consumer(s) ` +
            `this provider holds (${unasked.join(", ")}), ${bound(held)}. Nothing is retired until ` +
            `\`retire approve\` or \`retire reject\` (novox/hq ADR 0230)`,
        );
        await tell("waiting", { consumers: named(unasked), held, since: iso(now) });
      } else if (now - pending.saidAt >= sayAgainEveryMs) {
        pending.saidAt = now;
        await tell("waiting", { consumers: named(unasked), held: pending.held, since: iso(pending.since) });
      }
      return;
    }
    const why = `the mesh stopped asking for it: the same answer in ${stable.count} passes over ` +
      `${Math.round((Date.now() - stable.since) / 1000)}s, from ${iso(stable.since)}`;
    const done = await retireAll(unasked, why);
    if (done.length > 0) await tell("retired", { consumers: done.map((as) => retiredBody(as)), held, why });
  }

  function retiresByAccess(): "disabled" | "kept" {
    return adapter.retire ? "disabled" : "kept";
  }

  function overTheBound(n: number, held: number): boolean {
    return n > retireAtOnce || (held > 1 && n > retireFraction * held);
  }

  function bound(held: number): string {
    return `more than ${retireAtOnce}, or more than ${Math.round(retireFraction * 100)}% of the ${held} held`;
  }

  // Retire each, as it is said: disabled and marked by the adapter — or, where the adapter cannot
  // disable, only marked here, its access kept. Nothing is deleted, and `remove` is never called: it
  // is the delete of a provider without one. Answers those that went; one that failed stays active and
  // is asked again once its set is stable again.
  async function retireAll(names: readonly string[], why: string): Promise<string[]> {
    const done: string[] = [];
    for (const as of names) {
      const was = applied.get(as);
      if (was === undefined) continue;
      try {
        const at = new Date();
        if (adapter.retire) {
          say(`${as}: no longer in ${receives}; RETIRED — access disabled, nothing deleted: ${why}`);
          await adapter.retire({ as, derived: was.derived, why, at });
        } else {
          say(
            `${as}: no longer in ${receives}; RETIRED, MARK ONLY — this provider cannot disable a consumer ` +
              `(its adapter has no retire), so ${as} KEEPS ITS ACCESS and its data until a person deletes it ` +
              `with \`cleanup delete\`. Nothing was changed in the backend (novox/hq ADR 0230): ${why}`,
          );
        }
        applied.delete(as);
        lost.delete(as);
        await recovered(as, "retired");
        retired.set(as, { node: was.node, at: at.getTime(), why, derived: was.derived, access: adapter.retire ? "disabled" : "kept" });
        done.push(as);
      } catch (err) {
        say(`${as}: retiring failed, will retry: ${err}`);
      }
    }
    stable = { set: "", count: 0, since: 0 };
    return done;
  }

  function named(names: readonly string[]): { consumer: string; node?: string }[] {
    return names.map((as) => ({ consumer: as, node: applied.get(as)?.node ?? retired.get(as)?.node }));
  }

  function retiredBody(as: string): Record<string, unknown> {
    const r = retired.get(as);
    return { consumer: as, node: r?.node, retired_at: r ? iso(r.at) : "", why: r?.why ?? "", access: r?.access ?? retiresByAccess() };
  }

  // Said on the bus, never failing the act it reports (the journal still has it).
  async function tell(change: string, body: Record<string, unknown>): Promise<void> {
    await said(RETIREMENT_EVENT, {
      provider: resource, "provider-node": node, change, bound: bound(applied.size), at: iso(Date.now()), ...body,
    }, change);
  }

  async function said(event: string, body: Record<string, unknown>, what = ""): Promise<void> {
    try {
      await announce(event, body);
    } catch (err) {
      say(`could not announce ${event}${what ? ` ${what}` : ""}: ${err}`);
    }
  }

  // An error's class: the adapter's word when it has one, else read from the text (ADR 0224).
  function classed(err: unknown, text: string): string {
    try {
      const own = adapter.classify?.(err);
      if (own) return own;
    } catch {
      // An adapter that cannot class its own error is read by the words.
    }
    // A check that did not answer in time is a backend out of reach, whatever its words.
    if (err instanceof HoldsTimeout) return CLASS_UNREACHABLE;
    return classOf(text);
  }

  // One more failure in a consumer's unbroken run, announced once the run has lasted failingAfterMs,
  // then again every sayAgainEveryMs while it lasts (novox/hq ADR 0224).
  async function failed(as: string, consumerNode: string | undefined, cls: string, text: string): Promise<void> {
    const now = Date.now();
    let s = trouble.get(as);
    if (!s) {
      s = { since: now, attempts: 0, cls, text, saidAt: 0 };
      trouble.set(as, s);
    }
    s.node = consumerNode;
    s.cls = cls;
    s.text = text;
    s.attempts++;
    if (now - s.since < failingAfterMs) return;
    if (s.saidAt !== 0 && now - s.saidAt < sayAgainEveryMs) return;
    if (s.saidAt === 0) {
      say(
        `${as}: FAILING for ${Math.round((now - s.since) / 1000)}s (${s.attempts} attempts, ${cls}): ${text}. ` +
          `Announced as ${FAILING_EVENT}; \`status\` names it until it succeeds (novox/hq ADR 0224)`,
      );
    }
    s.saidAt = now;
    await said(FAILING_EVENT, {
      provider: resource, "provider-node": node, consumer: as, node: consumerNode ?? "",
      class: cls, error: clip(text), since: iso(s.since), attempts: s.attempts,
    });
  }

  // A success ends a consumer's run of failures; one that was announced is announced recovered. And
  // the first success for a consumer since this process started is announced too, failing or not: a
  // provider that announced a failure and was restarted has forgotten it, and without this the
  // controller would name the consumer failing for ever after it recovered unheard.
  async function succeeded(as: string): Promise<void> {
    if (!trouble.has(as) && !cleared.has(as)) {
      cleared.add(as);
      await said(RECOVERED_EVENT, { provider: resource, "provider-node": node, consumer: as, why: "first-success" });
      return;
    }
    cleared.add(as);
    await recovered(as, "");
  }

  async function recovered(as: string, why: string): Promise<void> {
    const s = trouble.get(as);
    if (!s) return;
    trouble.delete(as);
    if (s.saidAt === 0) return; // never announced, so there is nothing to take back
    if (!why) say(`${as}: recovered after ${Math.round((Date.now() - s.since) / 1000)}s and ${s.attempts} failed attempt(s)`);
    await said(RECOVERED_EVENT, {
      provider: resource, "provider-node": node, consumer: as, node: s.node ?? "",
      since: iso(s.since), attempts: s.attempts, ...(why ? { why } : {}),
    });
  }

  // What the backend holds that the mesh made: active ones become held, so one the mesh stopped asking
  // for while this process was down is still retired, by the same rules; retired ones are known.
  async function seed(): Promise<void> {
    try {
      for (const as of (await adapter.held?.()) ?? []) {
        if (!applied.has(as)) applied.set(as, { hash: "", derived: {} });
      }
      for (const r of (await adapter.retired?.()) ?? []) {
        if (r.kind === "consumer" && !retired.has(r.consumer)) {
          retired.set(r.consumer, {
            node: r.node, at: r.retired_at ? Date.parse(r.retired_at) : 0, why: r.why, access: r.access ?? retiresByAccess(),
          });
        }
      }
      seeded = true;
    } catch (err) {
      say(`cannot read what the backend holds yet, will ask again: ${err}`);
    }
  }

  // One act at a time: a person's approval and a pass never interleave.
  let queue: Promise<unknown> = Promise.resolve();
  function serially<T>(f: () => Promise<T>): Promise<T> {
    const run = queue.then(f, f);
    queue = run.catch(() => undefined);
    return run;
  }

  const sameSet = (a: readonly string[], b: readonly string[]): boolean =>
    [...a].sort().join(",") === [...b].sort().join(",");

  async function retirement(): Promise<Retirement> {
    const fromBackend = (await adapter.retired?.()) ?? [];
    const list: RetiredConsumer[] = [...fromBackend];
    for (const [as, r] of retired) {
      if (!list.some((x) => x.consumer === as && x.kind === "consumer")) {
        list.push({
          consumer: as, node: r.node, retired_at: r.at ? iso(r.at) : "", why: r.why, size_bytes: -1, kind: "consumer",
          access: r.access,
        });
      }
    }
    return {
      resource, node, held: [...applied.keys()].sort(), stable_passes: stablePasses,
      stable_for_seconds: Math.round(stableForMs / 1000), retires_by: retiresBy, bound: bound(applied.size),
      waiting: pending && { consumers: named(pending.consumers), since: iso(pending.since), held: pending.held },
      rejected: rejected && { consumers: named(rejected.consumers), by: rejected.by, why: rejected.why, at: iso(rejected.at) },
      retired: list,
    };
  }

  function needWhy(act: Act, verb: string): void {
    if (!act.why || !act.why.trim()) throw new Error(`${verb}: why is required`);
  }

  async function approve(consumers: readonly string[], act: Act): Promise<{ retired: string[] }> {
    needWhy(act, "approve");
    return serially(async () => {
      const set = pending?.consumers ?? rejected?.consumers;
      if (!set || !sameSet(set, consumers)) {
        throw new Error(
          `approve: nothing waits with exactly that set (${[...consumers].sort().join(", ")}); ` +
            `waiting: ${pending?.consumers.join(", ") || "none"}; rejected: ${rejected?.consumers.join(", ") || "none"}. Look again`,
        );
      }
      const held = applied.size;
      say(`retirement of ${set.join(", ")} approved by ${act.by || "someone"}: ${act.why}`);
      await tell("approved", { consumers: named(set), held, why: act.why, by: act.by ?? "", via: act.via ?? "" });
      pending = null;
      rejected = null;
      const why = `approved by ${act.by || "someone"}: ${act.why}`;
      const done = await retireAll(set, why);
      if (done.length > 0) await tell("retired", { consumers: done.map((as) => retiredBody(as)), held, why });
      return { retired: done };
    });
  }

  async function reject(consumers: readonly string[], act: Act): Promise<{ kept: string[] }> {
    needWhy(act, "reject");
    return serially(async () => {
      if (!pending || !sameSet(pending.consumers, consumers)) {
        throw new Error(
          `reject: nothing waits with exactly that set (${[...consumers].sort().join(", ")}); ` +
            `waiting: ${pending?.consumers.join(", ") || "none"}. Look again`,
        );
      }
      const kept = pending.consumers;
      rejected = { consumers: kept, by: act.by ?? "", why: act.why, at: Date.now() };
      pending = null;
      say(`retirement of ${kept.join(", ")} rejected by ${act.by || "someone"}: ${act.why}; kept active while the mesh says the same`);
      await tell("rejected", { consumers: named(kept), why: act.why, by: act.by ?? "", via: act.via ?? "" });
      return { kept };
    });
  }

  async function deleteRetired(consumer: string, act: Act & { readonly confirm?: string }): Promise<{ deleted: string; freed_bytes: number }> {
    needWhy(act, "delete");
    if (act.confirm !== consumer) throw new Error("delete: confirm must repeat the consumer's name");
    return serially(async () => {
      if (applied.has(consumer)) throw new Error(`delete: ${consumer} is active, not retired — only a retired consumer is deleted`);
      const known = retired.has(consumer) || ((await adapter.retired?.()) ?? []).some((r) => r.consumer === consumer);
      if (!known) throw new Error(`delete: ${consumer} is not retired here`);
      const was = retired.get(consumer);
      say(`${consumer}: DELETING, asked by ${act.by || "someone"}: ${act.why}`);
      let freed = -1;
      if (adapter.delete) {
        const r = await adapter.delete({ as: consumer, why: act.why });
        if (r && typeof r.freedBytes === "number") freed = r.freedBytes;
      } else {
        // The one path to `remove`: a person's approved deletion (novox/hq ADR 0230).
        await adapter.remove({ as: consumer, derived: was?.derived ?? {} });
      }
      retired.delete(consumer);
      await tell("deleted", {
        consumers: [{ consumer, node: was?.node, size_bytes: freed }], why: act.why, by: act.by ?? "", via: act.via ?? "",
      });
      return { deleted: consumer, freed_bytes: freed };
    });
  }

  const tick = async (): Promise<void> => {
    if (stopped) return;
    try {
      await serially(reconcile);
    } catch (err) {
      console.error(`[provisioner:${resource}] reconcile error: ${err}`);
    }
    if (!stopped) setTimeout(() => void tick(), everyMs);
  };
  void tick();

  const stop = () => {
    stopped = true;
  };
  const handle: ProviderLoop = Object.assign(stop, { retirement, approve, reject, deleteRetired });
  if (opts.module) registerModuleTools(opts.module, () => providerLoopTools(handle));
  return handle;
}

const strings = (v: unknown): string[] => (Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []);
const text = (v: unknown): string => (typeof v === "string" ? v : "");

/** The four tools every provider serves for retirement (novox/hq ADR 0230), the controller's `retire`
 *  and `cleanup` verbs asking them per machine. */
export function providerLoopTools(p: ProviderLoop): ToolDefinition[] {
  const act = { type: "string" };
  return [
    {
      name: "provisioner_retirement",
      description: "This provider's retirement: the consumers it holds, a retirement waiting for a person, one a person rejected, and every consumer retired (access disabled, data kept) with when, why and size.",
      input: {},
      run: async () => p.retirement(),
    },
    {
      name: "provisioner_retire_approve",
      description: "Retire exactly the set waiting for a person (or rejected earlier). Repeat the set in consumers; why is required.",
      input: { consumers: { type: "array", items: { type: "string" } }, why: act, by: act, via: act },
      run: async (a) => p.approve(strings(a.consumers), { why: text(a.why), by: text(a.by), via: text(a.via) }),
    },
    {
      name: "provisioner_retire_reject",
      description: "Keep the set waiting for a person active, though the mesh no longer asks for it. Repeat the set in consumers; why is required.",
      input: { consumers: { type: "array", items: { type: "string" } }, why: act, by: act, via: act },
      run: async (a) => p.reject(strings(a.consumers), { why: text(a.why), by: text(a.by), via: text(a.via) }),
    },
    {
      name: "provisioner_delete",
      description: "Delete one RETIRED consumer and its data from this provider's backend — never an active one. Repeat its name in confirm; why is required.",
      input: { consumer: act, confirm: act, why: act, by: act, via: act },
      run: async (a) =>
        p.deleteRetired(text(a.consumer), { why: text(a.why), by: text(a.by), via: text(a.via), confirm: text(a.confirm) }),
    },
  ];
}

function iso(ms: number): string {
  return new Date(ms).toISOString();
}

/**
 * Read the mesh's contributions file for one resource: the consumers it asks for, or `null` when the
 * file says nothing usable — absent, unreadable, not JSON, for another requirement, or without a
 * `given` list. `null` is not "no consumer": the mesh always writes the file, and writes `given: []`
 * when nobody asks, so only that makes every consumer unasked — retired by the rules of ADR 0230, never at once without a person.
 */
async function readContributions(
  path: string,
  resource: string,
  warn: (why: string) => void,
): Promise<Contribution[] | null> {
  let raw: string;
  try {
    raw = await readFile(path, "utf8");
  } catch (err) {
    warn(`contributions file unreadable (${path}): ${(err as Error).message}`);
    return null;
  }
  let doc: { requirement?: string; given?: unknown };
  try {
    doc = JSON.parse(raw) as { requirement?: string; given?: unknown };
  } catch (err) {
    warn(`contributions file is not JSON (${path}): ${err}`);
    return null;
  }
  if (doc.requirement && doc.requirement !== resource) {
    warn(`${path} is for ${doc.requirement}, not ${resource}`);
    return null;
  }
  if (!Array.isArray(doc.given)) {
    warn(`${path} has no given list`);
    return null;
  }
  warn("");
  // A contribution with no `as` is not a credential grant (a module offering something on its own
  // machine) — the provisioner has nothing to create for it.
  return (doc.given as Contribution[]).filter((g) => g.as && g.secret);
}

/** An error's class read from its text — the words the backends the mesh runs use (novox/hq ADR
 *  0224): the same words, in the same order, as the Go copy of this loop reads. */
export function classOf(text: string): string {
  const t = text.toLowerCase();
  const credentials = ["invalid_grant", "invalid user credentials", "password authentication failed",
    "authentication failed", "unauthorized", " 401"];
  if (credentials.some((w) => t.includes(w))) return CLASS_CREDENTIALS;
  const unreachable = ["connection refused", "no such host", "i/o timeout", "deadline exceeded",
    "connection reset", "network is unreachable", "no route to host", "eof",
    // Node's own words for the same faults.
    "econnrefused", "enotfound", "etimedout", "econnreset", "ehostunreach", "enetunreach", "eai_again"];
  if (unreachable.some((w) => t.includes(w))) return CLASS_UNREACHABLE;
  return CLASS_REFUSED;
}

/** An announced error kept to what belongs in a status line. */
function clip(text: string): string {
  const most = 300;
  return text.length <= most ? text : `${text.slice(0, most)}…`;
}

/** A check that did not answer in time: the backend, not the consumer, is the likely cause. */
class HoldsTimeout extends Error {}

/** Reject with a HoldsTimeout if `p` has not settled within `ms`. */
function withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
  let timer: NodeJS.Timeout | undefined;
  const timeout = new Promise<never>((_, reject) => {
    timer = setTimeout(() => reject(new HoldsTimeout(`no answer within ${ms}ms`)), ms);
  });
  return Promise.race([p, timeout]).finally(() => clearTimeout(timer));
}

/** An error as text with the consumer's password removed, raw and URL-encoded: a failed command's
 *  message can carry its arguments, and this log is not a place a password may appear. */
function scrub(err: unknown, password: string): string {
  let text = String(err);
  if (!password) return text;
  for (const form of new Set([password, encodeURIComponent(password)])) text = text.split(form).join("***");
  return text;
}

function hash(
  as: string,
  password: string,
  values: Readonly<Record<string, unknown>>,
  derived: Readonly<Record<string, unknown>>,
): string {
  // The derivation is in the hash because it can change without the login or the password
  // changing: a provider that renames what it derives has given this consumer a different
  // resource, and an adapter trusted from memory would never be told (novox/hq ADR 0201).
  return JSON.stringify([as, password, values, derived]);
}

function envOrThrow(name: string): string {
  const v = process.env[name];
  if (!v) throw new Error(`${name} is not set — the provisioner cannot run without it`);
  return v;
}

/** @deprecated The provider loop's name before design 57: {@link runProviderLoop}. */
export const runProvisioner = runProviderLoop;
/** @deprecated {@link providerLoopTools}. */
export const provisionerTools = providerLoopTools;
/** @deprecated {@link ProviderLoop}. */
export type Provisioner = ProviderLoop;
/** @deprecated {@link ProviderLoopOptions}. */
export type ProvisionerOptions = ProviderLoopOptions;