mesh-tools
1bc5719b8b
mesh/merge-gate head 1bc5719b8b85 merged into trunk a2d2af00dc5d beside mesh-host#116@bff1ac14 — pass, with a note: builds mesh-sdk (+40 dependent(s)) → ac…
mesh/repo-check head 1bc5719b8b85 merged into trunk a2d2af00dc5d beside mesh-host#116@bff1ac14 — pass: its merge-check.sh passed, each part in its toolchai…
mesh/delivery delivered
mesh/delivery-group group release/57-config-and-provisioning delivered: every member is delivered
922 lines
48 KiB
TypeScript
922 lines
48 KiB
TypeScript
// The provider loop: the reconcile harness every provider on the controller's grant files shares.
|
|
// Called `provisioner` until design 57 (novox/hq issue 678): that word is mesh-provisioner's, which
|
|
// places every value a module needs (ADR 0349), so the loop is named for what it is. The old path
|
|
// `@novox/mesh-sdk/provisioner` and the names `runProvisioner`, `provisionerTools`, `Provisioner` and
|
|
// `ProvisionerOptions` stay as aliases of these until the providers that import them move to the Go
|
|
// helper that answers consumers over the bus (mesh-sdk go/provider, design 57 B2 and B3) and this loop
|
|
// retires with the grant files. The events (`provisioner.failing`, `.recovered`, `.retirement`) and the
|
|
// tool names (`provisioner_retirement`, …) are what the controller and the operator read on the bus,
|
|
// and keep their names.
|
|
//
|
|
// The reconcile harness every provider shares. The loop below is identical for postgres, redis,
|
|
// minio and umami: read the contributions the mesh delivered, bring each consumer's resource into
|
|
// being through the provider's adapter under the login and password the mesh minted, and retire
|
|
// what the mesh no longer asks for. A module writes ONLY the adapter — the per-service half — which
|
|
// is why this lives in the sdk and the adapter lives in the module (novox/hq ADR 0039).
|
|
//
|
|
// **A provider creates the credential the mesh minted, and seals nothing (novox/hq ADR 0048).** The
|
|
// control plane mints one password per consumer and seals it to this node; the host unseals it into
|
|
// the file the contribution names. The provider does not generate a password, does not seal one, and
|
|
// does not hand one back — the consumer already receives its copy through the mesh's own channel.
|
|
// This is why the old $MESH_SEAL_KEY, the symmetric seal, and the *.grant.json / *.credential files
|
|
// are gone: they described a second, contradictory credential path the mesh does not have.
|
|
//
|
|
// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer
|
|
// whose create, check or secret has failed without one success in between for `failingAfterMs` is
|
|
// announced as `provisioner.failing` — naming the consumer, its machine and the class of error — and
|
|
// again every `sayAgainEveryMs` while it lasts; the first success after that is
|
|
// `provisioner.recovered`, as is a consumer the mesh stopped asking for, and the first success for
|
|
// each consumer since this process started. The controller keeps the newest per provider and consumer
|
|
// and `status` names it. Until novox/hq issue 272 only the Go copy of this loop (postgres, keycloak)
|
|
// said it; every provider on this one failed its consumers in its journal alone.
|
|
|
|
import { readFile } from "node:fs/promises";
|
|
|
|
import type { ToolDefinition } from "../contracts/index.js";
|
|
import { emit } from "../events/index.js";
|
|
import { registerModuleTools } from "../tools/index.js";
|
|
|
|
/** One consumer's resource to bring into being — everything the mesh derived and delivered. */
|
|
export interface Provision {
|
|
/** The login the mesh derived and gave the consumer to present. The provider creates exactly
|
|
* this name — a name the consumer cannot learn is a name it cannot authenticate with. */
|
|
readonly as: string;
|
|
/** The password the mesh minted for this consumer, read from the file the mesh sealed to this
|
|
* node and the host unsealed. The provider sets it; it never invents one. */
|
|
readonly password: string;
|
|
/** What the consumer contributed, per the interface's spec keys (e.g. `{ name: "umami" }`). */
|
|
readonly values: Readonly<Record<string, unknown>>;
|
|
/** What this provider's own definition said it derives for this consumer, already derived by
|
|
* the mesh (novox/hq ADR 0201).
|
|
*
|
|
* **The rule is declared, not computed here.** A provider that names the resource it gives each
|
|
* consumer — a bucket, a database, a vhost — writes that name in its `serves` block with
|
|
* `${consumer:as}` or `${consumer:as:dns}` in it. The mesh fills it for this consumer and hands
|
|
* the same filled value to the consumer's own configuration, so the two ends cannot disagree.
|
|
* An adapter reads it from here; an adapter that recomputed it would be the second rule this
|
|
* removed. Empty for a provider that derives nothing. */
|
|
readonly derived: Readonly<Record<string, unknown>>;
|
|
/** Where the consumer is, for a provider that must reach back to it. Usually unused. */
|
|
readonly at?: string;
|
|
/** The consumer node, for logging and lifecycle events. */
|
|
readonly consumer?: string;
|
|
}
|
|
|
|
/** What a provider implements — the only per-service code. It is handed the login and password the
|
|
* mesh made and brings the resource into being under them, or withdraws it. It returns nothing:
|
|
* the credential is the mesh's, already delivered to the consumer. */
|
|
export interface Adapter {
|
|
create(p: Provision): Promise<void>;
|
|
/** Remove what `create` made for this consumer. **Called only to DELETE a retired consumer, on a
|
|
* person's approved `cleanup delete`, and only when the adapter has no `delete`** (novox/hq ADR
|
|
* 0230): many providers' remove destroys something — a ledger file, a public DNS record, a user —
|
|
* and nothing is destroyed without a person. Retiring never calls it. `derived` is what the mesh
|
|
* last derived for it, remembered by the harness: the mesh has already dropped the consumer from
|
|
* the contributions file by then, so it cannot be read again, and a provider that recomputed it
|
|
* would be keeping the second copy of a rule this removed (novox/hq ADR 0201). */
|
|
remove(p: { readonly as: string; readonly derived: Readonly<Record<string, unknown>> }): Promise<void>;
|
|
/** Optional: whether the backend still holds this consumer's credential exactly as `p` says.
|
|
* Asked of every consumer already applied, every `verifyEveryMs`. `false` makes the harness apply
|
|
* it again on the same pass, so a backend that lost what it was given (a server restarted
|
|
* without persisting its users, a restore, a login removed by hand) is provisioned again instead
|
|
* of being trusted from memory (novox/hq issue 120). An adapter without it is trusted from memory,
|
|
* as before. It must only read: it is asked often, and must never change the backend. */
|
|
holds?(p: Provision): Promise<boolean>;
|
|
/** Optional: RETIRE a consumer the mesh stopped asking for (novox/hq ADR 0230) — DISABLE its
|
|
* access, reversibly, and mark it in the backend "to delete" with when and why; delete nothing.
|
|
* Asked for again, `reenable` (when given) and then `create` bring it back as it was.
|
|
*
|
|
* **An adapter without it is MARK-ONLY**: retiring a consumer then changes nothing in the backend —
|
|
* its access stays as it was — and it is only marked, in this harness, "retired, access kept, needs
|
|
* deletion", said loudly and announced. Only a person's `cleanup delete` then removes it. */
|
|
retire?(p: {
|
|
readonly as: string;
|
|
readonly derived: Readonly<Record<string, unknown>>;
|
|
readonly why: string;
|
|
readonly at: Date;
|
|
}): Promise<void>;
|
|
/** Optional, with `retire`: undo what `retire` disabled, before `create` runs for a consumer asked
|
|
* for again. An adapter whose `create` already re-enables needs none. */
|
|
reenable?(p: { readonly as: string; readonly derived: Readonly<Record<string, unknown>> }): Promise<void>;
|
|
/** Optional: every consumer the backend holds retired — what `cleanup list` shows. */
|
|
retired?(): Promise<RetiredConsumer[]>;
|
|
/** Optional: DELETE one retired consumer and its data. Only ever called for a consumer this
|
|
* harness or `retired()` knows retired, and only through `provisioner_delete`, a person's act. */
|
|
delete?(p: { readonly as: string; readonly why: string }): Promise<{ freedBytes?: number } | void>;
|
|
/** Optional: every consumer the backend holds ACTIVE that the mesh made, read at start, so one the
|
|
* mesh stopped asking for while this process was down is still retired (by the same rules). */
|
|
held?(): Promise<string[]>;
|
|
/** Optional: the class of one of the adapter's own errors (novox/hq ADR 0224), when its code knows
|
|
* better than the error's words; "" or undefined falls back to `classOf` of the text. */
|
|
classify?(err: unknown): string | undefined;
|
|
}
|
|
|
|
/** One retired consumer, as `provisioner_retirement` answers it. */
|
|
export interface RetiredConsumer {
|
|
readonly consumer: string;
|
|
readonly node?: string;
|
|
/** When it was retired, RFC 3339, or "" when the backend does not know. */
|
|
readonly retired_at: string;
|
|
readonly why: string;
|
|
/** Its size in bytes, or -1 when not knowable. */
|
|
readonly size_bytes: number;
|
|
/** "consumer", or a backend's own word for something else it keeps retired. */
|
|
readonly kind: string;
|
|
/** "disabled" when the adapter retired it; "kept" for a mark-only provider, whose consumer keeps its
|
|
* access until a person deletes it (novox/hq ADR 0230). */
|
|
readonly access?: "disabled" | "kept";
|
|
}
|
|
|
|
/** What `provisioner_retirement` answers: the provider's retirement state (novox/hq ADR 0230). */
|
|
export interface Retirement {
|
|
readonly resource: string;
|
|
readonly node: string;
|
|
readonly held: string[];
|
|
readonly stable_passes: number;
|
|
readonly stable_for_seconds: number;
|
|
/** "disable" when the adapter can retire; "mark-only" when it cannot (novox/hq ADR 0230). */
|
|
readonly retires_by: "disable" | "mark-only";
|
|
readonly bound: string;
|
|
readonly waiting: { consumers: { consumer: string; node?: string }[]; since: string; held: number } | null;
|
|
readonly rejected: { consumers: { consumer: string; node?: string }[]; by: string; why: string; at: string } | null;
|
|
readonly retired: RetiredConsumer[];
|
|
}
|
|
|
|
/** Who acts on a waiting retirement or a deletion, and why. `via` is "mesh-controller" when asked
|
|
* through the controller's verbs. */
|
|
export interface Act {
|
|
readonly why: string;
|
|
readonly by?: string;
|
|
readonly via?: string;
|
|
}
|
|
|
|
/** The running provider loop: a stop function, as before, that also answers and takes the person's
|
|
* acts on retirement (novox/hq ADR 0230). */
|
|
export type ProviderLoop = (() => void) & {
|
|
retirement(): Promise<Retirement>;
|
|
approve(consumers: readonly string[], act: Act): Promise<{ retired: string[] }>;
|
|
reject(consumers: readonly string[], act: Act): Promise<{ kept: string[] }>;
|
|
deleteRetired(consumer: string, act: Act & { readonly confirm?: string }): Promise<{ deleted: string; freed_bytes: number }>;
|
|
};
|
|
|
|
/** The one event a provider says about retirement (novox/hq ADR 0230). */
|
|
export const RETIREMENT_EVENT = "provisioner.retirement";
|
|
|
|
/** The events a provider's standing toward one consumer is announced as (novox/hq ADR 0224). The
|
|
* controller derives the permission to emit them for every module that receives contributions. */
|
|
export const FAILING_EVENT = "provisioner.failing";
|
|
export const RECOVERED_EVENT = "provisioner.recovered";
|
|
|
|
/** The classes a failing consumer is announced with: what a person reading `status` needs to know
|
|
* before reading the journal (novox/hq ADR 0224). An adapter may say better (`classify`). */
|
|
export const CLASS_CREDENTIALS = "credentials-rejected";
|
|
export const CLASS_UNREACHABLE = "unreachable";
|
|
export const CLASS_SECRET = "secret-unreadable";
|
|
export const CLASS_REFUSED = "refused";
|
|
|
|
/** How long a consumer fails without one success before it is announced failing (ADR 0224). */
|
|
export const FAILING_AFTER_MS = 5 * 60_000;
|
|
|
|
export interface ProviderLoopOptions {
|
|
/** The contributions file the mesh writes for this resource (the module's `receives` path).
|
|
* Defaults to $MESH_RECEIVES. */
|
|
receives?: string;
|
|
/** Reconcile interval in ms. Defaults to 5000. */
|
|
everyMs?: number;
|
|
/** How often, in ms, an adapter with `holds` is asked whether the backend still holds each
|
|
* applied consumer. Defaults to 60000: slower than reconciling, because it reads the backend for
|
|
* every consumer, and fast enough that a lost login is back within a minute. */
|
|
verifyEveryMs?: number;
|
|
/** How long one `holds` may take before it counts as "could not ask". Defaults to 30000: the
|
|
* loop is sequential, so a check that hangs would otherwise stop every consumer's provisioning. */
|
|
holdsTimeoutMs?: number;
|
|
/** How many consecutive passes that read the file must name the same set of consumers no longer
|
|
* asked for before they are retired (novox/hq ADR 0230). Defaults to 5. Additions and changes are
|
|
* never delayed. */
|
|
stablePasses?: number;
|
|
/** And how long, in ms, that same set must have held since the first pass that saw it. Defaults to
|
|
* STABLE_FOR_MS, ten minutes: five passes alone are 25 s, shorter than a controller restart, a store
|
|
* reconnecting or a file half written. Both must hold. */
|
|
stableForMs?: number;
|
|
/** A stable set larger than this is not retired without a person. Defaults to 3. */
|
|
retireAtOnce?: number;
|
|
/** Nor a set larger than this share of the consumers held, where more than one is held. Defaults
|
|
* to 0.5. */
|
|
retireFraction?: number;
|
|
/** How often a waiting retirement, and a consumer still failing, is said again, in ms. Defaults to
|
|
* 15 minutes. */
|
|
sayAgainEveryMs?: number;
|
|
/** How long, in ms, a consumer fails without one success before it is announced
|
|
* `provisioner.failing` (novox/hq ADR 0224). Defaults to FAILING_AFTER_MS, five minutes. */
|
|
failingAfterMs?: number;
|
|
/** Publishes one of the provider's events (`provisioner.failing`, `provisioner.recovered`,
|
|
* `provisioner.retirement`). Defaults to the SDK's `emit`, so every provider says them on the bus
|
|
* without asking (novox/hq issue 272); a failed publish is said in the journal and never fails
|
|
* the act it reports. */
|
|
announce?: (event: string, body: Record<string, unknown>) => Promise<void> | void;
|
|
/** The machine this provider runs on, said in each event. Defaults to $MESH_NODE. */
|
|
node?: string;
|
|
/** The module's name: when given, the four retirement tools are registered for it through
|
|
* registerModuleTools, so the controller's `retire` and `cleanup` verbs can reach them. */
|
|
module?: string;
|
|
}
|
|
|
|
/** The least time the same unasked set must hold before it is retired (novox/hq ADR 0230). */
|
|
export const STABLE_FOR_MS = 10 * 60_000;
|
|
|
|
/** The longest a consumer whose `create` keeps failing to satisfy `holds` waits between checks. */
|
|
const MAX_BACKOFF_MS = 60 * 60_000;
|
|
|
|
/** How many passes a secret may be unreadable before the harness stops calling it a race. At the
|
|
* default cadence this is about a minute, which is longer than the host takes to write a file
|
|
* and far shorter than the three hours issue 225 went unread. */
|
|
const patiently = 12;
|
|
|
|
/** Once it is saying so loudly, how often to repeat it. Rare enough to stay readable, often
|
|
* enough that somebody arriving at the log finds it. */
|
|
const loudlyEvery = 240;
|
|
|
|
/** One entry in the mesh's contributions file: a consumer the provider must serve. */
|
|
interface Contribution {
|
|
readonly as: string;
|
|
readonly secret: string;
|
|
readonly node?: string;
|
|
readonly at?: string;
|
|
readonly values?: Readonly<Record<string, unknown>>;
|
|
readonly derived?: Readonly<Record<string, unknown>>;
|
|
}
|
|
|
|
/**
|
|
* Run the reconcile loop for one provided resource. Returns a stop function, which also carries the
|
|
* provider's retirement state and a person's acts on it. Never throws for a single bad contribution —
|
|
* it logs and keeps converging, because one consumer's failure must not stop the others'
|
|
* provisioning.
|
|
*
|
|
* **A consumer the mesh stops asking for is retired, never withdrawn whole (novox/hq ADR 0230).**
|
|
* ACTIVE → RETIRED once the same set of consumers has gone unasked in `stablePasses` consecutive
|
|
* passes that read the file AND for `stableForMs` (ten minutes); RETIRED is access disabled and
|
|
* marked — or, for an adapter with no `retire`, only marked here with its access kept — nothing
|
|
* deleted, and asked for again it is re-enabled at once. `remove` is never called to retire. A stable set of more than `retireAtOnce`, or more than
|
|
* `retireFraction` of those held where more than one is held, is not retired without a person: the
|
|
* provider waits for approve or reject. RETIRED → DELETED only through `provisioner_delete`.
|
|
*/
|
|
export function runProviderLoop(resource: string, adapter: Adapter, opts: ProviderLoopOptions = {}): ProviderLoop {
|
|
const receives = opts.receives ?? envOrThrow("MESH_RECEIVES");
|
|
const everyMs = opts.everyMs ?? 5000;
|
|
const verifyEveryMs = opts.verifyEveryMs ?? 60_000;
|
|
const holdsTimeoutMs = opts.holdsTimeoutMs ?? 30_000;
|
|
const stablePasses = opts.stablePasses ?? 5;
|
|
const stableForMs = opts.stableForMs ?? STABLE_FOR_MS;
|
|
const retiresBy: "disable" | "mark-only" = adapter.retire ? "disable" : "mark-only";
|
|
const retireAtOnce = opts.retireAtOnce ?? 3;
|
|
const retireFraction = opts.retireFraction ?? 0.5;
|
|
const sayAgainEveryMs = opts.sayAgainEveryMs ?? 15 * 60_000;
|
|
const failingAfterMs = opts.failingAfterMs ?? FAILING_AFTER_MS;
|
|
const announce = opts.announce ?? ((event: string, body: Record<string, unknown>) => emit(event, body));
|
|
const node = opts.node ?? process.env.MESH_NODE ?? "";
|
|
const say = (line: string): void => console.error(`[provisioner:${resource}] ${line}`);
|
|
// Retirement (novox/hq ADR 0230). The unasked set last seen and in how many consecutive readable
|
|
// passes; a set too large, waiting for a person; a set a person rejected; and every consumer known
|
|
// retired, by name.
|
|
let stable = { set: "", count: 0, since: 0 };
|
|
let pending: { consumers: string[]; since: number; held: number; saidAt: number } | null = null;
|
|
let rejected: { consumers: string[]; by: string; why: string; at: number } | null = null;
|
|
const retired = new Map<string, {
|
|
node?: string; at: number; why: string; derived?: Readonly<Record<string, unknown>>; access: "disabled" | "kept";
|
|
}>();
|
|
let seeded = false;
|
|
let verifiedAt = 0;
|
|
// Consumers the backend reported lost, and how many times in a row. A `create` that succeeded and
|
|
// is still not held on the next check will never be: something in the adapter disagrees with
|
|
// itself. Each repeat doubles the wait before asking again, so such a bug costs one re-apply an
|
|
// hour rather than one a minute, and it is said loudly rather than quietly repeated.
|
|
const lost = new Map<string, { times: number; nextAt: number }>();
|
|
// Each consumer's unbroken run of failures — since when, how often, the last error, and when it was
|
|
// last announced (0: not yet) — and every consumer that has succeeded since this process started
|
|
// (novox/hq ADR 0224).
|
|
const trouble = new Map<string, {
|
|
node?: string; since: number; attempts: number; cls: string; text: string; saidAt: number;
|
|
}>();
|
|
const cleared = new Set<string>();
|
|
|
|
// login (`as`) -> what was last applied: the hash that says whether it changed, and the
|
|
// derivation to hand back on withdrawal, which by then is no longer in the file.
|
|
const applied = new Map<string, { hash: string; derived: Readonly<Record<string, unknown>>; node?: string }>();
|
|
// How many passes each consumer's secret has been unreadable for, so the wait can stop sounding
|
|
// like a wait.
|
|
const waiting = new Map<string, number>();
|
|
let stopped = false;
|
|
// A reason the contributions file could not be used, said once until it changes.
|
|
let lastWarning = "";
|
|
const warn = (why: string): void => {
|
|
if (why === lastWarning) return;
|
|
if (why) console.error(`[provisioner:${resource}] ${why}; nothing applied or removed until it can be read`);
|
|
else console.error(`[provisioner:${resource}] contributions file readable again`);
|
|
lastWarning = why;
|
|
};
|
|
|
|
async function reconcile(): Promise<void> {
|
|
const given = await readContributions(receives, resource, warn);
|
|
// **Nothing read is not nobody asking** (novox/hq issue 241). A file that could not be read,
|
|
// parsed or recognised says nothing about which consumers the mesh wants, and treating it as
|
|
// "none" withdrew every consumer at once: on 2026-10-04 one failed read made the postgres provider
|
|
// drop every database it held. Only a file that was read can withdraw anything, so the whole pass
|
|
// waits for the next one.
|
|
if (given === null) {
|
|
// Not a result: the passes that make a retirement stable must each have read the file.
|
|
stable = { set: "", count: 0, since: 0 };
|
|
return;
|
|
}
|
|
if (!seeded) await seed();
|
|
const wantByAs = new Map(given.map((g) => [g.as, g]));
|
|
// On this pass, ask the backend rather than memory whether each applied consumer is still there.
|
|
let verifying = adapter.holds !== undefined && Date.now() - verifiedAt >= verifyEveryMs;
|
|
if (verifying) verifiedAt = Date.now();
|
|
|
|
// Create or update every consumer whose login, password or values changed.
|
|
for (const g of given) {
|
|
let password: string;
|
|
try {
|
|
// The file the mesh sealed to this node, unsealed by the host into plaintext. Trailing
|
|
// newline trimmed: a sealed value is exactly the secret, and a host writing a file may add
|
|
// one.
|
|
password = (await readFile(g.secret, "utf8")).replace(/\n$/, "");
|
|
} catch (err) {
|
|
// The contribution names a secret the host has not written yet — a normal race on the
|
|
// first pass. Skipped, retried on the next tick, never fatal.
|
|
//
|
|
// **"Yet" stops being true, and then the word is the fault** (novox/hq issue 225). A
|
|
// permission the host will never grant reads exactly like a file that has not arrived,
|
|
// and the two want opposite responses: one is waiting, the other is a person. Said the
|
|
// patient way while it is plausibly a race, and then said plainly, once, and from then
|
|
// on only occasionally — because a line repeated every five seconds is a line nobody
|
|
// reads. Three hours of "not readable yet" is how a provider that served nobody looked
|
|
// healthy.
|
|
const n = (waiting.get(g.as) ?? 0) + 1;
|
|
waiting.set(g.as, n);
|
|
if (n <= patiently) {
|
|
console.error(`[provisioner:${resource}] ${g.as}: secret not readable yet (${g.secret}): ${err}`);
|
|
} else if (n === patiently + 1 || n % loudlyEvery === 0) {
|
|
console.error(
|
|
`[provisioner:${resource}] ${g.as}: CANNOT READ the secret after ${n} attempts ` +
|
|
`(${g.secret}): ${err}. This is not a race any more — nothing has been provisioned ` +
|
|
`for this consumer and nothing will be until somebody looks. Check who owns the ` +
|
|
`file and who this process runs as (novox/hq issue 225)`,
|
|
);
|
|
}
|
|
await failed(g.as, g.node, CLASS_SECRET, `secret not readable (${g.secret}): ${err}`);
|
|
continue;
|
|
}
|
|
waiting.delete(g.as);
|
|
const h = hash(g.as, password, g.values ?? {}, g.derived ?? {});
|
|
const p: Provision = {
|
|
as: g.as, password, values: g.values ?? {}, derived: g.derived ?? {}, at: g.at, consumer: g.node,
|
|
};
|
|
// Set when the backend reported this consumer lost: how many successful re-applies in a row
|
|
// it has now needed, counting this one.
|
|
let reapplying: number | undefined;
|
|
if (applied.get(g.as)?.hash === h) {
|
|
if (!verifying) continue;
|
|
const brake = lost.get(g.as);
|
|
if (brake && Date.now() < brake.nextAt) continue;
|
|
try {
|
|
if (await withTimeout(adapter.holds!(p), holdsTimeoutMs)) {
|
|
lost.delete(g.as);
|
|
await succeeded(g.as);
|
|
continue;
|
|
}
|
|
} catch (err) {
|
|
// Unable to ask is not evidence of loss. Kept as applied, asked again next time — but a
|
|
// check that keeps failing is a consumer failed all the same (ADR 0224, issue 179).
|
|
const text = scrub(err, password);
|
|
console.error(`[provisioner:${resource}] ${g.as}: could not check the backend, will ask again: ${text}`);
|
|
await failed(g.as, g.node, classed(err, text), text);
|
|
// A backend that did not answer in time will not answer for the next consumer either, and
|
|
// each would cost a timeout, so the rest of this pass is not asked. Any other failure may
|
|
// be this consumer's alone, and the others are still asked.
|
|
if (err instanceof HoldsTimeout) verifying = false;
|
|
continue;
|
|
}
|
|
reapplying = (brake?.times ?? 0) + 1;
|
|
if (reapplying === 1) {
|
|
// Said, because it means the backend lost something while nothing was looking.
|
|
console.error(`[provisioner:${resource}] ${g.as}: the backend no longer holds it; applying again`);
|
|
} else {
|
|
console.error(
|
|
`[provisioner:${resource}] ${g.as}: still not held after being applied again (${reapplying} times in a row) — ` +
|
|
`create does not produce what holds checks; applying again`,
|
|
);
|
|
}
|
|
}
|
|
try {
|
|
const was = retired.get(g.as);
|
|
if (was && was.access === "disabled" && adapter.reenable) {
|
|
await adapter.reenable({ as: g.as, derived: was.derived ?? p.derived });
|
|
}
|
|
await adapter.create(p);
|
|
await succeeded(g.as);
|
|
applied.set(g.as, { hash: h, derived: p.derived, node: g.node });
|
|
if (retired.has(g.as)) {
|
|
// Asked for again: the ordinary create re-enabled it and cleared its mark (ADR 0230).
|
|
retired.delete(g.as);
|
|
say(`${g.as}: asked for again; re-enabled as it was`);
|
|
await tell("reenabled", { consumers: [{ consumer: g.as, node: g.node }] });
|
|
}
|
|
if (reapplying === undefined) {
|
|
// Applied for a new login, password or values: whatever was counted before does not carry over.
|
|
lost.delete(g.as);
|
|
} else {
|
|
// Only a create that succeeded counts toward the brake: if the backend still does not hold
|
|
// it at the next check, create and holds disagree, and each repeat waits twice as long.
|
|
const waitMs = Math.min(MAX_BACKOFF_MS, verifyEveryMs * 2 ** (reapplying - 1));
|
|
lost.set(g.as, { times: reapplying, nextAt: Date.now() + waitMs });
|
|
if (reapplying > 1) {
|
|
console.error(`[provisioner:${resource}] ${g.as}: next check in ${Math.round(waitMs / 1000)}s`);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
const text = scrub(err, password);
|
|
console.error(`[provisioner:${resource}] ${g.as}: create failed, will retry: ${text}`);
|
|
await failed(g.as, g.node, classed(err, text), text);
|
|
// A failed create is not a disagreement: asked again at the next check, the count unchanged.
|
|
if (reapplying !== undefined) lost.set(g.as, { times: reapplying - 1, nextAt: 0 });
|
|
}
|
|
}
|
|
|
|
// Retire what the mesh no longer asks for — once the same set has gone unasked for
|
|
// `stablePasses` passes, and within the bound, or with a person's word (novox/hq ADR 0230).
|
|
// Only consumers this harness applied, or the backend says the mesh made, are touched.
|
|
const unasked = [...applied.keys()].filter((as) => !wantByAs.has(as)).sort();
|
|
// A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed
|
|
// by anyone: said, so the standing the controller keeps for it is cleared rather than left naming a
|
|
// consumer that is gone. One still held is said recovered when it is retired (ADR 0224, 0230).
|
|
for (const as of [...trouble.keys()]) {
|
|
if (!wantByAs.has(as) && !applied.has(as)) await recovered(as, "no longer asked for");
|
|
}
|
|
const key = unasked.join(",");
|
|
if (pending && pending.consumers.join(",") !== key) {
|
|
say(`the retirement waiting for a person (${pending.consumers.join(", ")}) is no longer what the mesh says; settled, nothing retired`);
|
|
await tell("settled", { consumers: named(pending.consumers) });
|
|
pending = null;
|
|
}
|
|
if (rejected && rejected.consumers.join(",") !== key) {
|
|
say(`the rejected retirement (${rejected.consumers.join(", ")}) is no longer what the mesh says`);
|
|
await tell("settled", { consumers: named(rejected.consumers) });
|
|
rejected = null;
|
|
}
|
|
if (unasked.length === 0) {
|
|
stable = { set: "", count: 0, since: 0 };
|
|
return;
|
|
}
|
|
stable = stable.set === key
|
|
? { set: key, count: stable.count + 1, since: stable.since }
|
|
: { set: key, count: 1, since: Date.now() };
|
|
if (stable.count < stablePasses || Date.now() - stable.since < stableForMs) return;
|
|
if (rejected) return; // a person said keep them, and the mesh still says the same
|
|
const held = applied.size;
|
|
if (overTheBound(unasked.length, held)) {
|
|
const now = Date.now();
|
|
if (pending === null) {
|
|
pending = { consumers: unasked, since: now, held, saidAt: now };
|
|
say(
|
|
`RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for ${unasked.length} of the ${held} consumer(s) ` +
|
|
`this provider holds (${unasked.join(", ")}), ${bound(held)}. Nothing is retired until ` +
|
|
`\`retire approve\` or \`retire reject\` (novox/hq ADR 0230)`,
|
|
);
|
|
await tell("waiting", { consumers: named(unasked), held, since: iso(now) });
|
|
} else if (now - pending.saidAt >= sayAgainEveryMs) {
|
|
pending.saidAt = now;
|
|
await tell("waiting", { consumers: named(unasked), held: pending.held, since: iso(pending.since) });
|
|
}
|
|
return;
|
|
}
|
|
const why = `the mesh stopped asking for it: the same answer in ${stable.count} passes over ` +
|
|
`${Math.round((Date.now() - stable.since) / 1000)}s, from ${iso(stable.since)}`;
|
|
const done = await retireAll(unasked, why);
|
|
if (done.length > 0) await tell("retired", { consumers: done.map((as) => retiredBody(as)), held, why });
|
|
}
|
|
|
|
function retiresByAccess(): "disabled" | "kept" {
|
|
return adapter.retire ? "disabled" : "kept";
|
|
}
|
|
|
|
function overTheBound(n: number, held: number): boolean {
|
|
return n > retireAtOnce || (held > 1 && n > retireFraction * held);
|
|
}
|
|
|
|
function bound(held: number): string {
|
|
return `more than ${retireAtOnce}, or more than ${Math.round(retireFraction * 100)}% of the ${held} held`;
|
|
}
|
|
|
|
// Retire each, as it is said: disabled and marked by the adapter — or, where the adapter cannot
|
|
// disable, only marked here, its access kept. Nothing is deleted, and `remove` is never called: it
|
|
// is the delete of a provider without one. Answers those that went; one that failed stays active and
|
|
// is asked again once its set is stable again.
|
|
async function retireAll(names: readonly string[], why: string): Promise<string[]> {
|
|
const done: string[] = [];
|
|
for (const as of names) {
|
|
const was = applied.get(as);
|
|
if (was === undefined) continue;
|
|
try {
|
|
const at = new Date();
|
|
if (adapter.retire) {
|
|
say(`${as}: no longer in ${receives}; RETIRED — access disabled, nothing deleted: ${why}`);
|
|
await adapter.retire({ as, derived: was.derived, why, at });
|
|
} else {
|
|
say(
|
|
`${as}: no longer in ${receives}; RETIRED, MARK ONLY — this provider cannot disable a consumer ` +
|
|
`(its adapter has no retire), so ${as} KEEPS ITS ACCESS and its data until a person deletes it ` +
|
|
`with \`cleanup delete\`. Nothing was changed in the backend (novox/hq ADR 0230): ${why}`,
|
|
);
|
|
}
|
|
applied.delete(as);
|
|
lost.delete(as);
|
|
await recovered(as, "retired");
|
|
retired.set(as, { node: was.node, at: at.getTime(), why, derived: was.derived, access: adapter.retire ? "disabled" : "kept" });
|
|
done.push(as);
|
|
} catch (err) {
|
|
say(`${as}: retiring failed, will retry: ${err}`);
|
|
}
|
|
}
|
|
stable = { set: "", count: 0, since: 0 };
|
|
return done;
|
|
}
|
|
|
|
function named(names: readonly string[]): { consumer: string; node?: string }[] {
|
|
return names.map((as) => ({ consumer: as, node: applied.get(as)?.node ?? retired.get(as)?.node }));
|
|
}
|
|
|
|
function retiredBody(as: string): Record<string, unknown> {
|
|
const r = retired.get(as);
|
|
return { consumer: as, node: r?.node, retired_at: r ? iso(r.at) : "", why: r?.why ?? "", access: r?.access ?? retiresByAccess() };
|
|
}
|
|
|
|
// Said on the bus, never failing the act it reports (the journal still has it).
|
|
async function tell(change: string, body: Record<string, unknown>): Promise<void> {
|
|
await said(RETIREMENT_EVENT, {
|
|
provider: resource, "provider-node": node, change, bound: bound(applied.size), at: iso(Date.now()), ...body,
|
|
}, change);
|
|
}
|
|
|
|
async function said(event: string, body: Record<string, unknown>, what = ""): Promise<void> {
|
|
try {
|
|
await announce(event, body);
|
|
} catch (err) {
|
|
say(`could not announce ${event}${what ? ` ${what}` : ""}: ${err}`);
|
|
}
|
|
}
|
|
|
|
// An error's class: the adapter's word when it has one, else read from the text (ADR 0224).
|
|
function classed(err: unknown, text: string): string {
|
|
try {
|
|
const own = adapter.classify?.(err);
|
|
if (own) return own;
|
|
} catch {
|
|
// An adapter that cannot class its own error is read by the words.
|
|
}
|
|
// A check that did not answer in time is a backend out of reach, whatever its words.
|
|
if (err instanceof HoldsTimeout) return CLASS_UNREACHABLE;
|
|
return classOf(text);
|
|
}
|
|
|
|
// One more failure in a consumer's unbroken run, announced once the run has lasted failingAfterMs,
|
|
// then again every sayAgainEveryMs while it lasts (novox/hq ADR 0224).
|
|
async function failed(as: string, consumerNode: string | undefined, cls: string, text: string): Promise<void> {
|
|
const now = Date.now();
|
|
let s = trouble.get(as);
|
|
if (!s) {
|
|
s = { since: now, attempts: 0, cls, text, saidAt: 0 };
|
|
trouble.set(as, s);
|
|
}
|
|
s.node = consumerNode;
|
|
s.cls = cls;
|
|
s.text = text;
|
|
s.attempts++;
|
|
if (now - s.since < failingAfterMs) return;
|
|
if (s.saidAt !== 0 && now - s.saidAt < sayAgainEveryMs) return;
|
|
if (s.saidAt === 0) {
|
|
say(
|
|
`${as}: FAILING for ${Math.round((now - s.since) / 1000)}s (${s.attempts} attempts, ${cls}): ${text}. ` +
|
|
`Announced as ${FAILING_EVENT}; \`status\` names it until it succeeds (novox/hq ADR 0224)`,
|
|
);
|
|
}
|
|
s.saidAt = now;
|
|
await said(FAILING_EVENT, {
|
|
provider: resource, "provider-node": node, consumer: as, node: consumerNode ?? "",
|
|
class: cls, error: clip(text), since: iso(s.since), attempts: s.attempts,
|
|
});
|
|
}
|
|
|
|
// A success ends a consumer's run of failures; one that was announced is announced recovered. And
|
|
// the first success for a consumer since this process started is announced too, failing or not: a
|
|
// provider that announced a failure and was restarted has forgotten it, and without this the
|
|
// controller would name the consumer failing for ever after it recovered unheard.
|
|
async function succeeded(as: string): Promise<void> {
|
|
if (!trouble.has(as) && !cleared.has(as)) {
|
|
cleared.add(as);
|
|
await said(RECOVERED_EVENT, { provider: resource, "provider-node": node, consumer: as, why: "first-success" });
|
|
return;
|
|
}
|
|
cleared.add(as);
|
|
await recovered(as, "");
|
|
}
|
|
|
|
async function recovered(as: string, why: string): Promise<void> {
|
|
const s = trouble.get(as);
|
|
if (!s) return;
|
|
trouble.delete(as);
|
|
if (s.saidAt === 0) return; // never announced, so there is nothing to take back
|
|
if (!why) say(`${as}: recovered after ${Math.round((Date.now() - s.since) / 1000)}s and ${s.attempts} failed attempt(s)`);
|
|
await said(RECOVERED_EVENT, {
|
|
provider: resource, "provider-node": node, consumer: as, node: s.node ?? "",
|
|
since: iso(s.since), attempts: s.attempts, ...(why ? { why } : {}),
|
|
});
|
|
}
|
|
|
|
// What the backend holds that the mesh made: active ones become held, so one the mesh stopped asking
|
|
// for while this process was down is still retired, by the same rules; retired ones are known.
|
|
async function seed(): Promise<void> {
|
|
try {
|
|
for (const as of (await adapter.held?.()) ?? []) {
|
|
if (!applied.has(as)) applied.set(as, { hash: "", derived: {} });
|
|
}
|
|
for (const r of (await adapter.retired?.()) ?? []) {
|
|
if (r.kind === "consumer" && !retired.has(r.consumer)) {
|
|
retired.set(r.consumer, {
|
|
node: r.node, at: r.retired_at ? Date.parse(r.retired_at) : 0, why: r.why, access: r.access ?? retiresByAccess(),
|
|
});
|
|
}
|
|
}
|
|
seeded = true;
|
|
} catch (err) {
|
|
say(`cannot read what the backend holds yet, will ask again: ${err}`);
|
|
}
|
|
}
|
|
|
|
// One act at a time: a person's approval and a pass never interleave.
|
|
let queue: Promise<unknown> = Promise.resolve();
|
|
function serially<T>(f: () => Promise<T>): Promise<T> {
|
|
const run = queue.then(f, f);
|
|
queue = run.catch(() => undefined);
|
|
return run;
|
|
}
|
|
|
|
const sameSet = (a: readonly string[], b: readonly string[]): boolean =>
|
|
[...a].sort().join(",") === [...b].sort().join(",");
|
|
|
|
async function retirement(): Promise<Retirement> {
|
|
const fromBackend = (await adapter.retired?.()) ?? [];
|
|
const list: RetiredConsumer[] = [...fromBackend];
|
|
for (const [as, r] of retired) {
|
|
if (!list.some((x) => x.consumer === as && x.kind === "consumer")) {
|
|
list.push({
|
|
consumer: as, node: r.node, retired_at: r.at ? iso(r.at) : "", why: r.why, size_bytes: -1, kind: "consumer",
|
|
access: r.access,
|
|
});
|
|
}
|
|
}
|
|
return {
|
|
resource, node, held: [...applied.keys()].sort(), stable_passes: stablePasses,
|
|
stable_for_seconds: Math.round(stableForMs / 1000), retires_by: retiresBy, bound: bound(applied.size),
|
|
waiting: pending && { consumers: named(pending.consumers), since: iso(pending.since), held: pending.held },
|
|
rejected: rejected && { consumers: named(rejected.consumers), by: rejected.by, why: rejected.why, at: iso(rejected.at) },
|
|
retired: list,
|
|
};
|
|
}
|
|
|
|
function needWhy(act: Act, verb: string): void {
|
|
if (!act.why || !act.why.trim()) throw new Error(`${verb}: why is required`);
|
|
}
|
|
|
|
async function approve(consumers: readonly string[], act: Act): Promise<{ retired: string[] }> {
|
|
needWhy(act, "approve");
|
|
return serially(async () => {
|
|
const set = pending?.consumers ?? rejected?.consumers;
|
|
if (!set || !sameSet(set, consumers)) {
|
|
throw new Error(
|
|
`approve: nothing waits with exactly that set (${[...consumers].sort().join(", ")}); ` +
|
|
`waiting: ${pending?.consumers.join(", ") || "none"}; rejected: ${rejected?.consumers.join(", ") || "none"}. Look again`,
|
|
);
|
|
}
|
|
const held = applied.size;
|
|
say(`retirement of ${set.join(", ")} approved by ${act.by || "someone"}: ${act.why}`);
|
|
await tell("approved", { consumers: named(set), held, why: act.why, by: act.by ?? "", via: act.via ?? "" });
|
|
pending = null;
|
|
rejected = null;
|
|
const why = `approved by ${act.by || "someone"}: ${act.why}`;
|
|
const done = await retireAll(set, why);
|
|
if (done.length > 0) await tell("retired", { consumers: done.map((as) => retiredBody(as)), held, why });
|
|
return { retired: done };
|
|
});
|
|
}
|
|
|
|
async function reject(consumers: readonly string[], act: Act): Promise<{ kept: string[] }> {
|
|
needWhy(act, "reject");
|
|
return serially(async () => {
|
|
if (!pending || !sameSet(pending.consumers, consumers)) {
|
|
throw new Error(
|
|
`reject: nothing waits with exactly that set (${[...consumers].sort().join(", ")}); ` +
|
|
`waiting: ${pending?.consumers.join(", ") || "none"}. Look again`,
|
|
);
|
|
}
|
|
const kept = pending.consumers;
|
|
rejected = { consumers: kept, by: act.by ?? "", why: act.why, at: Date.now() };
|
|
pending = null;
|
|
say(`retirement of ${kept.join(", ")} rejected by ${act.by || "someone"}: ${act.why}; kept active while the mesh says the same`);
|
|
await tell("rejected", { consumers: named(kept), why: act.why, by: act.by ?? "", via: act.via ?? "" });
|
|
return { kept };
|
|
});
|
|
}
|
|
|
|
async function deleteRetired(consumer: string, act: Act & { readonly confirm?: string }): Promise<{ deleted: string; freed_bytes: number }> {
|
|
needWhy(act, "delete");
|
|
if (act.confirm !== consumer) throw new Error("delete: confirm must repeat the consumer's name");
|
|
return serially(async () => {
|
|
if (applied.has(consumer)) throw new Error(`delete: ${consumer} is active, not retired — only a retired consumer is deleted`);
|
|
const known = retired.has(consumer) || ((await adapter.retired?.()) ?? []).some((r) => r.consumer === consumer);
|
|
if (!known) throw new Error(`delete: ${consumer} is not retired here`);
|
|
const was = retired.get(consumer);
|
|
say(`${consumer}: DELETING, asked by ${act.by || "someone"}: ${act.why}`);
|
|
let freed = -1;
|
|
if (adapter.delete) {
|
|
const r = await adapter.delete({ as: consumer, why: act.why });
|
|
if (r && typeof r.freedBytes === "number") freed = r.freedBytes;
|
|
} else {
|
|
// The one path to `remove`: a person's approved deletion (novox/hq ADR 0230).
|
|
await adapter.remove({ as: consumer, derived: was?.derived ?? {} });
|
|
}
|
|
retired.delete(consumer);
|
|
await tell("deleted", {
|
|
consumers: [{ consumer, node: was?.node, size_bytes: freed }], why: act.why, by: act.by ?? "", via: act.via ?? "",
|
|
});
|
|
return { deleted: consumer, freed_bytes: freed };
|
|
});
|
|
}
|
|
|
|
const tick = async (): Promise<void> => {
|
|
if (stopped) return;
|
|
try {
|
|
await serially(reconcile);
|
|
} catch (err) {
|
|
console.error(`[provisioner:${resource}] reconcile error: ${err}`);
|
|
}
|
|
if (!stopped) setTimeout(() => void tick(), everyMs);
|
|
};
|
|
void tick();
|
|
|
|
const stop = () => {
|
|
stopped = true;
|
|
};
|
|
const handle: ProviderLoop = Object.assign(stop, { retirement, approve, reject, deleteRetired });
|
|
if (opts.module) registerModuleTools(opts.module, () => providerLoopTools(handle));
|
|
return handle;
|
|
}
|
|
|
|
const strings = (v: unknown): string[] => (Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []);
|
|
const text = (v: unknown): string => (typeof v === "string" ? v : "");
|
|
|
|
/** The four tools every provider serves for retirement (novox/hq ADR 0230), the controller's `retire`
|
|
* and `cleanup` verbs asking them per machine. */
|
|
export function providerLoopTools(p: ProviderLoop): ToolDefinition[] {
|
|
const act = { type: "string" };
|
|
return [
|
|
{
|
|
name: "provisioner_retirement",
|
|
description: "This provider's retirement: the consumers it holds, a retirement waiting for a person, one a person rejected, and every consumer retired (access disabled, data kept) with when, why and size.",
|
|
input: {},
|
|
run: async () => p.retirement(),
|
|
},
|
|
{
|
|
name: "provisioner_retire_approve",
|
|
description: "Retire exactly the set waiting for a person (or rejected earlier). Repeat the set in consumers; why is required.",
|
|
input: { consumers: { type: "array", items: { type: "string" } }, why: act, by: act, via: act },
|
|
run: async (a) => p.approve(strings(a.consumers), { why: text(a.why), by: text(a.by), via: text(a.via) }),
|
|
},
|
|
{
|
|
name: "provisioner_retire_reject",
|
|
description: "Keep the set waiting for a person active, though the mesh no longer asks for it. Repeat the set in consumers; why is required.",
|
|
input: { consumers: { type: "array", items: { type: "string" } }, why: act, by: act, via: act },
|
|
run: async (a) => p.reject(strings(a.consumers), { why: text(a.why), by: text(a.by), via: text(a.via) }),
|
|
},
|
|
{
|
|
name: "provisioner_delete",
|
|
description: "Delete one RETIRED consumer and its data from this provider's backend — never an active one. Repeat its name in confirm; why is required.",
|
|
input: { consumer: act, confirm: act, why: act, by: act, via: act },
|
|
run: async (a) =>
|
|
p.deleteRetired(text(a.consumer), { why: text(a.why), by: text(a.by), via: text(a.via), confirm: text(a.confirm) }),
|
|
},
|
|
];
|
|
}
|
|
|
|
function iso(ms: number): string {
|
|
return new Date(ms).toISOString();
|
|
}
|
|
|
|
/**
|
|
* Read the mesh's contributions file for one resource: the consumers it asks for, or `null` when the
|
|
* file says nothing usable — absent, unreadable, not JSON, for another requirement, or without a
|
|
* `given` list. `null` is not "no consumer": the mesh always writes the file, and writes `given: []`
|
|
* when nobody asks, so only that makes every consumer unasked — retired by the rules of ADR 0230, never at once without a person.
|
|
*/
|
|
async function readContributions(
|
|
path: string,
|
|
resource: string,
|
|
warn: (why: string) => void,
|
|
): Promise<Contribution[] | null> {
|
|
let raw: string;
|
|
try {
|
|
raw = await readFile(path, "utf8");
|
|
} catch (err) {
|
|
warn(`contributions file unreadable (${path}): ${(err as Error).message}`);
|
|
return null;
|
|
}
|
|
let doc: { requirement?: string; given?: unknown };
|
|
try {
|
|
doc = JSON.parse(raw) as { requirement?: string; given?: unknown };
|
|
} catch (err) {
|
|
warn(`contributions file is not JSON (${path}): ${err}`);
|
|
return null;
|
|
}
|
|
if (doc.requirement && doc.requirement !== resource) {
|
|
warn(`${path} is for ${doc.requirement}, not ${resource}`);
|
|
return null;
|
|
}
|
|
if (!Array.isArray(doc.given)) {
|
|
warn(`${path} has no given list`);
|
|
return null;
|
|
}
|
|
warn("");
|
|
// A contribution with no `as` is not a credential grant (a module offering something on its own
|
|
// machine) — the provisioner has nothing to create for it.
|
|
return (doc.given as Contribution[]).filter((g) => g.as && g.secret);
|
|
}
|
|
|
|
/** An error's class read from its text — the words the backends the mesh runs use (novox/hq ADR
|
|
* 0224): the same words, in the same order, as the Go copy of this loop reads. */
|
|
export function classOf(text: string): string {
|
|
const t = text.toLowerCase();
|
|
const credentials = ["invalid_grant", "invalid user credentials", "password authentication failed",
|
|
"authentication failed", "unauthorized", " 401"];
|
|
if (credentials.some((w) => t.includes(w))) return CLASS_CREDENTIALS;
|
|
const unreachable = ["connection refused", "no such host", "i/o timeout", "deadline exceeded",
|
|
"connection reset", "network is unreachable", "no route to host", "eof",
|
|
// Node's own words for the same faults.
|
|
"econnrefused", "enotfound", "etimedout", "econnreset", "ehostunreach", "enetunreach", "eai_again"];
|
|
if (unreachable.some((w) => t.includes(w))) return CLASS_UNREACHABLE;
|
|
return CLASS_REFUSED;
|
|
}
|
|
|
|
/** An announced error kept to what belongs in a status line. */
|
|
function clip(text: string): string {
|
|
const most = 300;
|
|
return text.length <= most ? text : `${text.slice(0, most)}…`;
|
|
}
|
|
|
|
/** A check that did not answer in time: the backend, not the consumer, is the likely cause. */
|
|
class HoldsTimeout extends Error {}
|
|
|
|
/** Reject with a HoldsTimeout if `p` has not settled within `ms`. */
|
|
function withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
|
|
let timer: NodeJS.Timeout | undefined;
|
|
const timeout = new Promise<never>((_, reject) => {
|
|
timer = setTimeout(() => reject(new HoldsTimeout(`no answer within ${ms}ms`)), ms);
|
|
});
|
|
return Promise.race([p, timeout]).finally(() => clearTimeout(timer));
|
|
}
|
|
|
|
/** An error as text with the consumer's password removed, raw and URL-encoded: a failed command's
|
|
* message can carry its arguments, and this log is not a place a password may appear. */
|
|
function scrub(err: unknown, password: string): string {
|
|
let text = String(err);
|
|
if (!password) return text;
|
|
for (const form of new Set([password, encodeURIComponent(password)])) text = text.split(form).join("***");
|
|
return text;
|
|
}
|
|
|
|
function hash(
|
|
as: string,
|
|
password: string,
|
|
values: Readonly<Record<string, unknown>>,
|
|
derived: Readonly<Record<string, unknown>>,
|
|
): string {
|
|
// The derivation is in the hash because it can change without the login or the password
|
|
// changing: a provider that renames what it derives has given this consumer a different
|
|
// resource, and an adapter trusted from memory would never be told (novox/hq ADR 0201).
|
|
return JSON.stringify([as, password, values, derived]);
|
|
}
|
|
|
|
function envOrThrow(name: string): string {
|
|
const v = process.env[name];
|
|
if (!v) throw new Error(`${name} is not set — the provisioner cannot run without it`);
|
|
return v;
|
|
}
|
|
|
|
/** @deprecated The provider loop's name before design 57: {@link runProviderLoop}. */
|
|
export const runProvisioner = runProviderLoop;
|
|
/** @deprecated {@link providerLoopTools}. */
|
|
export const provisionerTools = providerLoopTools;
|
|
/** @deprecated {@link ProviderLoop}. */
|
|
export type Provisioner = ProviderLoop;
|
|
/** @deprecated {@link ProviderLoopOptions}. */
|
|
export type ProvisionerOptions = ProviderLoopOptions;
|