Provisioner asks the backend, not memory, whether a consumer is still there

An optional holds() on the adapter is asked for every applied consumer
every minute; false applies it again. A backend that forgets what it was
given while the provisioner runs (hq issue 120) is healed within a
minute instead of failing its consumers in silence. Unable to ask is not
treated as loss. Adapters without holds() behave as before.

src/provisioner/index.ts

@@ -35,6 +35,13 @@ export interface Provision {
 export interface Adapter {
   create(p: Provision): Promise<void>;
   remove(p: { readonly as: string }): Promise<void>;
+  /** Optional: whether the backend still holds this consumer's credential exactly as `p` says.
+   *  Asked of every consumer already applied, every `verifyEveryMs`. `false` makes the harness apply
+   *  it again on the same pass, so a backend that lost what it was given (a server restarted
+   *  without persisting its users, a restore, a login removed by hand) is provisioned again instead
+   *  of being trusted from memory (novox/hq issue 120). An adapter without it is trusted from memory,
+   *  as before. It must only read: it is asked often, and must never change the backend. */
+  holds?(p: Provision): Promise<boolean>;
 }
 
 export interface ProvisionerOptions {
@@ -43,6 +50,10 @@ export interface ProvisionerOptions {
   receives?: string;
   /** Reconcile interval in ms. Defaults to 5000. */
   everyMs?: number;
+  /** How often, in ms, an adapter with `holds` is asked whether the backend still holds each
+   *  applied consumer. Defaults to 60000: slower than reconciling, because it reads the backend for
+   *  every consumer, and fast enough that a lost login is back within a minute. */
+  verifyEveryMs?: number;
 }
 
 /** One entry in the mesh's contributions file: a consumer the provider must serve. */
@@ -62,6 +73,8 @@ interface Contribution {
 export function runProvisioner(resource: string, adapter: Adapter, opts: ProvisionerOptions = {}): () => void {
   const receives = opts.receives ?? envOrThrow("MESH_RECEIVES");
   const everyMs = opts.everyMs ?? 5000;
+  const verifyEveryMs = opts.verifyEveryMs ?? 60_000;
+  let verifiedAt = 0;
 
   const applied = new Map<string, string>(); // login (`as`) -> hash of what was last applied
   let stopped = false;
@@ -69,6 +82,9 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi
   async function reconcile(): Promise<void> {
     const given = await readContributions(receives, resource);
     const wantByAs = new Map(given.map((g) => [g.as, g]));
+    // On this pass, ask the backend rather than memory whether each applied consumer is still there.
+    const verifying = adapter.holds !== undefined && Date.now() - verifiedAt >= verifyEveryMs;
+    if (verifying) verifiedAt = Date.now();
 
     // Create or update every consumer whose login, password or values changed.
     for (const g of given) {
@@ -85,9 +101,21 @@ export function runProvisioner(resource: string, adapter: Adapter, opts: Provisi
         continue;
       }
       const h = hash(g.as, password, g.values ?? {});
-      if (applied.get(g.as) === h) continue;
+      const p: Provision = { as: g.as, password, values: g.values ?? {}, at: g.at, consumer: g.node };
+      if (applied.get(g.as) === h) {
+        if (!verifying) continue;
+        try {
+          if (await adapter.holds!(p)) continue;
+          // Said, because it means the backend lost something while nothing was looking.
+          console.error(`[provisioner:${resource}] ${g.as}: the backend no longer holds it; applying again`);
+        } catch (err) {
+          // Unable to ask is not evidence of loss. Kept as applied, asked again next time.
+          console.error(`[provisioner:${resource}] ${g.as}: could not check the backend, will ask again: ${err}`);
+          continue;
+        }
+      }
       try {
-        await adapter.create({ as: g.as, password, values: g.values ?? {}, at: g.at, consumer: g.node });
+        await adapter.create(p);
         applied.set(g.as, h);
       } catch (err) {
         console.error(`[provisioner:${resource}] ${g.as}: create failed, will retry: ${err}`);