Provisioner asks the backend, not memory, whether a consumer is still there

An optional holds() on the adapter is asked for every applied consumer
every minute; false applies it again. A backend that forgets what it was
given while the provisioner runs (hq issue 120) is healed within a
minute instead of failing its consumers in silence. Unable to ask is not
treated as loss. Adapters without holds() behave as before.

test/sdk.test.ts

@@ -74,6 +74,99 @@ test("provisioner creates each consumer with the mesh's login and password, remo
   stop();
 });
 
+test("provisioner applies again what the backend no longer holds, and trusts memory without holds", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "prov-holds-"));
+  await writeFile(join(dir, "webapp.secret"), "minted-pw\n");
+  const receives = join(dir, "cache.json");
+  await writeFile(
+    receives,
+    JSON.stringify({ contributions: 1, requirement: "cache", given: [{ from: "webapp", node: "anchor", as: "webapp-anchor", secret: join(dir, "webapp.secret") }] }),
+  );
+
+  // A backend that forgets: what was created is held until it "restarts".
+  const backend = new Set<string>();
+  let creates = 0;
+  let asked = 0;
+  const stop = runProvisioner(
+    "cache",
+    {
+      async create(p) {
+        creates++;
+        backend.add(`${p.as}:${p.password}`);
+      },
+      async remove(p) {
+        for (const k of backend) if (k.startsWith(`${p.as}:`)) backend.delete(k);
+      },
+      async holds(p) {
+        asked++;
+        return backend.has(`${p.as}:${p.password}`);
+      },
+    },
+    { receives, everyMs: 10, verifyEveryMs: 30 },
+  );
+
+  await waitFor(() => creates === 1, 2000);
+  // While the backend holds it, asking changes nothing: no second create.
+  await waitFor(() => asked >= 2, 2000);
+  assert.equal(creates, 1);
+
+  // The backend restarts and forgets. The contributions did not change; only asking can notice.
+  backend.clear();
+  await waitFor(() => creates === 2, 2000);
+  assert.ok(backend.has("webapp-anchor:minted-pw"));
+  stop();
+
+  // An adapter without holds is trusted from memory, as before: a forgotten backend stays forgotten.
+  const forgetful = new Set<string>();
+  let plainCreates = 0;
+  const stopPlain = runProvisioner(
+    "cache",
+    {
+      async create(p) {
+        plainCreates++;
+        forgetful.add(p.as);
+      },
+      async remove() {},
+    },
+    { receives, everyMs: 10, verifyEveryMs: 10 },
+  );
+  await waitFor(() => plainCreates === 1, 2000);
+  forgetful.clear();
+  await new Promise((r) => setTimeout(r, 100));
+  assert.equal(plainCreates, 1);
+  stopPlain();
+});
+
+test("provisioner keeps a consumer applied when the backend cannot be asked", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "prov-unreachable-"));
+  await writeFile(join(dir, "webapp.secret"), "minted-pw");
+  const receives = join(dir, "cache.json");
+  await writeFile(
+    receives,
+    JSON.stringify({ requirement: "cache", given: [{ as: "webapp-anchor", secret: join(dir, "webapp.secret") }] }),
+  );
+  let creates = 0;
+  let asked = 0;
+  const stop = runProvisioner(
+    "cache",
+    {
+      async create() {
+        creates++;
+      },
+      async remove() {},
+      async holds() {
+        asked++;
+        throw new Error("connection refused");
+      },
+    },
+    { receives, everyMs: 10, verifyEveryMs: 20 },
+  );
+  await waitFor(() => asked >= 3, 2000);
+  // Unable to ask is not evidence of loss: nothing is applied again.
+  assert.equal(creates, 1);
+  stop();
+});
+
 test("modules can SERVE: a real async tool, loaded and invoked over the broker", async () => {
   resetTools();