Merge pull request 'Every credential provider says whether it still holds a consumer (hq issue 120)' (#84) from fix/120-redis-says-what-it-holds into main

modules/gitea/client.ts

@@ -390,7 +390,7 @@ export class GiteaAdmin {
   }
 
   private async findTeam(org: string, team: string): Promise<number | null> {
-    const res = await this.request(`/orgs/${encodeURIComponent(org)}/teams`);
+    const res = await this.request(`/orgs/${encodeURIComponent(org)}/teams?limit=50`);
     if (res.status !== 200) return null;
     const match = (res.body as any[] | null)?.find((t) => t?.name === team);
     return match ? Number(match.id) : null;
@@ -414,7 +414,9 @@ export class GiteaAdmin {
         const patch = await this.request(`/admin/users/${encodeURIComponent(username)}`, {
           method: "PATCH",
           // login_name is required by the admin edit endpoint; for a local user it is the username.
-          body: JSON.stringify({ login_name: username, password, must_change_password: false }),
+          // active and prohibit_login: a deactivated or login-prohibited user is refused like a wrong
+          // password, so the provisioner's check reports it lost; applying again must undo both.
+          body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false }),
         });
         if (patch.status === 200) return;
         GiteaAdmin.fail(`/admin/users/${username}`, patch);
@@ -433,6 +435,29 @@ export class GiteaAdmin {
     GiteaAdmin.fail(`/teams/${teamId}/members/${username}`, res);
   }
 
+  /**
+   * Whether a consumer's user logs in with exactly this password and is still a member of the
+   * package team. Read-only: the password is checked as the consumer presents it, basic auth on the
+   * API, and membership through the admin API. `false` for a refused login or a missing member; any
+   * other answer rejects (novox/hq issue 120).
+   */
+  async holdsTeamMember(org: string, team: string, username: string, password: string): Promise<boolean> {
+    const me = await fetch(`${this.baseUrl}/api/v1/user`, {
+      headers: { Authorization: "Basic " + Buffer.from(`${username}:${password}`).toString("base64") },
+    });
+    if (me.status === 401 || me.status === 403) return false;
+    if (me.status !== 200) throw new Error(`Gitea GET /user as ${username}: ${me.status}`);
+    const teams = await this.request(`/orgs/${encodeURIComponent(org)}/teams?limit=50`);
+    if (teams.status === 404) return false;
+    if (teams.status !== 200) GiteaAdmin.fail(`/orgs/${org}/teams`, teams);
+    const found = (teams.body as { id: number; name: string }[]).find((t) => t.name === team);
+    if (!found) return false;
+    const member = await this.request(`/teams/${found.id}/members/${encodeURIComponent(username)}`);
+    if (member.status === 200 || member.status === 204) return true;
+    if (member.status === 404) return false;
+    GiteaAdmin.fail(`/teams/${found.id}/members/${username}`, member);
+  }
+
   /** Delete a user, purging what they own. A 404 means the mesh already withdrew them — success, not
    *  an error, so a re-run of remove is safe. */
   async deleteUser(username: string): Promise<void> {

modules/gitea/package.json

@@ -9,7 +9,7 @@
     "test": "npm run build && node --test --experimental-strip-types 'test/*.test.ts'"
   },
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/gitea/provisioner/index.ts

@@ -46,4 +46,9 @@ runProvisioner("package-registry", {
   async remove(p: { as: string }): Promise<void> {
     await gitea.deleteUser(p.as);
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return gitea.holdsTeamMember(ORG, PACKAGE_TEAM, p.as, p.password);
+  },
 });

modules/lavinmq/client.ts

@@ -94,6 +94,39 @@ export class LavinmqClient {
     await this.api("PUT", `/permissions/${v}/${u}`, { configure: ".*", write: ".*", read: ".*" });
   }
 
+  /**
+   * Whether a consumer's user exists with exactly this password and full permissions on its own
+   * vhost. Read-only: the stored hash is salted SHA-256, the scheme `rabbitHash` writes, so the
+   * password is checked by hashing it with the stored salt rather than by logging in. `false` when
+   * the user or its permission is gone or the password differs; an unreachable API rejects
+   * (novox/hq issue 120).
+   */
+  async holdsConsumer(login: string, password: string): Promise<boolean> {
+    const v = encodeURIComponent(login);
+    const u = encodeURIComponent(login);
+    const user = await this.getOrNull<{ password_hash?: string; hashing_algorithm?: string }>(`/users/${u}`);
+    if (!user?.password_hash) return false;
+    if (user.hashing_algorithm && !/sha256/i.test(user.hashing_algorithm)) {
+      throw new Error(`lavinmq user ${login} is hashed with ${user.hashing_algorithm}, which this check cannot verify`);
+    }
+    const stored = Buffer.from(user.password_hash, "base64");
+    if (stored.length < 5 || rabbitHash(password, stored.subarray(0, 4)) !== user.password_hash) return false;
+    const perm = await this.getOrNull<{ configure?: string; write?: string; read?: string }>(`/permissions/${v}/${u}`);
+    return perm?.configure === ".*" && perm?.write === ".*" && perm?.read === ".*";
+  }
+
+  /** A GET that answers null for a 404 and rejects on anything else that is not 2xx. */
+  private async getOrNull<T>(path: string): Promise<T | null> {
+    const resp = await fetch(`${this.conn.base}/api${path}`, {
+      headers: {
+        Authorization: "Basic " + Buffer.from(`${this.conn.adminUser}:${this.conn.adminPassword}`).toString("base64"),
+      },
+    });
+    if (resp.status === 404) return null;
+    if (!resp.ok) throw new Error(`lavinmq management API GET ${path} -> ${resp.status}: ${await resp.text()}`);
+    return (await resp.json()) as T;
+  }
+
   /** Remove a consumer's vhost and user, idempotently. A DELETE of what is already gone is tolerated. */
   async removeConsumer(login: string): Promise<void> {
     const v = encodeURIComponent(login);

modules/lavinmq/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/lavinmq/provisioner/index.ts

@@ -48,4 +48,9 @@ runProvisioner("amqp", {
     await lavinmq.removeConsumer(p.as);
     await announce("module.lavinmq.amqp.deprovisioned", { user: p.as, vhost: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return lavinmq.holdsConsumer(p.as, p.password);
+  },
 });

modules/mailu/client.ts

@@ -130,10 +130,39 @@ export class MailuClient {
     await this.api("PATCH", `/user/${encodeURIComponent(email)}`, { raw_password: password });
   }
 
+  /**
+   * Set the mesh's password on a mailbox the mesh provisions, and enable it. A disabled mailbox is
+   * what the provisioner's check reports as lost, so applying again must enable it, or the two would
+   * disagree for ever. Separate from changePassword, which an operator's tool uses and which must
+   * not re-enable a mailbox someone disabled.
+   */
+  async applyProvisioned(email: string, password: string): Promise<void> {
+    await this.api("PATCH", `/user/${encodeURIComponent(email)}`, { raw_password: password, enabled: true });
+  }
+
   async deleteUser(email: string): Promise<void> {
     await this.api("DELETE", `/user/${encodeURIComponent(email)}`);
   }
 
+  /**
+   * Whether a mailbox exists and is enabled. Read-only, through the admin API.
+   *
+   * **The password is not checked.** Mailu authenticates in its admin service, behind the front;
+   * the imap server's own password database accepts any password from Mailu's subnet, so asking it
+   * (`doveadm auth test`) proves nothing, or refuses everyone. A lost or disabled mailbox is caught;
+   * a password changed by hand is not (novox/hq issue 120).
+   */
+  async holdsUser(email: string): Promise<boolean> {
+    const res = await fetch(`${this.baseUrl}/user/${encodeURIComponent(email)}`, {
+      headers: { Authorization: this.apiKey, Accept: "application/json" },
+    });
+    if (res.status === 404) return false;
+    if (!res.ok) throw new Error(`Mailu API GET /user/${email}: ${res.status} ${await res.text()}`);
+    const user = (await res.json()) as { enabled?: boolean };
+    return user.enabled !== false;
+  }
+
+
   async listAliases(): Promise<MailuAlias[]> {
     const aliases = await this.api<any[]>("GET", "/alias");
     return (aliases ?? []).map((a) => ({

modules/mailu/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/mailu/provisioner/index.ts

@@ -48,7 +48,7 @@ runProvisioner("smtp", {
     try {
       await mailu.createUser(email, p.password);
     } catch {
-      await mailu.changePassword(email, p.password);
+      await mailu.applyProvisioned(email, p.password);
     }
   },
 
@@ -62,4 +62,9 @@ runProvisioner("smtp", {
     // named-account consumer is an operator action until the harness carries values here.
     await mailu.deleteUser(`${p.as}@${domain()}`).catch(() => {});
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return mailu.holdsUser(addressOf(p));
+  },
 });

modules/minio/client.ts

@@ -125,6 +125,18 @@ export class MinioClient {
     throw new Error(`minio bucketExists ${bucket}: ${status}`);
   }
 
+  /**
+   * Whether a consumer's access key, with exactly this secret, reaches its bucket: a HEAD of the
+   * bucket signed as the consumer, the way it signs. Read-only. `false` when the key is unknown, the
+   * secret wrong, access denied or the bucket gone; any other answer rejects (novox/hq issue 120).
+   */
+  async canReachAs(bucket: string, accessKey: string, secretKey: string): Promise<boolean> {
+    const { status } = await this.request("HEAD", `/${bucket}`, {}, { accessKey, secretKey });
+    if (status === 200) return true;
+    if (status === 403 || status === 404) return false;
+    throw new Error(`minio HEAD ${bucket} as ${accessKey}: ${status}`);
+  }
+
   async createBucket(bucket: string): Promise<void> {
     const { status, text } = await this.request("PUT", `/${bucket}`);
     // 200 created; 409 BucketAlreadyOwnedByYou — idempotent, a re-provision must not fail.
@@ -251,6 +263,7 @@ export class MinioClient {
     method: string,
     path: string,
     query: Record<string, string> = {},
+    as: { accessKey: string; secretKey: string } = { accessKey: this.rootUser, secretKey: this.rootPassword },
   ): Promise<{ status: number; headers: Headers; text: string }> {
     const { amzDate, dateStamp } = this.stamp();
     const host = new URL(this.baseUrl).host;
@@ -262,8 +275,8 @@ export class MinioClient {
     const canonicalRequest = [method, encodedPath, canonicalQuery, canonicalHeaders, signedHeaders, payloadHash].join("\n");
     const scope = `${dateStamp}/${this.region}/s3/aws4_request`;
     const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256hex(canonicalRequest)].join("\n");
-    const signature = hmac(this.signingKey(dateStamp), stringToSign).toString("hex");
-    const authorization = `AWS4-HMAC-SHA256 Credential=${this.rootUser}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
+    const signature = hmac(this.signingKey(dateStamp, as.secretKey), stringToSign).toString("hex");
+    const authorization = `AWS4-HMAC-SHA256 Credential=${as.accessKey}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
 
     const url = `${this.baseUrl}${encodedPath}${canonicalQuery ? `?${canonicalQuery}` : ""}`;
     const res = await fetch(url, {
@@ -275,8 +288,8 @@ export class MinioClient {
     return { status: res.status, headers: res.headers, text };
   }
 
-  private signingKey(dateStamp: string): Buffer {
-    const kDate = hmac(`AWS4${this.rootPassword}`, dateStamp);
+  private signingKey(dateStamp: string, secretKey: string = this.rootPassword): Buffer {
+    const kDate = hmac(`AWS4${secretKey}`, dateStamp);
     const kRegion = hmac(kDate, this.region);
     const kService = hmac(kRegion, "s3");
     return hmac(kService, "aws4_request");

modules/minio/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/minio/provisioner/index.ts

@@ -53,6 +53,12 @@ runProvisioner("s3-bucket", {
 
     await announce("module.minio.bucket.removed", { bucket, accessKey: p.as });
   },
+
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return minio.canReachAs(bucketFor(p.as), p.as, p.password);
+  },
 });
 
 /** Emit best-effort: a broker hiccup is logged and dropped, never allowed to throw back and fail a

modules/mongodb/client.ts

@@ -109,6 +109,34 @@ print(EJSON.stringify({ ok: 1 }));
     await this.evalJs<{ ok: number }>(js);
   }
 
+  /**
+   * Whether `user` authenticates against `database` with exactly `password` and holds `dbOwner`
+   * there: checked by connecting as the consumer, the way it connects. Read-only. `false` only on an
+   * authentication failure or a missing role; an unreachable server rejects (novox/hq issue 120).
+   */
+  async canAuthenticateAs(database: string, user: string, password: string): Promise<boolean> {
+    // Connected without credentials, then authenticated inside the eval from the environment, so
+    // the consumer's password is neither on argv nor in the message of a failed command.
+    const uri = `mongodb://${this.conn.host}:${this.conn.port}/?serverSelectionTimeoutMS=10000`;
+    const js =
+      "const t = db.getSiblingDB(process.env.MESH_HOLDS_DB);" +
+      "t.auth(process.env.MESH_HOLDS_USER, process.env.MESH_HOLDS_PW);" +
+      "print(EJSON.stringify(t.runCommand({ connectionStatus: 1 }).authInfo.authenticatedUserRoles))";
+    let stdout: string;
+    try {
+      ({ stdout } = await run("mongosh", [uri, "--quiet", "--eval", js], {
+        env: { ...process.env, MESH_HOLDS_DB: database, MESH_HOLDS_USER: user, MESH_HOLDS_PW: password },
+        timeout: 30_000,
+      }));
+    } catch (err) {
+      const text = `${(err as { stderr?: string }).stderr ?? ""}${(err as { stdout?: string }).stdout ?? ""}`;
+      if (/Authentication failed|AuthenticationFailed/i.test(text)) return false;
+      throw new Error(`mongosh could not check ${user}: ${text.trim().slice(0, 500) || String((err as Error).message).split("\n")[0]}`);
+    }
+    const roles = JSON.parse(stdout.trim()) as { role: string; db: string }[];
+    return roles.some((r) => r.role === "dbOwner" && r.db === database);
+  }
+
   /** Drop a database and its owning user, idempotently. Dropping the database evicts its data; the
    *  user is removed first so a re-grant of the same login starts clean. */
   async dropDatabaseAndUser(database: string, user: string): Promise<void> {

modules/mongodb/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/mongodb/provisioner/index.ts

@@ -45,4 +45,9 @@ runProvisioner("mongodb-database", {
     await mongo.dropDatabaseAndUser(p.as, p.as);
     await announce("module.mongodb.database.deprovisioned", { database: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return mongo.canAuthenticateAs(p.as, p.as, p.password);
+  },
 });

modules/mosquitto/client.ts

@@ -15,6 +15,7 @@
 // The one cost dynsec carries is the bootstrap file; see initBootstrapFile() and the module README.
 
 import { randomBytes } from "node:crypto";
+import { connect as tcpConnect } from "node:net";
 import { readFileSync } from "node:fs";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
@@ -87,9 +88,20 @@ export class MosquittoClient {
       "-u", this.conn.adminUser,
       "-P", this.conn.adminPassword,
     ];
-    const { stdout, stderr } = await run("mosquitto_ctrl", [...base, "dynsec", ...args], {
-      maxBuffer: 16 << 20,
-    });
+    let stdout: string;
+    let stderr: string;
+    try {
+      ({ stdout, stderr } = await run("mosquitto_ctrl", [...base, "dynsec", ...args], {
+        maxBuffer: 16 << 20,
+        timeout: 30_000,
+      }));
+    } catch (err) {
+      // A failed run's message repeats its argv, the admin password (-P) included; say what failed
+      // without it.
+      const e = err as { code?: unknown; signal?: unknown; stderr?: string; stdout?: string };
+      const detail = `${e.stderr ?? ""}${e.stdout ?? ""}`.trim().slice(0, 500);
+      throw new Error(`mosquitto_ctrl dynsec ${args[0] ?? ""} could not run (${e.code ?? e.signal ?? "error"}): ${detail}`);
+    }
     const failure = ctlError(`${stdout}\n${stderr}`);
     if (failure) {
       throw new Error(`mosquitto_ctrl dynsec ${args[0] ?? ""} failed: ${failure}`);
@@ -140,6 +152,11 @@ export class MosquittoClient {
 
     if (await this.clientExists(username)) {
       await this.ctl("setClientPassword", username, password);
+      // A disabled client is refused like a wrong password, so the check the provisioner runs
+      // reports it lost; applying again must enable it, or the two would disagree for ever.
+      if (/Disabled:\s*true/i.test(await this.ctl("getClient", username))) {
+        await this.ctl("enableClient", username);
+      }
     } else {
       await this.ctl("createClient", username, "-p", password);
     }
@@ -163,6 +180,28 @@ export class MosquittoClient {
     }
   }
 
+  /**
+   * Whether a consumer's client accepts exactly this password and still carries its own role.
+   * Read-only. The password is checked the way the consumer is checked, by an MQTT CONNECT as it,
+   * and the broker's CONNACK code is the answer: 0 accepted, 4 bad credentials, 5 not authorised.
+   * Nothing rides on argv. An unreachable broker rejects (novox/hq issue 120).
+   */
+  async holdsClient(username: string, password: string): Promise<boolean> {
+    const code = await mqttConnack(this.conn.host, this.conn.port, username, password);
+    if (code === 4 || code === 5) return false;
+    if (code !== 0) throw new Error(`mosquitto refused ${username} with CONNACK ${code}`);
+    // The role, asked directly: only "not found" means absent. Any other failure to ask rejects,
+    // unlike clientHasRole, which reads every failure as "no role".
+    let out: string;
+    try {
+      out = await this.ctl("getClient", username);
+    } catch (err) {
+      if (/not\s*found|does not exist|no such/i.test(String(err))) return false;
+      throw err;
+    }
+    return new RegExp(`(^|\\s)${escapeRegExp(username)}\\s+\\(priority`, "m").test(out);
+  }
+
   /** Remove a client and the per-client role created for it, idempotently. */
   async deleteScopedClient(username: string): Promise<void> {
     await ignoreMissing(this.ctl("deleteClient", username));
@@ -249,3 +288,55 @@ function readSecretFile(path: string | undefined): string | undefined {
     return undefined;
   }
 }
+
+/**
+ * Connect once over MQTT 3.1.1 with a username and password, return the broker's CONNACK return code,
+ * and disconnect. A clean session under a throwaway client id, so no consumer session is taken over.
+ */
+function mqttConnack(host: string, port: number, username: string, password: string): Promise<number> {
+  const str = (v: string): Buffer => {
+    const b = Buffer.from(v, "utf8");
+    const len = Buffer.alloc(2);
+    len.writeUInt16BE(b.length);
+    return Buffer.concat([len, b]);
+  };
+  const variable = Buffer.concat([str("MQTT"), Buffer.from([4, 0xc2, 0, 10])]); // level 4; user+pass+clean; keepalive 10s
+  const payload = Buffer.concat([str(`mesh-holds-${randomBytes(6).toString("hex")}`), str(username), str(password)]);
+  let remaining = variable.length + payload.length;
+  const lenBytes: number[] = [];
+  do {
+    let byte = remaining % 128;
+    remaining = Math.floor(remaining / 128);
+    if (remaining > 0) byte |= 0x80;
+    lenBytes.push(byte);
+  } while (remaining > 0);
+  const packet = Buffer.concat([Buffer.from([0x10, ...lenBytes]), variable, payload]);
+
+  return new Promise((resolve, reject) => {
+    const socket = tcpConnect({ host, port });
+    let buf = Buffer.alloc(0);
+    const timer = setTimeout(() => {
+      socket.destroy();
+      reject(new Error(`no CONNACK from ${host}:${port} within 10s`));
+    }, 10_000);
+    socket.on("connect", () => socket.write(packet));
+    socket.on("data", (chunk) => {
+      buf = Buffer.concat([buf, chunk]);
+      if (buf.length < 4) return;
+      clearTimeout(timer);
+      if (buf[0] !== 0x20) {
+        socket.destroy();
+        reject(new Error(`unexpected MQTT packet 0x${buf[0].toString(16)} instead of CONNACK`));
+        return;
+      }
+      const code = buf[3];
+      if (code === 0) socket.end(Buffer.from([0xe0, 0])); // DISCONNECT
+      else socket.destroy();
+      resolve(code);
+    });
+    socket.on("error", (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+  });
+}

modules/mosquitto/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/mosquitto/provisioner/index.ts

@@ -43,4 +43,9 @@ runProvisioner("mqtt-topic", {
     await mosquitto.deleteScopedClient(p.as);
     await announce("module.mosquitto.topic.deprovisioned", { username: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return mosquitto.holdsClient(p.as, p.password);
+  },
 });

modules/mssql/client.ts

@@ -75,14 +75,18 @@ export class MssqlClient {
    * prints (split across output lines for a large result, and reassembled here) is parsed. An
    * empty result yields no output at all — an empty array.
    */
-  async query(select: string, database = "master"): Promise<Record<string, unknown>[]> {
+  async query(
+    select: string,
+    database = "master",
+    variables: Record<string, string> = {},
+  ): Promise<Record<string, unknown>[]> {
     const wrapped = `SET NOCOUNT ON;\n${stripTrailingSemis(select)}\nFOR JSON PATH, INCLUDE_NULL_VALUES;`;
-    const stdout = await this.sqlcmd(wrapped, database);
+    const stdout = await this.sqlcmd(wrapped, database, variables);
     return parseJsonRows(stdout);
   }
 
   /** The one execution boundary: invoke `sqlcmd` and return its concatenated stdout. */
-  private async sqlcmd(sql: string, database: string): Promise<string> {
+  private async sqlcmd(sql: string, database: string, variables: Record<string, string> = {}): Promise<string> {
     // `-h -1` drops the column-header rule; `-y 0`/`-Y 0` lift the display-width cap so a long
     // JSON document is not truncated; `-W` trims trailing whitespace so the JSON chunks rejoin
     // cleanly. sqlcmd from the mssql-tools ships in the runtime container, the way `psql` ships
@@ -101,7 +105,9 @@ export class MssqlClient {
         "-W",
         "-Q", sql,
       ],
-      { env: { ...process.env, SQLCMDPASSWORD: this.conn.password }, maxBuffer: 16 << 20 },
+      // `variables` reach sqlcmd as environment variables, which it substitutes as `$(NAME)` scripting
+      // variables: a value that must not appear on argv, or in the message of a failed command.
+      { env: { ...process.env, ...variables, SQLCMDPASSWORD: this.conn.password }, maxBuffer: 16 << 20 },
     );
     return stdout;
   }
@@ -121,6 +127,9 @@ export class MssqlClient {
       );
     } else {
       await this.exec(`ALTER LOGIN ${ident(login)} WITH PASSWORD = ${literal(password)}`);
+      // A disabled login is refused like a wrong password; the check the provisioner runs reports it
+      // lost, so applying again must enable it or the two would disagree for ever.
+      await this.exec(`ALTER LOGIN ${ident(login)} ENABLE`);
     }
 
     const dbs = await this.query(
@@ -138,10 +147,51 @@ export class MssqlClient {
     );
     if (users.length === 0) {
       await this.exec(`CREATE USER ${ident(login)} FOR LOGIN ${ident(login)}`, database);
+    } else {
+      // Re-point an existing user at the login when its SID is not the login's: a database restored
+      // from elsewhere keeps its user under the old login's SID, orphaned. Only then, so a user that
+      // is already mapped is left alone.
+      const orphaned = await this.query(
+        `SELECT 1 AS ok FROM sys.database_principals WHERE name = ${literal(login)} ` +
+          `AND (sid IS NULL OR sid <> SUSER_SID(${literal(login)}))`,
+        database,
+      );
+      if (orphaned.length > 0) {
+        await this.exec(`ALTER USER ${ident(login)} WITH LOGIN = ${ident(login)}`, database);
+      }
     }
     await this.exec(`ALTER ROLE db_owner ADD MEMBER ${ident(login)}`, database);
   }
 
+  /**
+   * Whether `login` exists, is enabled, has exactly `password`, and is a db_owner user of
+   * `database`. Read-only: the password is compared with PWDCOMPARE against the stored hash, so
+   * nothing logs in and no failed-login is recorded (novox/hq issue 120).
+   */
+  async holdsLogin(database: string, login: string, password: string): Promise<boolean> {
+    // The password reaches sqlcmd as a scripting variable from the environment, never inside the
+    // query text, so it is neither on argv nor in the message of a failed command. It is the mesh's
+    // minted value, which carries no quote.
+    const server = await this.query(
+      `SELECT CAST(CASE WHEN EXISTS (SELECT 1 FROM sys.sql_logins WHERE name = ${literal(login)} ` +
+        `AND is_disabled = 0 AND PWDCOMPARE(N'$(MESHHOLDSPW)', password_hash) = 1) ` +
+        `AND DB_ID(${literal(database)}) IS NOT NULL THEN 1 ELSE 0 END AS int) AS ok`,
+      "master",
+      { MESHHOLDSPW: password },
+    );
+    if (Number(server[0]?.ok) !== 1) return false;
+    // The user must be this login's, by SID, and a db_owner. A user orphaned by a restore has the
+    // right name and the wrong SID, and cannot be reached through the login.
+    const owner = await this.query(
+      `SELECT CAST(CASE WHEN EXISTS (SELECT 1 FROM sys.database_principals dp ` +
+        `JOIN sys.server_principals sp ON dp.sid = sp.sid ` +
+        `WHERE dp.name = ${literal(login)} AND sp.name = ${literal(login)}) ` +
+        `AND IS_ROLEMEMBER('db_owner', ${literal(login)}) = 1 THEN 1 ELSE 0 END AS int) AS ok`,
+      database,
+    );
+    return Number(owner[0]?.ok) === 1;
+  }
+
   /** Drop a database and its login, idempotently, after evicting live connections. */
   async dropDatabaseAndLogin(database: string, login: string): Promise<void> {
     const dbs = await this.query(

modules/mssql/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/mssql/provisioner/index.ts

@@ -44,4 +44,9 @@ runProvisioner("mssql-database", {
     await mssql.dropDatabaseAndLogin(p.as, p.as);
     await announce("module.mssql.database.deprovisioned", { database: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return mssql.holdsLogin(p.as, p.as, p.password);
+  },
 });

modules/postgres/client.ts

@@ -88,9 +88,11 @@ export class PostgresClient {
   async createDatabaseAndRole(database: string, role: string, password: string): Promise<void> {
     const roles = await this.query("SELECT 1 FROM pg_roles WHERE rolname = " + literal(role));
     if (roles.rows.length === 0) {
-      await this.query(`CREATE ROLE ${ident(role)} WITH LOGIN PASSWORD ${literal(password)}`);
+      await this.query(`CREATE ROLE ${ident(role)} WITH LOGIN PASSWORD ${literal(password)} VALID UNTIL 'infinity'`);
     } else {
-      await this.query(`ALTER ROLE ${ident(role)} WITH LOGIN PASSWORD ${literal(password)}`);
+      // VALID UNTIL 'infinity': a password that expired is refused like a wrong one, so the check the
+      // provisioner runs would report it lost, and only clearing the expiry makes applying it again work.
+      await this.query(`ALTER ROLE ${ident(role)} WITH LOGIN PASSWORD ${literal(password)} VALID UNTIL 'infinity'`);
     }
     const dbs = await this.query("SELECT 1 FROM pg_database WHERE datname = " + literal(database));
     if (dbs.rows.length === 0) {
@@ -99,6 +101,30 @@ export class PostgresClient {
     await this.query(`GRANT ALL PRIVILEGES ON DATABASE ${ident(database)} TO ${ident(role)}`);
   }
 
+  /**
+   * Whether `role` can log in to `database` with exactly `password`: the consumer's own view of its
+   * credential, checked by connecting as it. Read-only. `false` only when the server says so (the
+   * role, the password or the database is wrong or gone); an unreachable server rejects instead,
+   * because being unable to ask is not evidence of loss (novox/hq issue 120).
+   */
+  async canConnectAs(database: string, role: string, password: string): Promise<boolean> {
+    try {
+      await run(
+        "psql",
+        ["-h", this.conn.host, "-p", String(this.conn.port), "-U", role, "-d", database,
+          "-v", "ON_ERROR_STOP=1", "--no-psqlrc", "-tAc", "SELECT 1"],
+        { env: { ...process.env, PGPASSWORD: password, PGCONNECT_TIMEOUT: "10" }, timeout: 20_000 },
+      );
+      return true;
+    } catch (err) {
+      const text = `${(err as { stderr?: string }).stderr ?? ""}`;
+      if (/password authentication failed|role ".*" does not exist|database ".*" does not exist|not permitted to log in|permission denied for database/i.test(text)) {
+        return false;
+      }
+      throw err;
+    }
+  }
+
   /** Drop a database and its owning role, idempotently, after evicting live connections. */
   async dropDatabaseAndRole(database: string, role: string): Promise<void> {
     await this.query(

modules/postgres/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/postgres/provisioner/index.ts

@@ -45,4 +45,9 @@ runProvisioner("postgres-database", {
     await postgres.dropDatabaseAndRole(p.as, p.as);
     await announce("module.postgres.database.deprovisioned", { database: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return postgres.canConnectAs(p.as, p.as, p.password);
+  },
 });

modules/redis/client.ts

@@ -8,7 +8,7 @@
 // order requests were sent, which is what the queue below relies on.
 
 import { createConnection, type Socket } from "node:net";
-import { randomBytes } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
 import { readFileSync } from "node:fs";
 
 /** A parsed RESP value. Errors are surfaced as rejected commands, not as this type. */
@@ -113,6 +113,27 @@ export class RedisClient {
     await this.command("ACL", "DELUSER", username);
   }
 
+  /**
+   * Whether an ACL user exists, is enabled, and accepts exactly this password. Read-only: it asks
+   * `ACL GETUSER`, which answers nil for an unknown user and otherwise a flat list of fields, among
+   * them `flags` and `passwords`, the latter as SHA-256 hex. This server keeps no ACL file, so its
+   * users live in memory and a restart forgets them. This is how the provisioner notices
+   * (novox/hq issue 120).
+   */
+  async holdsAclUser(username: string, password: string): Promise<boolean> {
+    const reply = await this.command("ACL", "GETUSER", username);
+    if (!Array.isArray(reply)) return false;
+    const field = (name: string): RespValue | undefined => {
+      const i = reply.indexOf(name);
+      return i >= 0 ? reply[i + 1] : undefined;
+    };
+    const flags = field("flags");
+    const passwords = field("passwords");
+    if (!Array.isArray(flags) || !flags.includes("on")) return false;
+    if (!Array.isArray(passwords)) return false;
+    return passwords.includes(createHash("sha256").update(password).digest("hex"));
+  }
+
   close(): void {
     if (this.socket) {
       this.socket.destroy();

modules/redis/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/redis/provisioner/index.ts

@@ -43,4 +43,11 @@ runProvisioner("redis-cache", {
     await redis.deleteAclUser(p.as);
     await announce("module.redis.cache.deprovisioned", { username: p.as });
   },
+
+  // This server keeps its ACL users in memory only, so a restart of it forgets every consumer while
+  // this provisioner keeps running. Asked every minute, so a forgotten user is made again instead
+  // of every consumer failing to authenticate in silence (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return redis.holdsAclUser(p.as, p.password);
+  },
 });