Every credential provider says whether it still holds a consumer

holds() for postgres, mssql, mongodb, minio, lavinmq, mosquitto, mailu
and gitea, so the harness makes again a login the backend lost (hq issue
120). Each checks the mesh's password as the consumer presents it, or
compares it read-only, and returns false only when the backend says the
credential is absent or wrong; an unreachable backend throws.

modules/gitea/client.ts

@@ -433,6 +433,29 @@ export class GiteaAdmin {
     GiteaAdmin.fail(`/teams/${teamId}/members/${username}`, res);
   }
 
+  /**
+   * Whether a consumer's user logs in with exactly this password and is still a member of the
+   * package team. Read-only: the password is checked as the consumer presents it, basic auth on the
+   * API, and membership through the admin API. `false` for a refused login or a missing member; any
+   * other answer rejects (novox/hq issue 120).
+   */
+  async holdsTeamMember(org: string, team: string, username: string, password: string): Promise<boolean> {
+    const me = await fetch(`${this.baseUrl}/api/v1/user`, {
+      headers: { Authorization: "Basic " + Buffer.from(`${username}:${password}`).toString("base64") },
+    });
+    if (me.status === 401 || me.status === 403) return false;
+    if (me.status !== 200) throw new Error(`Gitea GET /user as ${username}: ${me.status}`);
+    const teams = await this.request(`/orgs/${encodeURIComponent(org)}/teams`);
+    if (teams.status === 404) return false;
+    if (teams.status !== 200) GiteaAdmin.fail(`/orgs/${org}/teams`, teams);
+    const found = (teams.body as { id: number; name: string }[]).find((t) => t.name === team);
+    if (!found) return false;
+    const member = await this.request(`/teams/${found.id}/members/${encodeURIComponent(username)}`);
+    if (member.status === 200 || member.status === 204) return true;
+    if (member.status === 404) return false;
+    GiteaAdmin.fail(`/teams/${found.id}/members/${username}`, member);
+  }
+
   /** Delete a user, purging what they own. A 404 means the mesh already withdrew them — success, not
    *  an error, so a re-run of remove is safe. */
   async deleteUser(username: string): Promise<void> {

modules/gitea/package.json

@@ -9,7 +9,7 @@
     "test": "npm run build && node --test --experimental-strip-types 'test/*.test.ts'"
   },
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/gitea/provisioner/index.ts

@@ -46,4 +46,9 @@ runProvisioner("package-registry", {
   async remove(p: { as: string }): Promise<void> {
     await gitea.deleteUser(p.as);
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return gitea.holdsTeamMember(ORG, PACKAGE_TEAM, p.as, p.password);
+  },
 });

modules/lavinmq/client.ts

@@ -94,6 +94,39 @@ export class LavinmqClient {
     await this.api("PUT", `/permissions/${v}/${u}`, { configure: ".*", write: ".*", read: ".*" });
   }
 
+  /**
+   * Whether a consumer's user exists with exactly this password and full permissions on its own
+   * vhost. Read-only: the stored hash is salted SHA-256, the scheme `rabbitHash` writes, so the
+   * password is checked by hashing it with the stored salt rather than by logging in. `false` when
+   * the user or its permission is gone or the password differs; an unreachable API rejects
+   * (novox/hq issue 120).
+   */
+  async holdsConsumer(login: string, password: string): Promise<boolean> {
+    const v = encodeURIComponent(login);
+    const u = encodeURIComponent(login);
+    const user = await this.getOrNull<{ password_hash?: string; hashing_algorithm?: string }>(`/users/${u}`);
+    if (!user?.password_hash) return false;
+    if (user.hashing_algorithm && !/sha256/i.test(user.hashing_algorithm)) {
+      throw new Error(`lavinmq user ${login} is hashed with ${user.hashing_algorithm}, which this check cannot verify`);
+    }
+    const stored = Buffer.from(user.password_hash, "base64");
+    if (stored.length < 5 || rabbitHash(password, stored.subarray(0, 4)) !== user.password_hash) return false;
+    const perm = await this.getOrNull<{ configure?: string; write?: string; read?: string }>(`/permissions/${v}/${u}`);
+    return perm?.configure === ".*" && perm?.write === ".*" && perm?.read === ".*";
+  }
+
+  /** A GET that answers null for a 404 and rejects on anything else that is not 2xx. */
+  private async getOrNull<T>(path: string): Promise<T | null> {
+    const resp = await fetch(`${this.conn.base}/api${path}`, {
+      headers: {
+        Authorization: "Basic " + Buffer.from(`${this.conn.adminUser}:${this.conn.adminPassword}`).toString("base64"),
+      },
+    });
+    if (resp.status === 404) return null;
+    if (!resp.ok) throw new Error(`lavinmq management API GET ${path} -> ${resp.status}: ${await resp.text()}`);
+    return (await resp.json()) as T;
+  }
+
   /** Remove a consumer's vhost and user, idempotently. A DELETE of what is already gone is tolerated. */
   async removeConsumer(login: string): Promise<void> {
     const v = encodeURIComponent(login);

modules/lavinmq/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/lavinmq/provisioner/index.ts

@@ -48,4 +48,9 @@ runProvisioner("amqp", {
     await lavinmq.removeConsumer(p.as);
     await announce("module.lavinmq.amqp.deprovisioned", { user: p.as, vhost: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return lavinmq.holdsConsumer(p.as, p.password);
+  },
 });

modules/mailu/client.ts

@@ -134,6 +134,35 @@ export class MailuClient {
     await this.api("DELETE", `/user/${encodeURIComponent(email)}`);
   }
 
+  /**
+   * Whether a mailbox exists, is enabled, and accepts exactly this password. Read-only. Existence
+   * from the admin API; the password from `doveadm auth test` in the imap container, which is how
+   * the mail server itself authenticates, and which exits 77 for a refused login. The password
+   * reaches doveadm through the exec's environment, never the host's argv. An unreachable API or
+   * container rejects (novox/hq issue 120).
+   */
+  async holdsUser(email: string, password: string): Promise<boolean> {
+    const res = await fetch(`${this.baseUrl}/user/${encodeURIComponent(email)}`, {
+      headers: { Authorization: this.apiKey, Accept: "application/json" },
+    });
+    if (res.status === 404) return false;
+    if (!res.ok) throw new Error(`Mailu API GET /user/${email}: ${res.status} ${await res.text()}`);
+    const user = (await res.json()) as { enabled?: boolean };
+    if (user.enabled === false) return false;
+    try {
+      await run(
+        "docker",
+        ["exec", "-e", "MESH_USER", "-e", "MESH_PW", this.imapContainer,
+          "sh", "-c", 'doveadm auth test "$MESH_USER" "$MESH_PW"'],
+        { env: { ...process.env, MESH_USER: email, MESH_PW: password }, timeout: 30_000 },
+      );
+      return true;
+    } catch (err) {
+      if ((err as { code?: number }).code === 77) return false;
+      throw err;
+    }
+  }
+
   async listAliases(): Promise<MailuAlias[]> {
     const aliases = await this.api<any[]>("GET", "/alias");
     return (aliases ?? []).map((a) => ({

modules/mailu/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/mailu/provisioner/index.ts

@@ -62,4 +62,9 @@ runProvisioner("smtp", {
     // named-account consumer is an operator action until the harness carries values here.
     await mailu.deleteUser(`${p.as}@${domain()}`).catch(() => {});
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return mailu.holdsUser(addressOf(p), p.password);
+  },
 });

modules/minio/client.ts

@@ -125,6 +125,18 @@ export class MinioClient {
     throw new Error(`minio bucketExists ${bucket}: ${status}`);
   }
 
+  /**
+   * Whether a consumer's access key, with exactly this secret, reaches its bucket: a HEAD of the
+   * bucket signed as the consumer, the way it signs. Read-only. `false` when the key is unknown, the
+   * secret wrong, access denied or the bucket gone; any other answer rejects (novox/hq issue 120).
+   */
+  async canReachAs(bucket: string, accessKey: string, secretKey: string): Promise<boolean> {
+    const { status } = await this.request("HEAD", `/${bucket}`, {}, { accessKey, secretKey });
+    if (status === 200) return true;
+    if (status === 403 || status === 404) return false;
+    throw new Error(`minio HEAD ${bucket} as ${accessKey}: ${status}`);
+  }
+
   async createBucket(bucket: string): Promise<void> {
     const { status, text } = await this.request("PUT", `/${bucket}`);
     // 200 created; 409 BucketAlreadyOwnedByYou — idempotent, a re-provision must not fail.
@@ -251,6 +263,7 @@ export class MinioClient {
     method: string,
     path: string,
     query: Record<string, string> = {},
+    as: { accessKey: string; secretKey: string } = { accessKey: this.rootUser, secretKey: this.rootPassword },
   ): Promise<{ status: number; headers: Headers; text: string }> {
     const { amzDate, dateStamp } = this.stamp();
     const host = new URL(this.baseUrl).host;
@@ -262,8 +275,8 @@ export class MinioClient {
     const canonicalRequest = [method, encodedPath, canonicalQuery, canonicalHeaders, signedHeaders, payloadHash].join("\n");
     const scope = `${dateStamp}/${this.region}/s3/aws4_request`;
     const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256hex(canonicalRequest)].join("\n");
-    const signature = hmac(this.signingKey(dateStamp), stringToSign).toString("hex");
-    const authorization = `AWS4-HMAC-SHA256 Credential=${this.rootUser}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
+    const signature = hmac(this.signingKey(dateStamp, as.secretKey), stringToSign).toString("hex");
+    const authorization = `AWS4-HMAC-SHA256 Credential=${as.accessKey}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
 
     const url = `${this.baseUrl}${encodedPath}${canonicalQuery ? `?${canonicalQuery}` : ""}`;
     const res = await fetch(url, {
@@ -275,8 +288,8 @@ export class MinioClient {
     return { status: res.status, headers: res.headers, text };
   }
 
-  private signingKey(dateStamp: string): Buffer {
-    const kDate = hmac(`AWS4${this.rootPassword}`, dateStamp);
+  private signingKey(dateStamp: string, secretKey: string = this.rootPassword): Buffer {
+    const kDate = hmac(`AWS4${secretKey}`, dateStamp);
     const kRegion = hmac(kDate, this.region);
     const kService = hmac(kRegion, "s3");
     return hmac(kService, "aws4_request");

modules/minio/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/minio/provisioner/index.ts

@@ -53,6 +53,12 @@ runProvisioner("s3-bucket", {
 
     await announce("module.minio.bucket.removed", { bucket, accessKey: p.as });
   },
+
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return minio.canReachAs(bucketFor(p.as), p.as, p.password);
+  },
 });
 
 /** Emit best-effort: a broker hiccup is logged and dropped, never allowed to throw back and fail a

modules/mongodb/client.ts

@@ -109,6 +109,32 @@ print(EJSON.stringify({ ok: 1 }));
     await this.evalJs<{ ok: number }>(js);
   }
 
+  /**
+   * Whether `user` authenticates against `database` with exactly `password` and holds `dbOwner`
+   * there: checked by connecting as the consumer, the way it connects. Read-only. `false` only on an
+   * authentication failure or a missing role; an unreachable server rejects (novox/hq issue 120).
+   */
+  async canAuthenticateAs(database: string, user: string, password: string): Promise<boolean> {
+    const uri =
+      `mongodb://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${this.conn.host}:${this.conn.port}` +
+      `/${encodeURIComponent(database)}?authSource=${encodeURIComponent(database)}&serverSelectionTimeoutMS=10000`;
+    let stdout: string;
+    try {
+      ({ stdout } = await run(
+        "mongosh",
+        [uri, "--quiet", "--eval",
+          "print(EJSON.stringify(db.runCommand({ connectionStatus: 1 }).authInfo.authenticatedUserRoles))"],
+        { timeout: 30_000 },
+      ));
+    } catch (err) {
+      const text = `${(err as { stderr?: string }).stderr ?? ""}${(err as { stdout?: string }).stdout ?? ""}`;
+      if (/Authentication failed|AuthenticationFailed/i.test(text)) return false;
+      throw err;
+    }
+    const roles = JSON.parse(stdout.trim()) as { role: string; db: string }[];
+    return roles.some((r) => r.role === "dbOwner" && r.db === database);
+  }
+
   /** Drop a database and its owning user, idempotently. Dropping the database evicts its data; the
    *  user is removed first so a re-grant of the same login starts clean. */
   async dropDatabaseAndUser(database: string, user: string): Promise<void> {

modules/mongodb/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/mongodb/provisioner/index.ts

@@ -45,4 +45,9 @@ runProvisioner("mongodb-database", {
     await mongo.dropDatabaseAndUser(p.as, p.as);
     await announce("module.mongodb.database.deprovisioned", { database: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return mongo.canAuthenticateAs(p.as, p.as, p.password);
+  },
 });

modules/mosquitto/client.ts

@@ -15,6 +15,7 @@
 // The one cost dynsec carries is the bootstrap file; see initBootstrapFile() and the module README.
 
 import { randomBytes } from "node:crypto";
+import { connect as tcpConnect } from "node:net";
 import { readFileSync } from "node:fs";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
@@ -163,6 +164,19 @@ export class MosquittoClient {
     }
   }
 
+  /**
+   * Whether a consumer's client accepts exactly this password and still carries its own role.
+   * Read-only. The password is checked the way the consumer is checked, by an MQTT CONNECT as it,
+   * and the broker's CONNACK code is the answer: 0 accepted, 4 bad credentials, 5 not authorised.
+   * Nothing rides on argv. An unreachable broker rejects (novox/hq issue 120).
+   */
+  async holdsClient(username: string, password: string): Promise<boolean> {
+    const code = await mqttConnack(this.conn.host, this.conn.port, username, password);
+    if (code === 4 || code === 5) return false;
+    if (code !== 0) throw new Error(`mosquitto refused ${username} with CONNACK ${code}`);
+    return this.clientHasRole(username, username);
+  }
+
   /** Remove a client and the per-client role created for it, idempotently. */
   async deleteScopedClient(username: string): Promise<void> {
     await ignoreMissing(this.ctl("deleteClient", username));
@@ -249,3 +263,55 @@ function readSecretFile(path: string | undefined): string | undefined {
     return undefined;
   }
 }
+
+/**
+ * Connect once over MQTT 3.1.1 with a username and password, return the broker's CONNACK return code,
+ * and disconnect. A clean session under a throwaway client id, so no consumer session is taken over.
+ */
+function mqttConnack(host: string, port: number, username: string, password: string): Promise<number> {
+  const str = (v: string): Buffer => {
+    const b = Buffer.from(v, "utf8");
+    const len = Buffer.alloc(2);
+    len.writeUInt16BE(b.length);
+    return Buffer.concat([len, b]);
+  };
+  const variable = Buffer.concat([str("MQTT"), Buffer.from([4, 0xc2, 0, 10])]); // level 4; user+pass+clean; keepalive 10s
+  const payload = Buffer.concat([str(`mesh-holds-${randomBytes(6).toString("hex")}`), str(username), str(password)]);
+  let remaining = variable.length + payload.length;
+  const lenBytes: number[] = [];
+  do {
+    let byte = remaining % 128;
+    remaining = Math.floor(remaining / 128);
+    if (remaining > 0) byte |= 0x80;
+    lenBytes.push(byte);
+  } while (remaining > 0);
+  const packet = Buffer.concat([Buffer.from([0x10, ...lenBytes]), variable, payload]);
+
+  return new Promise((resolve, reject) => {
+    const socket = tcpConnect({ host, port });
+    let buf = Buffer.alloc(0);
+    const timer = setTimeout(() => {
+      socket.destroy();
+      reject(new Error(`no CONNACK from ${host}:${port} within 10s`));
+    }, 10_000);
+    socket.on("connect", () => socket.write(packet));
+    socket.on("data", (chunk) => {
+      buf = Buffer.concat([buf, chunk]);
+      if (buf.length < 4) return;
+      clearTimeout(timer);
+      if (buf[0] !== 0x20) {
+        socket.destroy();
+        reject(new Error(`unexpected MQTT packet 0x${buf[0].toString(16)} instead of CONNACK`));
+        return;
+      }
+      const code = buf[3];
+      if (code === 0) socket.end(Buffer.from([0xe0, 0])); // DISCONNECT
+      else socket.destroy();
+      resolve(code);
+    });
+    socket.on("error", (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+  });
+}

modules/mosquitto/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/mosquitto/provisioner/index.ts

@@ -43,4 +43,9 @@ runProvisioner("mqtt-topic", {
     await mosquitto.deleteScopedClient(p.as);
     await announce("module.mosquitto.topic.deprovisioned", { username: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return mosquitto.holdsClient(p.as, p.password);
+  },
 });

modules/mssql/client.ts

@@ -142,6 +142,25 @@ export class MssqlClient {
     await this.exec(`ALTER ROLE db_owner ADD MEMBER ${ident(login)}`, database);
   }
 
+  /**
+   * Whether `login` exists, is enabled, has exactly `password`, and is a db_owner user of
+   * `database`. Read-only: the password is compared with PWDCOMPARE against the stored hash, so
+   * nothing logs in and no failed-login is recorded (novox/hq issue 120).
+   */
+  async holdsLogin(database: string, login: string, password: string): Promise<boolean> {
+    const server = await this.query(
+      `SELECT CAST(CASE WHEN EXISTS (SELECT 1 FROM sys.sql_logins WHERE name = ${literal(login)} ` +
+        `AND is_disabled = 0 AND PWDCOMPARE(${literal(password)}, password_hash) = 1) ` +
+        `AND DB_ID(${literal(database)}) IS NOT NULL THEN 1 ELSE 0 END AS int) AS ok`,
+    );
+    if (Number(server[0]?.ok) !== 1) return false;
+    const owner = await this.query(
+      `SELECT CAST(IS_ROLEMEMBER('db_owner', ${literal(login)}) AS int) AS ok`,
+      database,
+    );
+    return Number(owner[0]?.ok) === 1;
+  }
+
   /** Drop a database and its login, idempotently, after evicting live connections. */
   async dropDatabaseAndLogin(database: string, login: string): Promise<void> {
     const dbs = await this.query(

modules/mssql/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/mssql/provisioner/index.ts

@@ -44,4 +44,9 @@ runProvisioner("mssql-database", {
     await mssql.dropDatabaseAndLogin(p.as, p.as);
     await announce("module.mssql.database.deprovisioned", { database: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return mssql.holdsLogin(p.as, p.as, p.password);
+  },
 });

modules/postgres/client.ts

@@ -99,6 +99,30 @@ export class PostgresClient {
     await this.query(`GRANT ALL PRIVILEGES ON DATABASE ${ident(database)} TO ${ident(role)}`);
   }
 
+  /**
+   * Whether `role` can log in to `database` with exactly `password`: the consumer's own view of its
+   * credential, checked by connecting as it. Read-only. `false` only when the server says so (the
+   * role, the password or the database is wrong or gone); an unreachable server rejects instead,
+   * because being unable to ask is not evidence of loss (novox/hq issue 120).
+   */
+  async canConnectAs(database: string, role: string, password: string): Promise<boolean> {
+    try {
+      await run(
+        "psql",
+        ["-h", this.conn.host, "-p", String(this.conn.port), "-U", role, "-d", database,
+          "-v", "ON_ERROR_STOP=1", "--no-psqlrc", "-tAc", "SELECT 1"],
+        { env: { ...process.env, PGPASSWORD: password, PGCONNECT_TIMEOUT: "10" }, timeout: 20_000 },
+      );
+      return true;
+    } catch (err) {
+      const text = `${(err as { stderr?: string }).stderr ?? ""}`;
+      if (/password authentication failed|role ".*" does not exist|database ".*" does not exist|not permitted to log in|permission denied for database/i.test(text)) {
+        return false;
+      }
+      throw err;
+    }
+  }
+
   /** Drop a database and its owning role, idempotently, after evicting live connections. */
   async dropDatabaseAndRole(database: string, role: string): Promise<void> {
     await this.query(

modules/postgres/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "private": true,
   "dependencies": {
-    "@novox/mesh-sdk": "^0.1.0"
+    "@novox/mesh-sdk": "^0.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

modules/postgres/provisioner/index.ts

@@ -45,4 +45,9 @@ runProvisioner("postgres-database", {
     await postgres.dropDatabaseAndRole(p.as, p.as);
     await announce("module.postgres.database.deprovisioned", { database: p.as });
   },
+  // Asked every minute by the harness: whether the backend still holds this consumer exactly as
+  // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120).
+  async holds(p: Provision): Promise<boolean> {
+    return postgres.canConnectAs(p.as, p.as, p.password);
+  },
 });