Prove rotation against a real database, with a real login

Two ends holding a matching string proves they agree, not that either is right.
So the check is three logins over the private network from the consumer's own
machine: the delivered credential works, the rotated one works, and the one
that was rotated away does not. Without the last, the test passes against a
provider that added a password without replacing one.

Not over loopback: pg_hba trusts anything there, and a deliberately wrong
password returned a row for a whole afternoon once.

scenarios/two-nodes.yml

@@ -31,6 +31,9 @@ images:
   # And the builder, because it is a module the mesh assigns rather than a program somebody
   # starts by hand — which is the only way its credential can be one the mesh delivered.
   - mesh-builder:development
+  # And the provisioner, which is what makes a sealed credential true on a machine — the mesh
+  # discarded the plaintext and cannot tell a database to start accepting it.
+  - mesh-provision-postgres:development
 
 place:
   all: [host, runtime]

test/integration/mesh.test.ts

@@ -755,3 +755,99 @@ test("the builder is a module the mesh assigns, with a credential the mesh deliv
     `the build was accepted and no build was recorded against the module:\n${recorded}`);
   assert.match(recorded, /built/, recorded);
 });
+
+test("rotating a credential moves both ends, and the old one stops working", {
+  skip, timeout: 900_000,
+}, async () => {
+  // The invariant novox/hq ADR 0001 records as unowned, and it was measurably false in HAL: on
+  // 2026-08-22 a provision documented as never rotating minted a new password on every adoption
+  // and updated only the provider's row. Consumers on three nodes held dead credentials for two
+  // days while the mesh reported success.
+  //
+  // So this is checked against a real database with a real login, three times: the delivered
+  // credential works, the rotated one works, and the one that was rotated away does not. Two ends
+  // holding a matching string proves they agree; only an authentication proves they are right.
+  const store = "/var/lib/mesh/postgres";
+  await must("anchor", `printf %s '{"module":"realstore","version":"1",` +
+    `"provides":[{"name":"realdatabase","scope":"mesh"}],` +
+    `"capabilities":["container-runtime"],` +
+    `"serves":{"realdatabase":{"port":5433}},` +
+    `"needs":{"superuser":"${store}/superuser"},` +
+    `"grants":{"realdatabase":"${store}/grants"},` +
+    `"listens":[{"port":5433,"from":"mesh","why":"a database the mesh provisions"}],` +
+    `"resources":[` +
+    `{"id":"state","type":"directory","path":"${store}","mode":"0755"},` +
+    `{"id":"grants","type":"directory","path":"${store}/grants","mode":"0755"},` +
+    `{"id":"database","type":"container","name":"real-store",` +
+    `"image":"${pinned("postgres")}",` +
+    `"ports":["5433:5432"],` +
+    `"volumes":["${store}/superuser:/run/superuser:ro"],` +
+    `"env":{"POSTGRES_PASSWORD_FILE":"/run/superuser"}},` +
+    `{"id":"provisioner","type":"container","name":"real-provisioner",` +
+    `"image":"${pinned("mesh-provision-postgres")}","network":"host",` +
+    `"volumes":["${store}:${store}:ro"],` +
+    `"env":{"GRANTS":"${store}/grants",` +
+    `"MESH_PROVISION_PASSWORD_FILE":"${store}/superuser",` +
+    `"MESH_PROVISION_POSTGRES":"postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable"}}]}' ` +
+    `> /tmp/realstore.json`);
+  await must("anchor", `printf %s '{"module":"realapp","version":"1",` +
+    `"requires":["realdatabase"],"contributes":{"realdatabase":{"name":"realapp"}},` +
+    `"binds":{"realdatabase":"/etc/realapp/where.json"},` +
+    `"secrets":{"realdatabase":"/etc/realapp/password"},` +
+    `"resources":[{"id":"dir","type":"directory","path":"/etc/realapp","mode":"0755"}]}' ` +
+    `> /tmp/realapp.json`);
+  for (const f of ["realstore", "realapp"]) {
+    await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`);
+    await mesh(`module add /${f}.json`);
+  }
+  await mesh("assign anchor realstore");
+  await mesh("assign laptop realapp");
+  await mesh("push");
+  await new Promise((r) => setTimeout(r, 30_000));
+
+  // A real login from the consumer's machine, over the private network — not over loopback, where
+  // pg_hba trusts anything and every password looks correct. That was done here once and the test
+  // passed for an afternoon while verifying nothing: a deliberately wrong password returned a row.
+  const login = async (password: string) =>
+    await on("laptop", `docker run --rm -e PGPASSWORD=${quote(password)} ` +
+      `${pinned("postgres")} psql -h anchor.internal -p 5433 -U realapp ` +
+      `-d postgres -qAt -c "select 1"`, 120_000);
+
+  const diagnostics = async () =>
+    `provisioner:\n${(await on("anchor", `docker logs real-provisioner 2>&1 | tail -20`)).out}\n` +
+    `grants:\n${(await on("anchor", `ls -l ${store}/grants`)).out}`;
+
+  const first = (await must("laptop", `cat /etc/realapp/password`)).trim();
+  assert.ok(first.length >= 40, `the consumer's credential is ${first.length} characters`);
+  let works = false;
+  for (let i = 0; i < 20 && !works; i++) {
+    works = (await login(first)).ok;
+    if (!works) await new Promise((r) => setTimeout(r, 5000));
+  }
+  assert.ok(works, `the delivered credential does not authenticate:\n${await diagnostics()}`);
+
+  // Now rotate. One command: the record changes AND both ends are sent, because leaving the
+  // sending to a later command is the fault above, exactly.
+  const said = await mesh("rotate realdatabase", 180_000);
+  assert.match(said, /anchor/, `rotation did not touch the provider:\n${said}`);
+  assert.match(said, /laptop/, `rotation did not touch the consumer:\n${said}`);
+  await new Promise((r) => setTimeout(r, 25_000));
+
+  const second = (await must("laptop", `cat /etc/realapp/password`)).trim();
+  assert.notEqual(second, first, "the consumer was handed back the credential just rotated away");
+
+  // The new one authenticates — the only proof the provider was told the same thing the consumer
+  // was given. Two files agreeing proves they agree, not that either is right.
+  let now = false;
+  for (let i = 0; i < 20 && !now; i++) {
+    now = (await login(second)).ok;
+    if (!now) await new Promise((r) => setTimeout(r, 5000));
+  }
+  assert.ok(now, `after rotation the new credential does not authenticate, so the two ends ` +
+    `disagree — which is the fault this exists to make impossible:\n${await diagnostics()}`);
+
+  // And the old one does not. Without this the test passes against a provider that added a
+  // password without replacing one, which is a rotation that rotates nothing.
+  assert.ok(!(await login(first)).ok,
+    "the password that was rotated away still authenticates, so nothing was rotated");
+});