diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index 2e34d6a..0912483 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -31,6 +31,9 @@ images: # And the builder, because it is a module the mesh assigns rather than a program somebody # starts by hand — which is the only way its credential can be one the mesh delivered. - mesh-builder:development + # And the provisioner, which is what makes a sealed credential true on a machine — the mesh + # discarded the plaintext and cannot tell a database to start accepting it. + - mesh-provision-postgres:development place: all: [host, runtime] diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index 860fda0..e657e6d 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -755,3 +755,99 @@ test("the builder is a module the mesh assigns, with a credential the mesh deliv `the build was accepted and no build was recorded against the module:\n${recorded}`); assert.match(recorded, /built/, recorded); }); + +test("rotating a credential moves both ends, and the old one stops working", { + skip, timeout: 900_000, +}, async () => { + // The invariant novox/hq ADR 0001 records as unowned, and it was measurably false in HAL: on + // 2026-08-22 a provision documented as never rotating minted a new password on every adoption + // and updated only the provider's row. Consumers on three nodes held dead credentials for two + // days while the mesh reported success. + // + // So this is checked against a real database with a real login, three times: the delivered + // credential works, the rotated one works, and the one that was rotated away does not. Two ends + // holding a matching string proves they agree; only an authentication proves they are right. + const store = "/var/lib/mesh/postgres"; + await must("anchor", `printf %s '{"module":"realstore","version":"1",` + + `"provides":[{"name":"realdatabase","scope":"mesh"}],` + + `"capabilities":["container-runtime"],` + + `"serves":{"realdatabase":{"port":5433}},` + + `"needs":{"superuser":"${store}/superuser"},` + + `"grants":{"realdatabase":"${store}/grants"},` + + `"listens":[{"port":5433,"from":"mesh","why":"a database the mesh provisions"}],` + + `"resources":[` + + `{"id":"state","type":"directory","path":"${store}","mode":"0755"},` + + `{"id":"grants","type":"directory","path":"${store}/grants","mode":"0755"},` + + `{"id":"database","type":"container","name":"real-store",` + + `"image":"${pinned("postgres")}",` + + `"ports":["5433:5432"],` + + `"volumes":["${store}/superuser:/run/superuser:ro"],` + + `"env":{"POSTGRES_PASSWORD_FILE":"/run/superuser"}},` + + `{"id":"provisioner","type":"container","name":"real-provisioner",` + + `"image":"${pinned("mesh-provision-postgres")}","network":"host",` + + `"volumes":["${store}:${store}:ro"],` + + `"env":{"GRANTS":"${store}/grants",` + + `"MESH_PROVISION_PASSWORD_FILE":"${store}/superuser",` + + `"MESH_PROVISION_POSTGRES":"postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable"}}]}' ` + + `> /tmp/realstore.json`); + await must("anchor", `printf %s '{"module":"realapp","version":"1",` + + `"requires":["realdatabase"],"contributes":{"realdatabase":{"name":"realapp"}},` + + `"binds":{"realdatabase":"/etc/realapp/where.json"},` + + `"secrets":{"realdatabase":"/etc/realapp/password"},` + + `"resources":[{"id":"dir","type":"directory","path":"/etc/realapp","mode":"0755"}]}' ` + + `> /tmp/realapp.json`); + for (const f of ["realstore", "realapp"]) { + await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`); + await mesh(`module add /${f}.json`); + } + await mesh("assign anchor realstore"); + await mesh("assign laptop realapp"); + await mesh("push"); + await new Promise((r) => setTimeout(r, 30_000)); + + // A real login from the consumer's machine, over the private network — not over loopback, where + // pg_hba trusts anything and every password looks correct. That was done here once and the test + // passed for an afternoon while verifying nothing: a deliberately wrong password returned a row. + const login = async (password: string) => + await on("laptop", `docker run --rm -e PGPASSWORD=${quote(password)} ` + + `${pinned("postgres")} psql -h anchor.internal -p 5433 -U realapp ` + + `-d postgres -qAt -c "select 1"`, 120_000); + + const diagnostics = async () => + `provisioner:\n${(await on("anchor", `docker logs real-provisioner 2>&1 | tail -20`)).out}\n` + + `grants:\n${(await on("anchor", `ls -l ${store}/grants`)).out}`; + + const first = (await must("laptop", `cat /etc/realapp/password`)).trim(); + assert.ok(first.length >= 40, `the consumer's credential is ${first.length} characters`); + let works = false; + for (let i = 0; i < 20 && !works; i++) { + works = (await login(first)).ok; + if (!works) await new Promise((r) => setTimeout(r, 5000)); + } + assert.ok(works, `the delivered credential does not authenticate:\n${await diagnostics()}`); + + // Now rotate. One command: the record changes AND both ends are sent, because leaving the + // sending to a later command is the fault above, exactly. + const said = await mesh("rotate realdatabase", 180_000); + assert.match(said, /anchor/, `rotation did not touch the provider:\n${said}`); + assert.match(said, /laptop/, `rotation did not touch the consumer:\n${said}`); + await new Promise((r) => setTimeout(r, 25_000)); + + const second = (await must("laptop", `cat /etc/realapp/password`)).trim(); + assert.notEqual(second, first, "the consumer was handed back the credential just rotated away"); + + // The new one authenticates — the only proof the provider was told the same thing the consumer + // was given. Two files agreeing proves they agree, not that either is right. + let now = false; + for (let i = 0; i < 20 && !now; i++) { + now = (await login(second)).ok; + if (!now) await new Promise((r) => setTimeout(r, 5000)); + } + assert.ok(now, `after rotation the new credential does not authenticate, so the two ends ` + + `disagree — which is the fault this exists to make impossible:\n${await diagnostics()}`); + + // And the old one does not. Without this the test passes against a provider that added a + // password without replacing one, which is a rotation that rotates nothing. + assert.ok(!(await login(first)).ok, + "the password that was rotated away still authenticates, so nothing was rotated"); +});