mesh-lab/test/integration/provisioner.test.ts

/**
 * The last step of a credential, against a real database.
 *
 * The mesh generates a password, seals it to the machine that must accept it, and discards the
 * plaintext — so it cannot tell PostgreSQL to start accepting it. Something on that machine reads
 * what the host wrote and makes it true. Everything up to that point is proven elsewhere; this is
 * the step where a password either becomes a login or does not.
 *
 * Against a real PostgreSQL because there is no version of this worth asserting against a fake:
 * what is under test is whether `create role ... password` and a connection agree, which is
 * exactly what a fake would be told to agree about (novox/hq ADR 0017).
 */

import { test, after, before } from "node:test";
import assert from "node:assert/strict";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";

const capability = await labIsUsable();
const provisioner = process.env["MESH_LAB_PROVISIONER"] ?? "";
const skip = !capability.usable
  ? `lab not usable: ${capability.why}`
  : !provisioner
    ? "set MESH_LAB_PROVISIONER to a built provisioner (mesh-controller: go build ./examples/postgres-provisioner)"
    : false;

const SCENARIO = "a-provider";
const MACHINE = "anchor";
const GRANTS = "/var/lib/postgres/grants";
const SUPER = "postgres://postgres:super@127.0.0.1:5432/postgres?sslmode=disable";

let instanceId = "";
/**
 * The database, pinned upstream and pulled by the machine over its uplink.
 *
 * The same digest the mesh's own postgres module pins, so this is the database the mesh runs
 * rather than a lookalike. It used to come from a registry the lab raised inside the scenario;
 * nothing outside the lab has one, so what that proved about fetching an image was true only here.
 */
const image =
  "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";

function shellQuote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

/** Run something on the machine and return what it said, with its exit status. */
async function on(command: string): Promise<{ out: string; ok: boolean }> {
  const { stdout } = await exec(instanceId, MACHINE, [
    "sh", "-c", `${command} 2>&1; echo "__exit=$?"`,
  ]);
  const marker = stdout.lastIndexOf("__exit=");
  const status = Number(stdout.slice(marker + 7).trim());
  return { out: stdout.slice(0, marker), ok: status === 0 };
}

/** The same, refusing to continue past a failure nobody would otherwise see. */
async function must(command: string): Promise<string> {
  const { out, ok } = await on(command);
  if (!ok) throw new Error(`${command}\n${out}`);
  return out;
}

/** psql as the superuser, inside the database container. */
async function sql(query: string): Promise<string> {
  return (await must(`docker exec mesh-db psql -U postgres -qAt -c ${shellQuote(query)}`)).trim();
}

/**
 * Write what the host would have written from a declaration: the manifest of who asked, and one
 * file per consumer holding its password alone.
 *
 * Written here rather than by running the host, because what is under test is the step *after*
 * the host — and that the host writes these exact shapes is asserted in its own suite.
 */
async function meshWrote(
  consumers: { node: string; module: string; name: string; password: string }[],
): Promise<void> {
  const manifest = {
    contributions: 1,
    requirement: "postgres-database",
    generated: "by the mesh",
    given: consumers.map((c) => ({
      from: c.module,
      node: c.node,
      secret: `${GRANTS}/${c.node}.${c.module}.secret`,
      values: { name: c.name },
    })),
  };
  await must(`mkdir -p ${GRANTS}`);
  await must(`printf %s ${shellQuote(JSON.stringify(manifest))} > ${GRANTS}/mesh.json`);
  // Every credential file rewritten from nothing, so a removed consumer's does not linger and
  // make the revocation test pass for a reason that is not the one being tested.
  await must(`find ${GRANTS} -name '*.secret' -delete`);
  for (const c of consumers) {
    await must(`printf %s ${shellQuote(c.password)} > ${GRANTS}/${c.node}.${c.module}.secret`);
    await must(`chmod 600 ${GRANTS}/${c.node}.${c.module}.secret`);
  }
}

/** The provisioner, as the module shipping PostgreSQL would run it. */
async function provision(): Promise<{ out: string; ok: boolean }> {
  return on(
    `GRANTS=${GRANTS} MESH_PROVISION_POSTGRES=${shellQuote(SUPER)} /usr/local/bin/mesh-provision-postgres`,
  );
}

/**
 * Can this role log in with this password?
 *
 * Over the bridge, from a container of its own. `--network container:mesh-db` would share the
 * database's namespace and put us back on its loopback, which is the very thing being avoided.
 *
 * The address comes from `.NetworkSettings.Networks.bridge.IPAddress` rather than the top-level
 * `.NetworkSettings.IPAddress`, which docker 29 no longer populates — it templates to empty, psql
 * silently falls back to a unix socket that is not there, and every login looks impossible.
 *
 * From a separate container, reaching the database over the bridge — **not** from inside it over
 * loopback. PostgreSQL's default `pg_hba.conf` trusts `127.0.0.1`, so a check made from inside
 * the container authenticates nothing and returns true for any password at all. Which is what the
 * first version of this did: two tests passed without ever verifying a password, and only the
 * rotation test noticed, by asserting that an old password had *stopped* working.
 */
async function canLogIn(role: string, password: string, database: string): Promise<boolean> {
  return (await tryLogIn(role, password, database)).ok;
}

/** The same, keeping what the database said — so a failure says why rather than only that. */
async function tryLogIn(
  role: string,
  password: string,
  database: string,
): Promise<{ ok: boolean; out: string }> {
  const { out } = await on(
    `docker run --rm -e PGPASSWORD=${shellQuote(password)} ${image} ` +
      `psql -h "$(docker inspect -f '{{.NetworkSettings.Networks.bridge.IPAddress}}' mesh-db)" ` +
      `-U ${role} -d ${database} -qAt -c 'select 1'`,
  );
  return { ok: out.trim() === "1", out };
}

before(async () => {
  if (skip) return;
  const scenario = loadScenario(`scenarios/${SCENARIO}.yml`);
  const instance = await raise(scenario, {});
  instanceId = instance.instanceId;

  await must(
    `docker run -d --name mesh-db -e POSTGRES_PASSWORD=super ` +
      `-p 127.0.0.1:5432:5432 ${image}`,
  );
  let ready = false;
  for (let i = 0; i < 90 && !ready; i++) {
    ({ ok: ready } = await on(`docker exec mesh-db pg_isready -U postgres`));
    if (!ready) await new Promise((r) => setTimeout(r, 1000));
  }
  assert.ok(ready, "the database never became ready");

  await incus([
    "file", "push", provisioner,
    `${machineName(instanceId, MACHINE)}/usr/local/bin/mesh-provision-postgres`,
    "--mode", "0755",
  ], 180_000);
}, { timeout: 1_200_000 });

after(async () => {
  if (instanceId) await destroy(instanceId);
  await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });

test("a password the mesh generated becomes a login that works", { skip, timeout: 300_000 }, async () => {
  await meshWrote([
    { node: "workstation", module: "meshboard", name: "meshboard", password: "first-password-aaa" },
  ]);
  const { out, ok } = await provision();
  assert.ok(ok, out);

  assert.equal(await sql(`select rolcanlogin from pg_roles where rolname = 'mesh_workstation_meshboard'`), "t");
  assert.equal(await sql(`select 1 from pg_database where datname = 'meshboard'`), "1");
  const attempt = await tryLogIn("mesh_workstation_meshboard", "first-password-aaa", "meshboard");
  assert.ok(attempt.ok, `the consumer cannot log in with the password the mesh gave it:\n${attempt.out}`);
});

test("running it again reaches the same state and says nothing", { skip, timeout: 300_000 }, async () => {
  // It runs after every declaration and is never told what changed, so arriving at an already
  // correct state is the ordinary case rather than an edge one.
  const { out, ok } = await provision();
  assert.ok(ok, out);
  assert.equal(out.trim(), "", `it did work on a second run: ${out}`);
  assert.ok(await canLogIn("mesh_workstation_meshboard", "first-password-aaa", "meshboard"));
});

test("rotating the password makes the new one work and the old one stop", { skip, timeout: 300_000 }, async () => {
  // The failure this guards is a provisioner that only ever creates: the mesh replaces the file,
  // the role exists, nothing happens, and a rotation reports success while changing nothing.
  await meshWrote([
    { node: "workstation", module: "meshboard", name: "meshboard", password: "second-password-bbb" },
  ]);
  const { out, ok } = await provision();
  assert.ok(ok, out);

  assert.ok(
    await canLogIn("mesh_workstation_meshboard", "second-password-bbb", "meshboard"),
    "the rotated password does not work",
  );
  assert.equal(
    await canLogIn("mesh_workstation_meshboard", "first-password-aaa", "meshboard"),
    false,
    "the old password still works, so the rotation changed nothing",
  );
});

test("a consumer that goes away loses its login", { skip, timeout: 300_000 }, async () => {
  // The half usually missing. A consumer removed from the mesh otherwise keeps a working login
  // for ever and nothing says so — the same rule the host follows about removing what it declared
  // and no longer declares.
  await meshWrote([]);
  const { out, ok } = await provision();
  assert.ok(ok, out);
  assert.match(out, /revoked mesh_workstation_meshboard/);

  assert.equal(await sql(`select rolcanlogin from pg_roles where rolname = 'mesh_workstation_meshboard'`), "f");
  assert.equal(
    await canLogIn("mesh_workstation_meshboard", "second-password-bbb", "meshboard"),
    false,
    "a consumer nobody asks for any more can still log in",
  );
});

test("a role nobody here made is left alone", { skip, timeout: 300_000 }, async () => {
  // A provisioner that removed every role it did not recognise could not safely be run on a
  // database that predates it — which is every database anybody would want to adopt.
  await sql(`create role someone_elses with login password 'theirs'`);
  await sql(`create database theirs owner someone_elses`);
  await meshWrote([]);
  const { ok } = await provision();
  assert.ok(ok);
  assert.equal(await sql(`select rolcanlogin from pg_roles where rolname = 'someone_elses'`), "t");
  assert.ok(await canLogIn("someone_elses", "theirs", "theirs"));
});

test("a manifest naming a credential that was never written is refused", { skip, timeout: 300_000 }, async () => {
  // Rather than creating a role with no password — a login nothing can use, which nothing would
  // report until something tried to connect.
  await meshWrote([]);
  await must(
    `printf %s '{"contributions":1,"requirement":"postgres-database","given":[` +
      `{"from":"meshboard","node":"ghost","secret":"${GRANTS}/ghost.meshboard.secret","values":{"name":"ghost"}}` +
      `]}' > ${GRANTS}/mesh.json`,
  );
  const { out, ok } = await provision();
  assert.equal(ok, false, "it carried on past a missing credential");
  assert.match(out, /should be at .*ghost\.meshboard\.secret/);
  assert.equal(await sql(`select count(*) from pg_roles where rolname = 'mesh_ghost_meshboard'`), "0");
});