mesh-lab/test/integration/objectstore.test.ts

/**
 * The last step of a credential, against a real object store.
 *
 * The mesh generates a secret, seals it to the machine that must accept it, and discards the
 * plaintext — so it cannot tell the store to start accepting it. Something on that machine reads
 * what the host wrote and makes it true. This is the step where a secret either becomes a working
 * key or does not.
 *
 * **Phase 1.1 of the work breakdown**, and the finding that shaped it: the control plane
 * special-cases nothing. `provides`, `requires`, `contributes` and `grants` are name-agnostic, so
 * asking for a bucket needed no change to the mesh at all — only a provider that answers. What is
 * proven here is that half.
 *
 * **And the half a database does not have.** One PostgreSQL server holds separate databases, and
 * a role that cannot reach another's is a boundary the product enforces. One object store holds
 * everybody's buckets behind one endpoint, so a consumer being unable to reach another's is a
 * policy somebody wrote — which means it is a thing that can be written wrongly, and therefore a
 * thing to assert rather than assume.
 */

import { test, after, before } from "node:test";
import assert from "node:assert/strict";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";

const capability = await labIsUsable();
const provisioner = process.env["MESH_LAB_OBJECTSTORE_PROVISIONER"] ?? "";
const skip = !capability.usable
  ? `lab not usable: ${capability.why}`
  : !provisioner
    ? "set MESH_LAB_OBJECTSTORE_PROVISIONER to a built provisioner " +
      "(mesh-control: go build ./examples/objectstore-provisioner)"
    : false;

const SCENARIO = "an-object-store";
const MACHINE = "anchor";
const GRANTS = "/var/lib/objectstore/grants";
const ROOT_USER = "meshroot";
const ROOT_PASSWORD = "meshroot-super-secret";
const ROOT_PASSWORD_FILE = "/var/lib/objectstore/root.secret";
const ENDPOINT = "http://127.0.0.1:9000";

let instanceId = "";
/** The store's image, by digest, from the registry the scenario raised. */
let storeImage = "";

function shellQuote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

async function on(command: string): Promise<{ out: string; ok: boolean }> {
  const { stdout } = await exec(instanceId, MACHINE, [
    "sh", "-c", `${command} 2>&1; echo "__exit=$?"`,
  ]);
  const marker = stdout.lastIndexOf("__exit=");
  const status = Number(stdout.slice(marker + 7).trim());
  return { out: stdout.slice(0, marker), ok: status === 0 };
}

/** The same, refusing to continue past a failure nobody would otherwise see. */
async function must(command: string): Promise<string> {
  const { out, ok } = await on(command);
  if (!ok) throw new Error(`${command}\n${out}`);
  return out;
}

/** `mc` on the machine, against the store as root. */
async function admin(args: string): Promise<{ out: string; ok: boolean }> {
  return on(`mc --config-dir /tmp/root-mc ${args}`);
}

/**
 * Write what the host would have written from a declaration: the manifest of who asked, and one
 * file per consumer holding its secret alone.
 *
 * Written here rather than by running the host, because what is under test is the step *after*
 * the host — and that the host writes these exact shapes is asserted in its own suite.
 */
async function meshWrote(
  consumers: { node: string; module: string; bucket: string; secret: string }[],
): Promise<void> {
  const manifest = {
    contributions: consumers.length,
    requirement: "s3-bucket",
    generated: "by the mesh",
    given: consumers.map((c) => ({
      from: c.module,
      node: c.node,
      secret: `${GRANTS}/${c.node}.secret`,
      values: { bucket: c.bucket },
    })),
  };
  await must(`mkdir -p ${GRANTS}`);
  await must(`printf %s ${shellQuote(JSON.stringify(manifest))} > ${GRANTS}/mesh.json`);
  // Every credential file rewritten from nothing, so a removed consumer's does not linger and
  // make the revocation test pass for a reason that is not the one being tested.
  await must(`find ${GRANTS} -name '*.secret' -delete`);
  for (const c of consumers) {
    await must(`printf %s ${shellQuote(c.secret)} > ${GRANTS}/${c.node}.secret`);
    await must(`chmod 600 ${GRANTS}/${c.node}.secret`);
  }
}

/** The provisioner, as the module shipping the store would run it. */
async function provision(): Promise<{ out: string; ok: boolean }> {
  return on(
    `GRANTS=${GRANTS} ` +
      `MESH_OBJECTSTORE_URL=${ENDPOINT} ` +
      `MESH_OBJECTSTORE_ROOT_USER=${ROOT_USER} ` +
      `MESH_OBJECTSTORE_ROOT_PASSWORD_FILE=${ROOT_PASSWORD_FILE} ` +
      `/usr/local/bin/mesh-provision-objectstore`,
  );
}

/**
 * Can this key write to and read from this bucket?
 *
 * As the consumer, with its own `mc` configuration directory — never the root one. A check made
 * with the root alias still in scope would pass for any key at all, which is the object-store
 * shape of the mistake the database suite records: two of its tests once passed without verifying
 * a password, because they ran where PostgreSQL trusts the caller.
 */
async function canUse(key: string, secret: string, bucket: string): Promise<{ ok: boolean; out: string }> {
  const dir = `/tmp/as-${key}`;
  const { out, ok } = await on(
    `rm -rf ${dir} && mc --config-dir ${dir} alias set probe ${ENDPOINT} ${shellQuote(key)} ${shellQuote(secret)} && ` +
      `echo hello > /tmp/probe.txt && ` +
      `mc --config-dir ${dir} cp /tmp/probe.txt probe/${bucket}/probe.txt && ` +
      `mc --config-dir ${dir} cat probe/${bucket}/probe.txt`,
  );
  return { ok: ok && out.includes("hello"), out };
}

before(async () => {
  if (skip) return;
  const scenario = loadScenario(`scenarios/${SCENARIO}.yml`);
  const instance = await raise(scenario, {});
  instanceId = instance.instanceId;

  // From the registry the scenario raised, by digest. There is no route to a public registry from
  // a documentation range, which is the point of the lab having its own.
  const store = instance.images.find((r) => r.includes("minio/minio"));
  const client = instance.images.find((r) => r.includes("minio/mc"));
  assert.ok(store, `the scenario stocked no store image: ${instance.images.join(", ")}`);
  assert.ok(client, `the scenario stocked no client image: ${instance.images.join(", ")}`);
  storeImage = store;

  // The client, taken out of the vendor's own image onto the machine. The provisioner drives it,
  // so it has to be here — and taking it from the stocked image is what keeps this test off any
  // public network.
  await must(`docker create --name mc-source ${client}`);
  await must(`docker cp mc-source:/usr/bin/mc /usr/local/bin/mc && chmod 755 /usr/local/bin/mc`);
  await must(`docker rm mc-source`);

  await must(`mkdir -p ${GRANTS}`);
  await must(`printf %s ${shellQuote(ROOT_PASSWORD)} > ${ROOT_PASSWORD_FILE} && chmod 600 ${ROOT_PASSWORD_FILE}`);

  await must(
    `docker run -d --name mesh-store ` +
      `-e MINIO_ROOT_USER=${ROOT_USER} -e MINIO_ROOT_PASSWORD=${shellQuote(ROOT_PASSWORD)} ` +
      `-p 127.0.0.1:9000:9000 ${storeImage} server /data`,
  );

  // Ready over the endpoint the provisioner will use, not by the container being up. A store that
  // is starting answers the port and refuses every operation, which is indistinguishable from a
  // wrong credential if it is not waited for.
  let ready = false;
  for (let i = 0; i < 90 && !ready; i++) {
    ({ ok: ready } = await on(
      `mc --config-dir /tmp/root-mc alias set root ${ENDPOINT} ${ROOT_USER} ${shellQuote(ROOT_PASSWORD)}`,
    ));
    if (!ready) await new Promise((r) => setTimeout(r, 1000));
  }
  assert.ok(ready, "the store never became ready");

  await incus([
    "file", "push", provisioner,
    `${machineName(instanceId, MACHINE)}/usr/local/bin/mesh-provision-objectstore`,
    "--mode", "0755",
  ], 180_000);
}, { timeout: 1_200_000 });

after(async () => {
  if (instanceId) await destroy(instanceId);
  await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });

test("a secret the mesh generated becomes a key that works", { skip, timeout: 300_000 }, async () => {
  await meshWrote([
    { node: "workstation", module: "photos", bucket: "photos", secret: "first-secret-aaaaaaaa" },
  ]);
  const { out, ok } = await provision();
  assert.ok(ok, out);

  const listed = await admin(`admin user list root --json`);
  assert.ok(listed.out.includes("mesh_workstation"), `no key was made for the consumer:\n${listed.out}`);

  const used = await canUse("mesh_workstation", "first-secret-aaaaaaaa", "photos");
  assert.ok(used.ok, `the consumer cannot use the bucket the mesh gave it:\n${used.out}`);
});

test("a consumer cannot reach another consumer's bucket", { skip, timeout: 300_000 }, async () => {
  // **The assertion this whole scenario exists for.** One store holds every bucket behind one
  // endpoint, so isolation is a policy rather than a property, and a policy granting
  // `arn:aws:s3:::*` would pass every other test in this file.
  await meshWrote([
    { node: "workstation", module: "photos", bucket: "photos", secret: "first-secret-aaaaaaaa" },
    { node: "laptop", module: "invoices", bucket: "invoices", secret: "second-secret-bbbbbbbb" },
  ]);
  const { out, ok } = await provision();
  assert.ok(ok, out);

  const own = await canUse("mesh_laptop", "second-secret-bbbbbbbb", "invoices");
  assert.ok(own.ok, `a consumer cannot use its own bucket:\n${own.out}`);

  const other = await canUse("mesh_laptop", "second-secret-bbbbbbbb", "photos");
  assert.ok(!other.ok, `a consumer reached another consumer's bucket:\n${other.out}`);
});

test("rotating the secret makes the new one work and the old one stop", { skip, timeout: 300_000 }, async () => {
  // The failure this guards is a provisioner that only ever creates: the mesh replaces the file,
  // the user exists, nothing happens, and a rotation reports success while changing nothing.
  await meshWrote([
    { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" },
  ]);
  const { out, ok } = await provision();
  assert.ok(ok, out);

  const now = await canUse("mesh_workstation", "rotated-secret-cccccccc", "photos");
  assert.ok(now.ok, `the rotated secret does not work:\n${now.out}`);

  const before = await canUse("mesh_workstation", "first-secret-aaaaaaaa", "photos");
  assert.ok(!before.ok, "the secret that was rotated away still works");
});

test("a consumer that goes away loses its key", { skip, timeout: 300_000 }, async () => {
  // The half usually missing. Nothing reports a key that outlives its consumer, and it keeps
  // working for as long as nobody looks.
  //
  // **Stages its own precondition rather than inheriting one.** The first version asserted that
  // `mesh_laptop` was present, having been left by an earlier test — and by then the rotation
  // test had already rewritten the manifest without it, so revocation had happened for the right
  // reason two tests too early. The behaviour was correct and the test was measuring residue.
  await meshWrote([
    { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" },
    { node: "laptop", module: "invoices", bucket: "invoices", secret: "second-secret-bbbbbbbb" },
  ]);
  const staged = await provision();
  assert.ok(staged.ok, staged.out);
  const present = await admin(`admin user list root --json`);
  assert.ok(present.out.includes("mesh_laptop"), `the consumer to be removed was never made:\n${present.out}`);

  await meshWrote([
    { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" },
  ]);
  const { out, ok } = await provision();
  assert.ok(ok, out);

  const after = await admin(`admin user list root --json`);
  assert.ok(!after.out.includes("mesh_laptop"), `a key nobody asks for survived:\n${after.out}`);
  const still = await canUse("mesh_laptop", "second-secret-bbbbbbbb", "invoices");
  assert.ok(!still.ok, "a revoked key still works");
});

test("a key nobody here made is left alone", { skip, timeout: 300_000 }, async () => {
  // A provisioner that removed every key it did not recognise would be one nobody could safely
  // run against a store that predates it (novox/hq 04-ISSUES/010).
  await must(
    `mc --config-dir /tmp/root-mc admin user add root somebody-elses-key somebody-elses-secret`,
  );
  const { out, ok } = await provision();
  assert.ok(ok, out);

  const listed = await admin(`admin user list root --json`);
  assert.ok(listed.out.includes("somebody-elses-key"),
    `a key this provisioner did not make was removed:\n${listed.out}`);
});

test("a manifest naming a credential that was never written is refused", { skip, timeout: 300_000 }, async () => {
  // Refused rather than creating a user with no secret — a login nothing can use, which nothing
  // would report until something tried to connect.
  await meshWrote([
    { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" },
  ]);
  await must(`rm -f ${GRANTS}/workstation.secret`);

  const { out, ok } = await provision();
  assert.ok(!ok, `it carried on without the credential:\n${out}`);
  assert.match(out, /workstation's credential/);
});

test("a bucket name that would not work is refused by name", { skip, timeout: 300_000 }, async () => {
  // The refusal names the consumer that asked. The store would refuse it too, as an error inside
  // a provisioner log with nothing saying whose manifest caused it.
  await meshWrote([
    { node: "workstation", module: "photos", bucket: "Photos_2026", secret: "rotated-secret-cccccccc" },
  ]);
  const { out, ok } = await provision();
  assert.ok(!ok, `an unusable bucket name was accepted:\n${out}`);
  assert.match(out, /workstation asked for a bucket named/);
});