/** * The last step of a credential, against a real object store. * * The mesh generates a secret, seals it to the machine that must accept it, and discards the * plaintext — so it cannot tell the store to start accepting it. Something on that machine reads * what the host wrote and makes it true. This is the step where a secret either becomes a working * key or does not. * * **Phase 1.1 of the work breakdown**, and the finding that shaped it: the control plane * special-cases nothing. `provides`, `requires`, `contributes` and `grants` are name-agnostic, so * asking for a bucket needed no change to the mesh at all — only a provider that answers. What is * proven here is that half. * * **And the half a database does not have.** One PostgreSQL server holds separate databases, and * a role that cannot reach another's is a boundary the product enforces. One object store holds * everybody's buckets behind one endpoint, so a consumer being unable to reach another's is a * policy somebody wrote — which means it is a thing that can be written wrongly, and therefore a * thing to assert rather than assume. */ import { test, after, before } from "node:test"; import assert from "node:assert/strict"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { labIsUsable, destroyAll } from "./harness.ts"; import { incus } from "../../src/incus/client.ts"; import { machineName } from "../../src/lifecycle/names.ts"; const capability = await labIsUsable(); const provisioner = process.env["MESH_LAB_OBJECTSTORE_PROVISIONER"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` : !provisioner ? "set MESH_LAB_OBJECTSTORE_PROVISIONER to a built provisioner " + "(mesh-control: go build ./examples/objectstore-provisioner)" : false; const SCENARIO = "an-object-store"; const MACHINE = "anchor"; const GRANTS = "/var/lib/objectstore/grants"; const ROOT_USER = "meshroot"; const ROOT_PASSWORD = "meshroot-super-secret"; const ROOT_PASSWORD_FILE = "/var/lib/objectstore/root.secret"; const ENDPOINT = "http://127.0.0.1:9000"; let instanceId = ""; /** * The store and the vendor's client, pinned upstream and pulled by the machine over its uplink. * * The store is the digest the mesh's own minio module pins, so this is the store the mesh runs. * Both used to come from a registry the lab raised inside the scenario; no production mesh has * one, so a test that could only fetch from it was proving something about the lab. */ const storeImage = "minio/minio@sha256:aefec8a86702aff0b0dcfdd9284bd7ab7c5631cbf9be63275799e6edcb30dfa2"; const clientImage = "minio/mc:RELEASE.2025-08-13T08-35-41Z"; function shellQuote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } async function on(command: string): Promise<{ out: string; ok: boolean }> { const { stdout } = await exec(instanceId, MACHINE, [ "sh", "-c", `${command} 2>&1; echo "__exit=$?"`, ]); const marker = stdout.lastIndexOf("__exit="); const status = Number(stdout.slice(marker + 7).trim()); return { out: stdout.slice(0, marker), ok: status === 0 }; } /** The same, refusing to continue past a failure nobody would otherwise see. */ async function must(command: string): Promise { const { out, ok } = await on(command); if (!ok) throw new Error(`${command}\n${out}`); return out; } /** `mc` on the machine, against the store as root. */ async function admin(args: string): Promise<{ out: string; ok: boolean }> { return on(`mc --config-dir /tmp/root-mc ${args}`); } /** * Write what the host would have written from a declaration: the manifest of who asked, and one * file per consumer holding its secret alone. * * Written here rather than by running the host, because what is under test is the step *after* * the host — and that the host writes these exact shapes is asserted in its own suite. */ async function meshWrote( consumers: { node: string; module: string; bucket: string; secret: string }[], ): Promise { const manifest = { contributions: consumers.length, requirement: "s3-bucket", generated: "by the mesh", given: consumers.map((c) => ({ from: c.module, node: c.node, secret: `${GRANTS}/${c.node}.${c.module}.secret`, values: { bucket: c.bucket }, })), }; await must(`mkdir -p ${GRANTS}`); await must(`printf %s ${shellQuote(JSON.stringify(manifest))} > ${GRANTS}/mesh.json`); // Every credential file rewritten from nothing, so a removed consumer's does not linger and // make the revocation test pass for a reason that is not the one being tested. await must(`find ${GRANTS} -name '*.secret' -delete`); for (const c of consumers) { await must(`printf %s ${shellQuote(c.secret)} > ${GRANTS}/${c.node}.${c.module}.secret`); await must(`chmod 600 ${GRANTS}/${c.node}.${c.module}.secret`); } } /** The provisioner, as the module shipping the store would run it. */ async function provision(): Promise<{ out: string; ok: boolean }> { return on( `GRANTS=${GRANTS} ` + `MESH_OBJECTSTORE_URL=${ENDPOINT} ` + `MESH_OBJECTSTORE_ROOT_USER=${ROOT_USER} ` + `MESH_OBJECTSTORE_ROOT_PASSWORD_FILE=${ROOT_PASSWORD_FILE} ` + `/usr/local/bin/mesh-provision-objectstore`, ); } /** * Can this key write to and read from this bucket? * * As the consumer, with its own `mc` configuration directory — never the root one. A check made * with the root alias still in scope would pass for any key at all, which is the object-store * shape of the mistake the database suite records: two of its tests once passed without verifying * a password, because they ran where PostgreSQL trusts the caller. */ async function canUse(key: string, secret: string, bucket: string): Promise<{ ok: boolean; out: string }> { const dir = `/tmp/as-${key}`; const { out, ok } = await on( `rm -rf ${dir} && mc --config-dir ${dir} alias set probe ${ENDPOINT} ${shellQuote(key)} ${shellQuote(secret)} && ` + `echo hello > /tmp/probe.txt && ` + `mc --config-dir ${dir} cp /tmp/probe.txt probe/${bucket}/probe.txt && ` + `mc --config-dir ${dir} cat probe/${bucket}/probe.txt`, ); return { ok: ok && out.includes("hello"), out }; } before(async () => { if (skip) return; const scenario = loadScenario(`scenarios/${SCENARIO}.yml`); const instance = await raise(scenario, {}); instanceId = instance.instanceId; // The client, taken out of the vendor's own image onto the machine. The provisioner drives it, // so it has to be here. The machine pulls the image itself, over its uplink, the way it pulls // everything third-party. await must(`docker create --name mc-source ${clientImage}`); await must(`docker cp mc-source:/usr/bin/mc /usr/local/bin/mc && chmod 755 /usr/local/bin/mc`); await must(`docker rm mc-source`); await must(`mkdir -p ${GRANTS}`); await must(`printf %s ${shellQuote(ROOT_PASSWORD)} > ${ROOT_PASSWORD_FILE} && chmod 600 ${ROOT_PASSWORD_FILE}`); await must( `docker run -d --name mesh-store ` + `-e MINIO_ROOT_USER=${ROOT_USER} -e MINIO_ROOT_PASSWORD=${shellQuote(ROOT_PASSWORD)} ` + `-p 127.0.0.1:9000:9000 ${storeImage} server /data`, ); // Ready over the endpoint the provisioner will use, not by the container being up. A store that // is starting answers the port and refuses every operation, which is indistinguishable from a // wrong credential if it is not waited for. let ready = false; for (let i = 0; i < 90 && !ready; i++) { ({ ok: ready } = await on( `mc --config-dir /tmp/root-mc alias set root ${ENDPOINT} ${ROOT_USER} ${shellQuote(ROOT_PASSWORD)}`, )); if (!ready) await new Promise((r) => setTimeout(r, 1000)); } assert.ok(ready, "the store never became ready"); await incus([ "file", "push", provisioner, `${machineName(instanceId, MACHINE)}/usr/local/bin/mesh-provision-objectstore`, "--mode", "0755", ], 180_000); }, { timeout: 1_200_000 }); after(async () => { if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 600_000 }); test("a secret the mesh generated becomes a key that works", { skip, timeout: 300_000 }, async () => { await meshWrote([ { node: "workstation", module: "photos", bucket: "photos", secret: "first-secret-aaaaaaaa" }, ]); const { out, ok } = await provision(); assert.ok(ok, out); const listed = await admin(`admin user list root --json`); assert.ok(listed.out.includes("mesh_workstation_photos"), `no key was made for the consumer:\n${listed.out}`); const used = await canUse("mesh_workstation_photos", "first-secret-aaaaaaaa", "photos"); assert.ok(used.ok, `the consumer cannot use the bucket the mesh gave it:\n${used.out}`); }); test("a consumer cannot reach another consumer's bucket", { skip, timeout: 300_000 }, async () => { // **The assertion this whole scenario exists for.** One store holds every bucket behind one // endpoint, so isolation is a policy rather than a property, and a policy granting // `arn:aws:s3:::*` would pass every other test in this file. await meshWrote([ { node: "workstation", module: "photos", bucket: "photos", secret: "first-secret-aaaaaaaa" }, { node: "laptop", module: "invoices", bucket: "invoices", secret: "second-secret-bbbbbbbb" }, ]); const { out, ok } = await provision(); assert.ok(ok, out); const own = await canUse("mesh_laptop_invoices", "second-secret-bbbbbbbb", "invoices"); assert.ok(own.ok, `a consumer cannot use its own bucket:\n${own.out}`); const other = await canUse("mesh_laptop_invoices", "second-secret-bbbbbbbb", "photos"); assert.ok(!other.ok, `a consumer reached another consumer's bucket:\n${other.out}`); }); test("rotating the secret makes the new one work and the old one stop", { skip, timeout: 300_000 }, async () => { // The failure this guards is a provisioner that only ever creates: the mesh replaces the file, // the user exists, nothing happens, and a rotation reports success while changing nothing. await meshWrote([ { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" }, ]); const { out, ok } = await provision(); assert.ok(ok, out); const now = await canUse("mesh_workstation_photos", "rotated-secret-cccccccc", "photos"); assert.ok(now.ok, `the rotated secret does not work:\n${now.out}`); const before = await canUse("mesh_workstation_photos", "first-secret-aaaaaaaa", "photos"); assert.ok(!before.ok, "the secret that was rotated away still works"); }); test("a consumer that goes away loses its key", { skip, timeout: 300_000 }, async () => { // The half usually missing. Nothing reports a key that outlives its consumer, and it keeps // working for as long as nobody looks. // // **Stages its own precondition rather than inheriting one.** The first version asserted that // `mesh_laptop` was present, having been left by an earlier test — and by then the rotation // test had already rewritten the manifest without it, so revocation had happened for the right // reason two tests too early. The behaviour was correct and the test was measuring residue. await meshWrote([ { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" }, { node: "laptop", module: "invoices", bucket: "invoices", secret: "second-secret-bbbbbbbb" }, ]); const staged = await provision(); assert.ok(staged.ok, staged.out); const present = await admin(`admin user list root --json`); assert.ok(present.out.includes("mesh_laptop_invoices"), `the consumer to be removed was never made:\n${present.out}`); await meshWrote([ { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" }, ]); const { out, ok } = await provision(); assert.ok(ok, out); const after = await admin(`admin user list root --json`); assert.ok(!after.out.includes("mesh_laptop_invoices"), `a key nobody asks for survived:\n${after.out}`); const still = await canUse("mesh_laptop_invoices", "second-secret-bbbbbbbb", "invoices"); assert.ok(!still.ok, "a revoked key still works"); }); test("a key nobody here made is left alone", { skip, timeout: 300_000 }, async () => { // A provisioner that removed every key it did not recognise would be one nobody could safely // run against a store that predates it (novox/hq 04-ISSUES/010). await must( `mc --config-dir /tmp/root-mc admin user add root somebody-elses-key somebody-elses-secret`, ); const { out, ok } = await provision(); assert.ok(ok, out); const listed = await admin(`admin user list root --json`); assert.ok(listed.out.includes("somebody-elses-key"), `a key this provisioner did not make was removed:\n${listed.out}`); }); test("a manifest naming a credential that was never written is refused", { skip, timeout: 300_000 }, async () => { // Refused rather than creating a user with no secret — a login nothing can use, which nothing // would report until something tried to connect. await meshWrote([ { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" }, ]); await must(`rm -f ${GRANTS}/workstation.photos.secret`); const { out, ok } = await provision(); assert.ok(!ok, `it carried on without the credential:\n${out}`); assert.match(out, /workstation's credential/); }); test("a bucket name that would not work is refused by name", { skip, timeout: 300_000 }, async () => { // The refusal names the consumer that asked. The store would refuse it too, as an error inside // a provisioner log with nothing saying whose manifest caused it. await meshWrote([ { node: "workstation", module: "photos", bucket: "Photos_2026", secret: "rotated-secret-cccccccc" }, ]); const { out, ok } = await provision(); assert.ok(!ok, `an unusable bucket name was accepted:\n${out}`); assert.match(out, /workstation asked for a bucket named/); });