diff --git a/scenarios/an-object-store.yml b/scenarios/an-object-store.yml new file mode 100644 index 0000000..7145f21 --- /dev/null +++ b/scenarios/an-object-store.yml @@ -0,0 +1,31 @@ +# One machine running an object store that other machines use. +# +# The same shape as `a-provider`, against a different kind of provision, and that is the whole +# reason it exists: novox/hq 04-ISSUES and the work breakdown's Phase 1.1 ask whether a module can +# be given a bucket the way it is given a database. The provisioning model is name-agnostic — the +# control plane special-cases neither — so what is unproven is not the mesh's half but the last +# step, where something on the machine turns a delivered secret into a key that works. +# +# It also proves the half a database does not: **a consumer must not be able to reach another +# consumer's bucket.** One store holds everybody's, where one PostgreSQL server holds separate +# databases, so isolation here is a policy somebody wrote rather than a boundary the product has. +scenario: an-object-store + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + +images: + - minio/minio:RELEASE.2025-09-07T16-13-09Z + # The vendor's client, stocked so the provisioner has the thing it drives without reaching a + # public registry from a documentation range. + - minio/mc:RELEASE.2025-08-13T08-35-41Z + +place: + all: [runtime] diff --git a/test/integration/objectstore.test.ts b/test/integration/objectstore.test.ts new file mode 100644 index 0000000..41b2f47 --- /dev/null +++ b/test/integration/objectstore.test.ts @@ -0,0 +1,305 @@ +/** + * The last step of a credential, against a real object store. + * + * The mesh generates a secret, seals it to the machine that must accept it, and discards the + * plaintext — so it cannot tell the store to start accepting it. Something on that machine reads + * what the host wrote and makes it true. This is the step where a secret either becomes a working + * key or does not. + * + * **Phase 1.1 of the work breakdown**, and the finding that shaped it: the control plane + * special-cases nothing. `provides`, `requires`, `contributes` and `grants` are name-agnostic, so + * asking for a bucket needed no change to the mesh at all — only a provider that answers. What is + * proven here is that half. + * + * **And the half a database does not have.** One PostgreSQL server holds separate databases, and + * a role that cannot reach another's is a boundary the product enforces. One object store holds + * everybody's buckets behind one endpoint, so a consumer being unable to reach another's is a + * policy somebody wrote — which means it is a thing that can be written wrongly, and therefore a + * thing to assert rather than assume. + */ + +import { test, after, before } from "node:test"; +import assert from "node:assert/strict"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { labIsUsable, destroyAll } from "./harness.ts"; +import { incus } from "../../src/incus/client.ts"; +import { machineName } from "../../src/lifecycle/names.ts"; + +const capability = await labIsUsable(); +const provisioner = process.env["MESH_LAB_OBJECTSTORE_PROVISIONER"] ?? ""; +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !provisioner + ? "set MESH_LAB_OBJECTSTORE_PROVISIONER to a built provisioner " + + "(mesh-control: go build ./examples/objectstore-provisioner)" + : false; + +const SCENARIO = "an-object-store"; +const MACHINE = "anchor"; +const GRANTS = "/var/lib/objectstore/grants"; +const ROOT_USER = "meshroot"; +const ROOT_PASSWORD = "meshroot-super-secret"; +const ROOT_PASSWORD_FILE = "/var/lib/objectstore/root.secret"; +const ENDPOINT = "http://127.0.0.1:9000"; + +let instanceId = ""; +/** The store's image, by digest, from the registry the scenario raised. */ +let storeImage = ""; + +function shellQuote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(command: string): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, MACHINE, [ + "sh", "-c", `${command} 2>&1; echo "__exit=$?"`, + ]); + const marker = stdout.lastIndexOf("__exit="); + const status = Number(stdout.slice(marker + 7).trim()); + return { out: stdout.slice(0, marker), ok: status === 0 }; +} + +/** The same, refusing to continue past a failure nobody would otherwise see. */ +async function must(command: string): Promise { + const { out, ok } = await on(command); + if (!ok) throw new Error(`${command}\n${out}`); + return out; +} + +/** `mc` on the machine, against the store as root. */ +async function admin(args: string): Promise<{ out: string; ok: boolean }> { + return on(`mc --config-dir /tmp/root-mc ${args}`); +} + +/** + * Write what the host would have written from a declaration: the manifest of who asked, and one + * file per consumer holding its secret alone. + * + * Written here rather than by running the host, because what is under test is the step *after* + * the host — and that the host writes these exact shapes is asserted in its own suite. + */ +async function meshWrote( + consumers: { node: string; module: string; bucket: string; secret: string }[], +): Promise { + const manifest = { + contributions: consumers.length, + requirement: "s3-bucket", + generated: "by the mesh", + given: consumers.map((c) => ({ + from: c.module, + node: c.node, + secret: `${GRANTS}/${c.node}.secret`, + values: { bucket: c.bucket }, + })), + }; + await must(`mkdir -p ${GRANTS}`); + await must(`printf %s ${shellQuote(JSON.stringify(manifest))} > ${GRANTS}/mesh.json`); + // Every credential file rewritten from nothing, so a removed consumer's does not linger and + // make the revocation test pass for a reason that is not the one being tested. + await must(`find ${GRANTS} -name '*.secret' -delete`); + for (const c of consumers) { + await must(`printf %s ${shellQuote(c.secret)} > ${GRANTS}/${c.node}.secret`); + await must(`chmod 600 ${GRANTS}/${c.node}.secret`); + } +} + +/** The provisioner, as the module shipping the store would run it. */ +async function provision(): Promise<{ out: string; ok: boolean }> { + return on( + `GRANTS=${GRANTS} ` + + `MESH_OBJECTSTORE_URL=${ENDPOINT} ` + + `MESH_OBJECTSTORE_ROOT_USER=${ROOT_USER} ` + + `MESH_OBJECTSTORE_ROOT_PASSWORD_FILE=${ROOT_PASSWORD_FILE} ` + + `/usr/local/bin/mesh-provision-objectstore`, + ); +} + +/** + * Can this key write to and read from this bucket? + * + * As the consumer, with its own `mc` configuration directory — never the root one. A check made + * with the root alias still in scope would pass for any key at all, which is the object-store + * shape of the mistake the database suite records: two of its tests once passed without verifying + * a password, because they ran where PostgreSQL trusts the caller. + */ +async function canUse(key: string, secret: string, bucket: string): Promise<{ ok: boolean; out: string }> { + const dir = `/tmp/as-${key}`; + const { out, ok } = await on( + `rm -rf ${dir} && mc --config-dir ${dir} alias set probe ${ENDPOINT} ${shellQuote(key)} ${shellQuote(secret)} && ` + + `echo hello > /tmp/probe.txt && ` + + `mc --config-dir ${dir} cp /tmp/probe.txt probe/${bucket}/probe.txt && ` + + `mc --config-dir ${dir} cat probe/${bucket}/probe.txt`, + ); + return { ok: ok && out.includes("hello"), out }; +} + +before(async () => { + if (skip) return; + const scenario = loadScenario(`scenarios/${SCENARIO}.yml`); + const instance = await raise(scenario, {}); + instanceId = instance.instanceId; + + // From the registry the scenario raised, by digest. There is no route to a public registry from + // a documentation range, which is the point of the lab having its own. + const store = instance.images.find((r) => r.includes("minio/minio")); + const client = instance.images.find((r) => r.includes("minio/mc")); + assert.ok(store, `the scenario stocked no store image: ${instance.images.join(", ")}`); + assert.ok(client, `the scenario stocked no client image: ${instance.images.join(", ")}`); + storeImage = store; + + // The client, taken out of the vendor's own image onto the machine. The provisioner drives it, + // so it has to be here — and taking it from the stocked image is what keeps this test off any + // public network. + await must(`docker create --name mc-source ${client}`); + await must(`docker cp mc-source:/usr/bin/mc /usr/local/bin/mc && chmod 755 /usr/local/bin/mc`); + await must(`docker rm mc-source`); + + await must(`mkdir -p ${GRANTS}`); + await must(`printf %s ${shellQuote(ROOT_PASSWORD)} > ${ROOT_PASSWORD_FILE} && chmod 600 ${ROOT_PASSWORD_FILE}`); + + await must( + `docker run -d --name mesh-store ` + + `-e MINIO_ROOT_USER=${ROOT_USER} -e MINIO_ROOT_PASSWORD=${shellQuote(ROOT_PASSWORD)} ` + + `-p 127.0.0.1:9000:9000 ${storeImage} server /data`, + ); + + // Ready over the endpoint the provisioner will use, not by the container being up. A store that + // is starting answers the port and refuses every operation, which is indistinguishable from a + // wrong credential if it is not waited for. + let ready = false; + for (let i = 0; i < 90 && !ready; i++) { + ({ ok: ready } = await on( + `mc --config-dir /tmp/root-mc alias set root ${ENDPOINT} ${ROOT_USER} ${shellQuote(ROOT_PASSWORD)}`, + )); + if (!ready) await new Promise((r) => setTimeout(r, 1000)); + } + assert.ok(ready, "the store never became ready"); + + await incus([ + "file", "push", provisioner, + `${machineName(instanceId, MACHINE)}/usr/local/bin/mesh-provision-objectstore`, + "--mode", "0755", + ], 180_000); +}, { timeout: 1_200_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 600_000 }); + +test("a secret the mesh generated becomes a key that works", { skip, timeout: 300_000 }, async () => { + await meshWrote([ + { node: "workstation", module: "photos", bucket: "photos", secret: "first-secret-aaaaaaaa" }, + ]); + const { out, ok } = await provision(); + assert.ok(ok, out); + + const listed = await admin(`admin user list root --json`); + assert.ok(listed.out.includes("mesh_workstation"), `no key was made for the consumer:\n${listed.out}`); + + const used = await canUse("mesh_workstation", "first-secret-aaaaaaaa", "photos"); + assert.ok(used.ok, `the consumer cannot use the bucket the mesh gave it:\n${used.out}`); +}); + +test("a consumer cannot reach another consumer's bucket", { skip, timeout: 300_000 }, async () => { + // **The assertion this whole scenario exists for.** One store holds every bucket behind one + // endpoint, so isolation is a policy rather than a property, and a policy granting + // `arn:aws:s3:::*` would pass every other test in this file. + await meshWrote([ + { node: "workstation", module: "photos", bucket: "photos", secret: "first-secret-aaaaaaaa" }, + { node: "laptop", module: "invoices", bucket: "invoices", secret: "second-secret-bbbbbbbb" }, + ]); + const { out, ok } = await provision(); + assert.ok(ok, out); + + const own = await canUse("mesh_laptop", "second-secret-bbbbbbbb", "invoices"); + assert.ok(own.ok, `a consumer cannot use its own bucket:\n${own.out}`); + + const other = await canUse("mesh_laptop", "second-secret-bbbbbbbb", "photos"); + assert.ok(!other.ok, `a consumer reached another consumer's bucket:\n${other.out}`); +}); + +test("rotating the secret makes the new one work and the old one stop", { skip, timeout: 300_000 }, async () => { + // The failure this guards is a provisioner that only ever creates: the mesh replaces the file, + // the user exists, nothing happens, and a rotation reports success while changing nothing. + await meshWrote([ + { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" }, + ]); + const { out, ok } = await provision(); + assert.ok(ok, out); + + const now = await canUse("mesh_workstation", "rotated-secret-cccccccc", "photos"); + assert.ok(now.ok, `the rotated secret does not work:\n${now.out}`); + + const before = await canUse("mesh_workstation", "first-secret-aaaaaaaa", "photos"); + assert.ok(!before.ok, "the secret that was rotated away still works"); +}); + +test("a consumer that goes away loses its key", { skip, timeout: 300_000 }, async () => { + // The half usually missing. Nothing reports a key that outlives its consumer, and it keeps + // working for as long as nobody looks. + // + // **Stages its own precondition rather than inheriting one.** The first version asserted that + // `mesh_laptop` was present, having been left by an earlier test — and by then the rotation + // test had already rewritten the manifest without it, so revocation had happened for the right + // reason two tests too early. The behaviour was correct and the test was measuring residue. + await meshWrote([ + { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" }, + { node: "laptop", module: "invoices", bucket: "invoices", secret: "second-secret-bbbbbbbb" }, + ]); + const staged = await provision(); + assert.ok(staged.ok, staged.out); + const present = await admin(`admin user list root --json`); + assert.ok(present.out.includes("mesh_laptop"), `the consumer to be removed was never made:\n${present.out}`); + + await meshWrote([ + { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" }, + ]); + const { out, ok } = await provision(); + assert.ok(ok, out); + + const after = await admin(`admin user list root --json`); + assert.ok(!after.out.includes("mesh_laptop"), `a key nobody asks for survived:\n${after.out}`); + const still = await canUse("mesh_laptop", "second-secret-bbbbbbbb", "invoices"); + assert.ok(!still.ok, "a revoked key still works"); +}); + +test("a key nobody here made is left alone", { skip, timeout: 300_000 }, async () => { + // A provisioner that removed every key it did not recognise would be one nobody could safely + // run against a store that predates it (novox/hq 04-ISSUES/010). + await must( + `mc --config-dir /tmp/root-mc admin user add root somebody-elses-key somebody-elses-secret`, + ); + const { out, ok } = await provision(); + assert.ok(ok, out); + + const listed = await admin(`admin user list root --json`); + assert.ok(listed.out.includes("somebody-elses-key"), + `a key this provisioner did not make was removed:\n${listed.out}`); +}); + +test("a manifest naming a credential that was never written is refused", { skip, timeout: 300_000 }, async () => { + // Refused rather than creating a user with no secret — a login nothing can use, which nothing + // would report until something tried to connect. + await meshWrote([ + { node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" }, + ]); + await must(`rm -f ${GRANTS}/workstation.secret`); + + const { out, ok } = await provision(); + assert.ok(!ok, `it carried on without the credential:\n${out}`); + assert.match(out, /workstation's credential/); +}); + +test("a bucket name that would not work is refused by name", { skip, timeout: 300_000 }, async () => { + // The refusal names the consumer that asked. The store would refuse it too, as an error inside + // a provisioner log with nothing saying whose manifest caused it. + await meshWrote([ + { node: "workstation", module: "photos", bucket: "Photos_2026", secret: "rotated-secret-cccccccc" }, + ]); + const { out, ok } = await provision(); + assert.ok(!ok, `an unusable bucket name was accepted:\n${out}`); + assert.match(out, /workstation asked for a bucket named/); +});