Prove a bucket is provisioned the way a database is
Seven assertions against a real store, the important one being that a consumer cannot reach another consumer's bucket — isolation here is a policy somebody wrote rather than a boundary the product has. The revocation test stages its own precondition. The first version asserted a key left by an earlier test, and the rotation test had already revoked it two tests early: the behaviour was correct and the test was measuring residue. Its precondition assertion is what caught that, rather than it passing green having verified nothing.
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
# One machine running an object store that other machines use.
|
||||
#
|
||||
# The same shape as `a-provider`, against a different kind of provision, and that is the whole
|
||||
# reason it exists: novox/hq 04-ISSUES and the work breakdown's Phase 1.1 ask whether a module can
|
||||
# be given a bucket the way it is given a database. The provisioning model is name-agnostic — the
|
||||
# control plane special-cases neither — so what is unproven is not the mesh's half but the last
|
||||
# step, where something on the machine turns a delivered secret into a key that works.
|
||||
#
|
||||
# It also proves the half a database does not: **a consumer must not be able to reach another
|
||||
# consumer's bucket.** One store holds everybody's, where one PostgreSQL server holds separate
|
||||
# databases, so isolation here is a policy somebody wrote rather than a boundary the product has.
|
||||
scenario: an-object-store
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
inbound: allow
|
||||
|
||||
images:
|
||||
- minio/minio:RELEASE.2025-09-07T16-13-09Z
|
||||
# The vendor's client, stocked so the provisioner has the thing it drives without reaching a
|
||||
# public registry from a documentation range.
|
||||
- minio/mc:RELEASE.2025-08-13T08-35-41Z
|
||||
|
||||
place:
|
||||
all: [runtime]
|
||||
@@ -0,0 +1,305 @@
|
||||
/**
|
||||
* The last step of a credential, against a real object store.
|
||||
*
|
||||
* The mesh generates a secret, seals it to the machine that must accept it, and discards the
|
||||
* plaintext — so it cannot tell the store to start accepting it. Something on that machine reads
|
||||
* what the host wrote and makes it true. This is the step where a secret either becomes a working
|
||||
* key or does not.
|
||||
*
|
||||
* **Phase 1.1 of the work breakdown**, and the finding that shaped it: the control plane
|
||||
* special-cases nothing. `provides`, `requires`, `contributes` and `grants` are name-agnostic, so
|
||||
* asking for a bucket needed no change to the mesh at all — only a provider that answers. What is
|
||||
* proven here is that half.
|
||||
*
|
||||
* **And the half a database does not have.** One PostgreSQL server holds separate databases, and
|
||||
* a role that cannot reach another's is a boundary the product enforces. One object store holds
|
||||
* everybody's buckets behind one endpoint, so a consumer being unable to reach another's is a
|
||||
* policy somebody wrote — which means it is a thing that can be written wrongly, and therefore a
|
||||
* thing to assert rather than assume.
|
||||
*/
|
||||
|
||||
import { test, after, before } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { labIsUsable, destroyAll } from "./harness.ts";
|
||||
import { incus } from "../../src/incus/client.ts";
|
||||
import { machineName } from "../../src/lifecycle/names.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const provisioner = process.env["MESH_LAB_OBJECTSTORE_PROVISIONER"] ?? "";
|
||||
const skip = !capability.usable
|
||||
? `lab not usable: ${capability.why}`
|
||||
: !provisioner
|
||||
? "set MESH_LAB_OBJECTSTORE_PROVISIONER to a built provisioner " +
|
||||
"(mesh-control: go build ./examples/objectstore-provisioner)"
|
||||
: false;
|
||||
|
||||
const SCENARIO = "an-object-store";
|
||||
const MACHINE = "anchor";
|
||||
const GRANTS = "/var/lib/objectstore/grants";
|
||||
const ROOT_USER = "meshroot";
|
||||
const ROOT_PASSWORD = "meshroot-super-secret";
|
||||
const ROOT_PASSWORD_FILE = "/var/lib/objectstore/root.secret";
|
||||
const ENDPOINT = "http://127.0.0.1:9000";
|
||||
|
||||
let instanceId = "";
|
||||
/** The store's image, by digest, from the registry the scenario raised. */
|
||||
let storeImage = "";
|
||||
|
||||
function shellQuote(s: string): string {
|
||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
|
||||
async function on(command: string): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, MACHINE, [
|
||||
"sh", "-c", `${command} 2>&1; echo "__exit=$?"`,
|
||||
]);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
const status = Number(stdout.slice(marker + 7).trim());
|
||||
return { out: stdout.slice(0, marker), ok: status === 0 };
|
||||
}
|
||||
|
||||
/** The same, refusing to continue past a failure nobody would otherwise see. */
|
||||
async function must(command: string): Promise<string> {
|
||||
const { out, ok } = await on(command);
|
||||
if (!ok) throw new Error(`${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** `mc` on the machine, against the store as root. */
|
||||
async function admin(args: string): Promise<{ out: string; ok: boolean }> {
|
||||
return on(`mc --config-dir /tmp/root-mc ${args}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write what the host would have written from a declaration: the manifest of who asked, and one
|
||||
* file per consumer holding its secret alone.
|
||||
*
|
||||
* Written here rather than by running the host, because what is under test is the step *after*
|
||||
* the host — and that the host writes these exact shapes is asserted in its own suite.
|
||||
*/
|
||||
async function meshWrote(
|
||||
consumers: { node: string; module: string; bucket: string; secret: string }[],
|
||||
): Promise<void> {
|
||||
const manifest = {
|
||||
contributions: consumers.length,
|
||||
requirement: "s3-bucket",
|
||||
generated: "by the mesh",
|
||||
given: consumers.map((c) => ({
|
||||
from: c.module,
|
||||
node: c.node,
|
||||
secret: `${GRANTS}/${c.node}.secret`,
|
||||
values: { bucket: c.bucket },
|
||||
})),
|
||||
};
|
||||
await must(`mkdir -p ${GRANTS}`);
|
||||
await must(`printf %s ${shellQuote(JSON.stringify(manifest))} > ${GRANTS}/mesh.json`);
|
||||
// Every credential file rewritten from nothing, so a removed consumer's does not linger and
|
||||
// make the revocation test pass for a reason that is not the one being tested.
|
||||
await must(`find ${GRANTS} -name '*.secret' -delete`);
|
||||
for (const c of consumers) {
|
||||
await must(`printf %s ${shellQuote(c.secret)} > ${GRANTS}/${c.node}.secret`);
|
||||
await must(`chmod 600 ${GRANTS}/${c.node}.secret`);
|
||||
}
|
||||
}
|
||||
|
||||
/** The provisioner, as the module shipping the store would run it. */
|
||||
async function provision(): Promise<{ out: string; ok: boolean }> {
|
||||
return on(
|
||||
`GRANTS=${GRANTS} ` +
|
||||
`MESH_OBJECTSTORE_URL=${ENDPOINT} ` +
|
||||
`MESH_OBJECTSTORE_ROOT_USER=${ROOT_USER} ` +
|
||||
`MESH_OBJECTSTORE_ROOT_PASSWORD_FILE=${ROOT_PASSWORD_FILE} ` +
|
||||
`/usr/local/bin/mesh-provision-objectstore`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Can this key write to and read from this bucket?
|
||||
*
|
||||
* As the consumer, with its own `mc` configuration directory — never the root one. A check made
|
||||
* with the root alias still in scope would pass for any key at all, which is the object-store
|
||||
* shape of the mistake the database suite records: two of its tests once passed without verifying
|
||||
* a password, because they ran where PostgreSQL trusts the caller.
|
||||
*/
|
||||
async function canUse(key: string, secret: string, bucket: string): Promise<{ ok: boolean; out: string }> {
|
||||
const dir = `/tmp/as-${key}`;
|
||||
const { out, ok } = await on(
|
||||
`rm -rf ${dir} && mc --config-dir ${dir} alias set probe ${ENDPOINT} ${shellQuote(key)} ${shellQuote(secret)} && ` +
|
||||
`echo hello > /tmp/probe.txt && ` +
|
||||
`mc --config-dir ${dir} cp /tmp/probe.txt probe/${bucket}/probe.txt && ` +
|
||||
`mc --config-dir ${dir} cat probe/${bucket}/probe.txt`,
|
||||
);
|
||||
return { ok: ok && out.includes("hello"), out };
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
const scenario = loadScenario(`scenarios/${SCENARIO}.yml`);
|
||||
const instance = await raise(scenario, {});
|
||||
instanceId = instance.instanceId;
|
||||
|
||||
// From the registry the scenario raised, by digest. There is no route to a public registry from
|
||||
// a documentation range, which is the point of the lab having its own.
|
||||
const store = instance.images.find((r) => r.includes("minio/minio"));
|
||||
const client = instance.images.find((r) => r.includes("minio/mc"));
|
||||
assert.ok(store, `the scenario stocked no store image: ${instance.images.join(", ")}`);
|
||||
assert.ok(client, `the scenario stocked no client image: ${instance.images.join(", ")}`);
|
||||
storeImage = store;
|
||||
|
||||
// The client, taken out of the vendor's own image onto the machine. The provisioner drives it,
|
||||
// so it has to be here — and taking it from the stocked image is what keeps this test off any
|
||||
// public network.
|
||||
await must(`docker create --name mc-source ${client}`);
|
||||
await must(`docker cp mc-source:/usr/bin/mc /usr/local/bin/mc && chmod 755 /usr/local/bin/mc`);
|
||||
await must(`docker rm mc-source`);
|
||||
|
||||
await must(`mkdir -p ${GRANTS}`);
|
||||
await must(`printf %s ${shellQuote(ROOT_PASSWORD)} > ${ROOT_PASSWORD_FILE} && chmod 600 ${ROOT_PASSWORD_FILE}`);
|
||||
|
||||
await must(
|
||||
`docker run -d --name mesh-store ` +
|
||||
`-e MINIO_ROOT_USER=${ROOT_USER} -e MINIO_ROOT_PASSWORD=${shellQuote(ROOT_PASSWORD)} ` +
|
||||
`-p 127.0.0.1:9000:9000 ${storeImage} server /data`,
|
||||
);
|
||||
|
||||
// Ready over the endpoint the provisioner will use, not by the container being up. A store that
|
||||
// is starting answers the port and refuses every operation, which is indistinguishable from a
|
||||
// wrong credential if it is not waited for.
|
||||
let ready = false;
|
||||
for (let i = 0; i < 90 && !ready; i++) {
|
||||
({ ok: ready } = await on(
|
||||
`mc --config-dir /tmp/root-mc alias set root ${ENDPOINT} ${ROOT_USER} ${shellQuote(ROOT_PASSWORD)}`,
|
||||
));
|
||||
if (!ready) await new Promise((r) => setTimeout(r, 1000));
|
||||
}
|
||||
assert.ok(ready, "the store never became ready");
|
||||
|
||||
await incus([
|
||||
"file", "push", provisioner,
|
||||
`${machineName(instanceId, MACHINE)}/usr/local/bin/mesh-provision-objectstore`,
|
||||
"--mode", "0755",
|
||||
], 180_000);
|
||||
}, { timeout: 1_200_000 });
|
||||
|
||||
after(async () => {
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 600_000 });
|
||||
|
||||
test("a secret the mesh generated becomes a key that works", { skip, timeout: 300_000 }, async () => {
|
||||
await meshWrote([
|
||||
{ node: "workstation", module: "photos", bucket: "photos", secret: "first-secret-aaaaaaaa" },
|
||||
]);
|
||||
const { out, ok } = await provision();
|
||||
assert.ok(ok, out);
|
||||
|
||||
const listed = await admin(`admin user list root --json`);
|
||||
assert.ok(listed.out.includes("mesh_workstation"), `no key was made for the consumer:\n${listed.out}`);
|
||||
|
||||
const used = await canUse("mesh_workstation", "first-secret-aaaaaaaa", "photos");
|
||||
assert.ok(used.ok, `the consumer cannot use the bucket the mesh gave it:\n${used.out}`);
|
||||
});
|
||||
|
||||
test("a consumer cannot reach another consumer's bucket", { skip, timeout: 300_000 }, async () => {
|
||||
// **The assertion this whole scenario exists for.** One store holds every bucket behind one
|
||||
// endpoint, so isolation is a policy rather than a property, and a policy granting
|
||||
// `arn:aws:s3:::*` would pass every other test in this file.
|
||||
await meshWrote([
|
||||
{ node: "workstation", module: "photos", bucket: "photos", secret: "first-secret-aaaaaaaa" },
|
||||
{ node: "laptop", module: "invoices", bucket: "invoices", secret: "second-secret-bbbbbbbb" },
|
||||
]);
|
||||
const { out, ok } = await provision();
|
||||
assert.ok(ok, out);
|
||||
|
||||
const own = await canUse("mesh_laptop", "second-secret-bbbbbbbb", "invoices");
|
||||
assert.ok(own.ok, `a consumer cannot use its own bucket:\n${own.out}`);
|
||||
|
||||
const other = await canUse("mesh_laptop", "second-secret-bbbbbbbb", "photos");
|
||||
assert.ok(!other.ok, `a consumer reached another consumer's bucket:\n${other.out}`);
|
||||
});
|
||||
|
||||
test("rotating the secret makes the new one work and the old one stop", { skip, timeout: 300_000 }, async () => {
|
||||
// The failure this guards is a provisioner that only ever creates: the mesh replaces the file,
|
||||
// the user exists, nothing happens, and a rotation reports success while changing nothing.
|
||||
await meshWrote([
|
||||
{ node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" },
|
||||
]);
|
||||
const { out, ok } = await provision();
|
||||
assert.ok(ok, out);
|
||||
|
||||
const now = await canUse("mesh_workstation", "rotated-secret-cccccccc", "photos");
|
||||
assert.ok(now.ok, `the rotated secret does not work:\n${now.out}`);
|
||||
|
||||
const before = await canUse("mesh_workstation", "first-secret-aaaaaaaa", "photos");
|
||||
assert.ok(!before.ok, "the secret that was rotated away still works");
|
||||
});
|
||||
|
||||
test("a consumer that goes away loses its key", { skip, timeout: 300_000 }, async () => {
|
||||
// The half usually missing. Nothing reports a key that outlives its consumer, and it keeps
|
||||
// working for as long as nobody looks.
|
||||
//
|
||||
// **Stages its own precondition rather than inheriting one.** The first version asserted that
|
||||
// `mesh_laptop` was present, having been left by an earlier test — and by then the rotation
|
||||
// test had already rewritten the manifest without it, so revocation had happened for the right
|
||||
// reason two tests too early. The behaviour was correct and the test was measuring residue.
|
||||
await meshWrote([
|
||||
{ node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" },
|
||||
{ node: "laptop", module: "invoices", bucket: "invoices", secret: "second-secret-bbbbbbbb" },
|
||||
]);
|
||||
const staged = await provision();
|
||||
assert.ok(staged.ok, staged.out);
|
||||
const present = await admin(`admin user list root --json`);
|
||||
assert.ok(present.out.includes("mesh_laptop"), `the consumer to be removed was never made:\n${present.out}`);
|
||||
|
||||
await meshWrote([
|
||||
{ node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" },
|
||||
]);
|
||||
const { out, ok } = await provision();
|
||||
assert.ok(ok, out);
|
||||
|
||||
const after = await admin(`admin user list root --json`);
|
||||
assert.ok(!after.out.includes("mesh_laptop"), `a key nobody asks for survived:\n${after.out}`);
|
||||
const still = await canUse("mesh_laptop", "second-secret-bbbbbbbb", "invoices");
|
||||
assert.ok(!still.ok, "a revoked key still works");
|
||||
});
|
||||
|
||||
test("a key nobody here made is left alone", { skip, timeout: 300_000 }, async () => {
|
||||
// A provisioner that removed every key it did not recognise would be one nobody could safely
|
||||
// run against a store that predates it (novox/hq 04-ISSUES/010).
|
||||
await must(
|
||||
`mc --config-dir /tmp/root-mc admin user add root somebody-elses-key somebody-elses-secret`,
|
||||
);
|
||||
const { out, ok } = await provision();
|
||||
assert.ok(ok, out);
|
||||
|
||||
const listed = await admin(`admin user list root --json`);
|
||||
assert.ok(listed.out.includes("somebody-elses-key"),
|
||||
`a key this provisioner did not make was removed:\n${listed.out}`);
|
||||
});
|
||||
|
||||
test("a manifest naming a credential that was never written is refused", { skip, timeout: 300_000 }, async () => {
|
||||
// Refused rather than creating a user with no secret — a login nothing can use, which nothing
|
||||
// would report until something tried to connect.
|
||||
await meshWrote([
|
||||
{ node: "workstation", module: "photos", bucket: "photos", secret: "rotated-secret-cccccccc" },
|
||||
]);
|
||||
await must(`rm -f ${GRANTS}/workstation.secret`);
|
||||
|
||||
const { out, ok } = await provision();
|
||||
assert.ok(!ok, `it carried on without the credential:\n${out}`);
|
||||
assert.match(out, /workstation's credential/);
|
||||
});
|
||||
|
||||
test("a bucket name that would not work is refused by name", { skip, timeout: 300_000 }, async () => {
|
||||
// The refusal names the consumer that asked. The store would refuse it too, as an error inside
|
||||
// a provisioner log with nothing saying whose manifest caused it.
|
||||
await meshWrote([
|
||||
{ node: "workstation", module: "photos", bucket: "Photos_2026", secret: "rotated-secret-cccccccc" },
|
||||
]);
|
||||
const { out, ok } = await provision();
|
||||
assert.ok(!ok, `an unusable bucket name was accepted:\n${out}`);
|
||||
assert.match(out, /workstation asked for a bucket named/);
|
||||
});
|
||||
Reference in New Issue
Block a user