diff --git a/scenarios/vault-node.yml b/scenarios/vault-node.yml new file mode 100644 index 0000000..1b6be51 --- /dev/null +++ b/scenarios/vault-node.yml @@ -0,0 +1,32 @@ +# One machine that becomes a mesh and then assigns itself mesh-vault and redis — a provider whose own +# password is a `secret` the vault provides (novox/hq ADR 0085, design 24). +# +# redis-node proves a provider's runtime. This proves the vault: redis requires `secret`, the mesh +# mints the pair credential, the host fills redis.conf from it, redis authenticates with it, and the +# vault's ledger records its fingerprint. Then `rotate secret` moves both ends: the new password +# works, the old one is refused, and the vault says it was rotated — design 13's three logins, for a +# secret that had no owner before. +scenario: vault-node + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + +images: + - mesh-controller:development + # Built by scripts/build-module-runtime.sh mesh-vault / redis into the local daemon; the machine holds + # each by its own image ID. redis's server image is pulled upstream by digest. + - mesh-runtime-mesh-vault:development + - mesh-runtime-redis:development + +place: + all: [host, runtime] diff --git a/test/integration/assigned-vault.test.ts b/test/integration/assigned-vault.test.ts new file mode 100644 index 0000000..918cd7d --- /dev/null +++ b/test/integration/assigned-vault.test.ts @@ -0,0 +1,424 @@ +/** + * A module's own secret, provided by the vault — and rotated like any other credential. + * + * novox/hq ADR 0085: a secret a module needs for itself is a `secret` provision from a vault module, + * and the credential of that consumer↔vault pair IS the value. So everything design 13 built for a + * database password applies to it unchanged, and this bed checks exactly what 13 checks: three + * logins against the real software, from the machine that holds the secret — + * + * 1. the delivered password authenticates against redis + * 2. after `rotate secret`, the new one authenticates + * 3. the one that was rotated away does not + * + * — plus the half that makes the vault an owner rather than a bystander: its ledger records who + * holds the secret and its fingerprint, notices the rotation, and answers over the mesh without ever + * returning a value. + * + * And the amended half of ADR 0085: the mesh has an operator key, every secret a module holds for + * itself is sealed to it as well, the vault keeps that export on its disk, and the operator — with + * the private key that never entered the mesh — recovers exactly the value the node was given. + * Checked against the value the host unsealed on the machine, and with the store out of the loop. + * + * The manifests are the catalogue's own (../mesh-catalog/modules/{mesh-vault,redis}/module.json), with + * the runtime artifact named as the image the lab built, exactly as the other assigned-* beds do. + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock + * scripts/build-module-runtime.sh mesh-vault / redis build the two runtime images into the local + * daemon; scenarios/vault-node.yml stocks them. + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const catalogue = process.env["MESH_LAB_CATALOG"] ?? "../mesh-catalog"; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" + : !existsSync(join(catalogue, "modules/mesh-vault/module.json")) + ? `no catalogue at ${catalogue} (set MESH_LAB_CATALOG)` + : false; + +const SCENARIO = "vault-node"; +const MACHINE = "anchor"; +const SECRET_FILE = "/var/lib/redis-module/default.secret"; +const LEDGER = "/var/lib/mesh-vault/ledger"; +const ROOT = "/var/lib/mesh-vault/root"; +// Where the operator keeps things on this machine: outside every container, bind-mounted into a +// throwaway one when the operator's own commands run. The controller's container is a scratch +// image with no writable path — the private key could not be written there even by mistake. +const OPERATOR_DIR = "/var/lib/lab-operator"; + +let instanceId = ""; +let held: HeldImage[] = []; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, MACHINE, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(command, timeoutMs); + if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`); + return out; +} + +async function mesh(command: string, timeoutMs?: number): Promise { + return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); +} + +function pinned(reference: string): string { + return onTheMachine(reference, held); +} + +function bundleFor(images: HeldImage[]): string { + return foundationBundle(bundle, images); +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +/** sha256:, the only form in which a secret's value is ever spoken here. */ +function fingerprint(value: string): string { + return "sha256:" + createHash("sha256").update(value, "utf8").digest("hex"); +} + +/** + * The catalogue's own manifest, as the lab can run it: the runtime artifact becomes the image this + * scenario loaded, third-party images are pinned upstream, and `build` goes — nothing here builds. + */ +function catalogueManifest(name: string): string { + const m = JSON.parse(readFileSync(join(catalogue, `modules/${name}/module.json`), "utf8")) as { + build?: unknown; + resources: Record[]; + }; + delete m.build; + for (const r of m.resources) { + if (r["artifact"] === "runtime") { + delete r["artifact"]; + r["image"] = pinned(`mesh-runtime-${name}`); + } else if (typeof r["image"] === "string") { + r["image"] = pinned(r["image"]); + } + } + return JSON.stringify(m); +} + +async function until(what: string, ms: number, probe: () => Promise): Promise { + const deadline = Date.now() + ms; + let last: unknown; + while (Date.now() < deadline) { + try { + const got = await probe(); + if (got !== undefined) return got; + } catch (err) { + last = err; + } + await new Promise((r) => setTimeout(r, 3000)); + } + throw new Error(`${what} did not happen within ${Math.round(ms / 1000)}s${last ? `; last: ${String(last)}` : ""}`); +} + +async function settled(withinMs = 480_000): Promise { + const deadline = Date.now() + withinMs; + let last = ""; + while (Date.now() < deadline) { + const asked = await on(`docker exec mesh-controller /mesh-controller status --json`); + if (asked.ok) { + try { + const state = JSON.parse(asked.out) as { + wrong: { node: string; outcome: string }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + const bad = state.wrong.find((w) => w.node === MACHINE); + if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`); + const word = state.reported.find((r) => r.node === MACHINE); + if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return; + last = asked.out; + } catch (err) { + if (err instanceof Error && err.message.includes("did not apply")) throw err; + last = asked.out; + } + } + await new Promise((r) => setTimeout(r, 5000)); + } + throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`); +} + +/** redis's answer to PING as `password`, from inside its own container — PONG, or why not. */ +async function pingAs(password: string): Promise { + return (await on(`docker exec redis redis-cli --no-auth-warning -a ${quote(password)} ping`)).out.trim(); +} + +/** The operator's own commands: the controller binary in a throwaway container, with the operator's + * directory at /work and nothing of the mesh in reach — no store, no broker. What an operator runs + * on their own machine with the binary and a directory. */ +async function operator(command: string): Promise { + return must( + `docker run --rm -v ${OPERATOR_DIR}:/work --entrypoint /mesh-controller ${pinned("mesh-controller")} ${command}`, + 120_000, + ); +} + +/** Invoke one of the vault's tools over the mesh, the way a caller would, and parse its answer. */ +async function vaultTool(tool: string, args: Record = {}): Promise> { + const said = await must( + `docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` + + `${pinned("mesh-runtime-mesh-vault")} invoke mesh-vault ${tool} ${quote(JSON.stringify(args))}`, + 120_000, + ); + const line = said.split("\n").map((l) => l.trim()).filter(Boolean).pop() ?? ""; + return JSON.parse(line) as Record; +} + +interface Held { + as: string; + consumer: string; + fingerprint: string; + rotations: number; + history: { fingerprint: string; until: string }[]; +} + +before(async () => { + if (skip) return; + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + held = raised.images; + + await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); + await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000); + const up = await must(`docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { + assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`); + } + + await mesh(`node add ${MACHINE}`); + const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`)); + await must(`${HOST_PATH} enrol --token ${quote(token)}`); + await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); +}, { timeout: 1_800_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 600_000 }); + +let delivered = ""; +let holder = ""; + +test("redis's own password is a secret the vault provides: it authenticates, and the vault holds its fingerprint", { + skip, timeout: 900_000, +}, async () => { + // The operator key first, so everything minted from here on is sealed to it as well. Made off + // the mesh — a throwaway container with the operator's directory mounted, no store in reach — + // and only the public half is given to the mesh. + await must(`mkdir -p ${OPERATOR_DIR} && chmod 777 ${OPERATOR_DIR}`); + const made = await operator(`operator key make --out /work/operator.key`); + assert.match(made, /operator key sha256:/, made); + const publicKey = made.split("\n").map((l) => l.trim()).map((l) => /^public ([A-Za-z0-9+\/]{40,}={0,2})$/.exec(l)?.[1]).find(Boolean); + assert.ok(publicKey, `no public key in:\n${made}`); + const privateKey = (await must(`cat ${OPERATOR_DIR}/operator.key`)).trim(); + assert.ok(privateKey.length > 30 && !made.includes(privateKey), "the private key was printed"); + const set = await mesh(`operator key set ${quote(publicKey)}`); + assert.match(set, /operator key sha256:/, set); + + for (const name of ["mesh-vault", "redis"]) { + await must(`printf %s ${quote(catalogueManifest(name))} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`); + await mesh(`module add /${name}.json`); + const issued = await mesh(`module issue ${name} --node ${MACHINE}`); + assert.match(issued, /scoped to what it emits and consumes/, issued); + await mesh(`assign ${MACHINE} ${name}`); + } + await mesh(`push ${MACHINE}`); + await settled(); + + const running = await must(`docker ps --format '{{.Names}}'`); + for (const c of ["mesh-vault", "redis", "mesh-redis"]) { + assert.match(running, new RegExp(`(^|\\n)${c}(\\n|$)`), + `${c} is not running:\n${running}\n---host log---\n${(await on(`tail -40 /var/log/mesh-host.log`)).out}`); + } + + // 1. The pair credential the mesh minted is on redis's machine as redis's own password — and + // redis accepts it. Not because two files agree: because the login works. + delivered = (await must(`cat ${SECRET_FILE}`)).replace(/\n$/, ""); + assert.ok(delivered.length >= 20, "the delivered secret is empty or implausibly short"); + const mode = (await must(`stat -c %a ${SECRET_FILE}`)).trim(); + assert.equal(mode, "600", `the secret file is readable by others (mode ${mode})`); + const pong = await until("redis accepting the delivered password", 120_000, async () => { + const said = await pingAs(delivered); + return said === "PONG" ? said : undefined; + }); + assert.equal(pong, "PONG"); + assert.notEqual(await pingAs("not-the-password"), "PONG", "redis accepted a wrong password — requirepass is not set"); + + // 2. The vault took custody: its ledger names the holder, with the fingerprint of exactly the + // value redis holds — and no value. + const recorded = await until("the vault recording redis's secret", 90_000, async () => { + const ls = await on(`ls ${LEDGER}`); + const name = ls.ok ? ls.out.split("\n").map((l) => l.trim()).find((l) => l.endsWith(".json")) : undefined; + if (!name) return undefined; + return JSON.parse(await must(`cat ${LEDGER}/${name}`)) as Held; + }); + holder = recorded.as; + assert.equal(recorded.consumer, MACHINE); + assert.equal(recorded.fingerprint, fingerprint(delivered), "the vault recorded a fingerprint of something other than what redis holds"); + assert.equal(recorded.rotations, 0); + const ledgerRaw = await must(`cat ${LEDGER}/*.json`); + assert.ok(!ledgerRaw.includes(delivered), "the vault's ledger contains the secret's value"); + // The holder is the login the mesh derived for redis on this machine, as the contributions file says. + const asked = JSON.parse(await must(`cat /var/lib/mesh-vault/grants/mesh.json`)) as { given: { as: string; from?: string }[] }; + assert.ok(asked.given.some((g) => g.as === holder), `the vault was not asked to serve ${holder}:\n${JSON.stringify(asked)}`); + + // 3. And it answers over the mesh — who holds what, verified by fingerprint, never by value. + const holders = await until("the vault serving its tools", 90_000, async () => { + const got = await vaultTool("secret_holders"); + return Array.isArray(got["holders"]) && (got["holders"] as Held[]).length > 0 ? got : undefined; + }); + const mine = (holders["holders"] as (Held & { module: string | null })[]).find((h) => h.as === holder); + assert.ok(mine, `secret_holders does not list ${holder}:\n${JSON.stringify(holders)}`); + assert.equal(mine.module, "redis"); + assert.ok(!JSON.stringify(holders).includes(delivered), "secret_holders returned the value"); + + const verified = await vaultTool("secret_verify", { as: holder, fingerprint: fingerprint(delivered) }); + assert.equal(verified["ok"], true, JSON.stringify(verified)); + assert.equal(verified["deliveredMatchesRecorded"], true, JSON.stringify(verified)); + assert.equal(verified["givenMatchesRecorded"], true, JSON.stringify(verified)); + + const vaultLog = (await on(`docker logs mesh-vault 2>&1`)).out; + assert.doesNotMatch(vaultLog, /emit .*failed/, `the vault's lifecycle event was refused:\n${vaultLog}`); + assert.match(vaultLog, /granted: /, `the vault never recorded the grant:\n${vaultLog}`); +}); + +test("rotating the secret moves both ends: the new password works, the old one is refused, and the vault says so", { + skip, timeout: 900_000, +}, async () => { + assert.ok(delivered && holder, "the first test did not run"); + const before = delivered; + + const said = await mesh("rotate secret", 300_000); + assert.doesNotMatch(said, /nobody holds/i, said); + + // The consumer's file moves — the rotation was delivered, not only recorded. + const after = await until("the rotated secret reaching redis's machine", 180_000, async () => { + const now = (await must(`cat ${SECRET_FILE}`)).replace(/\n$/, ""); + return now !== before ? now : undefined; + }); + assert.notEqual(after, before); + + // Three logins. The new one works (redis was restarted on its config — `restart-on`), the old one + // does not: that third check is what makes it a rotation rather than an addition. + await until("redis accepting the rotated password", 180_000, async () => { + return (await pingAs(after)) === "PONG" ? true : undefined; + }); + assert.notEqual(await pingAs(before), "PONG", "the rotated-away password still authenticates — redis kept the credential it started with"); + + // The vault noticed: same holder, one rotation, new fingerprint, old one in the history. + const recorded = await until("the vault recording the rotation", 90_000, async () => { + const h = JSON.parse(await must(`cat ${LEDGER}/${holder}.json`)) as Held; + return h.rotations >= 1 ? h : undefined; + }); + assert.equal(recorded.rotations, 1); + assert.equal(recorded.fingerprint, fingerprint(after)); + assert.deepEqual(recorded.history.map((h) => h.fingerprint), [fingerprint(before)]); + + const stale = await vaultTool("secret_verify", { as: holder, fingerprint: fingerprint(before) }); + assert.equal(stale["ok"], false, JSON.stringify(stale)); + assert.equal(stale["givenIsAnEarlierValue"], true, JSON.stringify(stale)); + const fresh = await vaultTool("secret_verify", { as: holder, fingerprint: fingerprint(after) }); + assert.equal(fresh["ok"], true, JSON.stringify(fresh)); + + const vaultLog = (await on(`docker logs mesh-vault 2>&1`)).out; + assert.match(vaultLog, /rotated: /, `the vault never announced the rotation:\n${vaultLog}`); + assert.doesNotMatch(vaultLog, /emit .*failed/, vaultLog); +}); + +test("the operator recovers a root secret with a key the mesh never held, from the vault's copy and from the store", { + skip, timeout: 900_000, +}, async () => { + assert.ok(holder, "the first test did not run"); + + // The vault keeps the export on its own disk, written by the mesh, ciphertext throughout. + const exported = await until("the vault holding the export", 90_000, async () => { + const got = await on(`cat ${ROOT}/export.json`); + return got.ok && /"kept"/.test(got.out) ? got.out : undefined; + }); + const doc = JSON.parse(exported) as { + "operator-key": string; + kept: { node: string; module: string; name: string; origin: string; sealed: string }[]; + unrecoverable?: unknown[]; + }; + const mode = (await must(`stat -c %a ${ROOT}/export.json`)).trim(); + assert.equal(mode, "600", `the export is readable by others (mode ${mode})`); + // mesh-vault's own broker account was issued after the key existed, so it is in the export. + const ours = doc.kept.find((k) => k.node === MACHINE && k.module === "mesh-vault" && k.name === "broker"); + assert.ok(ours, `the vault's own broker account is not in the export:\n${exported}`); + assert.equal(ours.origin, "accepted"); + // The value the host unsealed on this machine — what the module actually runs with. + const onDisk = (await must(`cat /var/lib/mesh/mesh-vault/broker`)).replace(/\n$/, ""); + assert.ok(onDisk.length > 20, "the node's own copy is empty"); + assert.ok(!exported.includes(onDisk), "the export contains a plaintext value"); + + // 1. Recovered from the store, with the operator key: the same bytes the node holds. The key is + // handed to the running controller for this one call, readable by its own account. + await must(`chown 65534:65534 ${OPERATOR_DIR}/operator.key && docker cp -a ${OPERATOR_DIR}/operator.key mesh-controller:/operator.key`); + const fromStore = (await mesh(`secret recover ${MACHINE} mesh-vault broker --key /operator.key --out -`)).replace(/\n$/, ""); + assert.equal(fromStore, onDisk, "the operator recovered a different value from the one the node was given"); + + // 2. Recovered from the vault's export alone, off the mesh — no store, no broker, a directory and + // the key — so a mesh whose store is gone is still a mesh whose root secrets a person can open. + await must(`cp ${ROOT}/export.json ${OPERATOR_DIR}/export.json && chmod 644 ${OPERATOR_DIR}/export.json`); + const said = await operator(`secret recover ${MACHINE} mesh-vault broker --key /work/operator.key --from-export /work/export.json --out /work/recovered`); + assert.match(said, /recovered to \/work\/recovered/, said); + assert.ok(!said.includes(onDisk), "recover printed the value"); + const fromExport = (await must(`cat ${OPERATOR_DIR}/recovered`)).replace(/\n$/, ""); + assert.equal(fromExport, onDisk); + const recoveredMode = (await must(`stat -c %a ${OPERATOR_DIR}/recovered`)).trim(); + assert.equal(recoveredMode, "600"); + + // 3. Another key — made, never given to the mesh — opens nothing, and says so. + await operator(`operator key make --out /work/another.key`); + const wrong = await on( + `docker run --rm -v ${OPERATOR_DIR}:/work --entrypoint /mesh-controller ${pinned("mesh-controller")} ` + + `secret recover ${MACHINE} mesh-vault broker --key /work/another.key --from-export /work/export.json --out /work/nope`, + ); + assert.equal(wrong.ok, false, `a different operator key opened the secret:\n${wrong.out}`); + assert.match(wrong.out, /does not open it/, wrong.out); + + // 4. The vault serves the export over the mesh — ciphertext, plus what is NOT recoverable. + const served = await vaultTool("secret_export", { sealed: false }); + assert.equal(served["available"], true, JSON.stringify(served)); + assert.equal(served["operator-key"], doc["operator-key"]); + const listed = served["kept"] as { module: string; name: string }[]; + assert.ok(listed.some((k) => k.module === "mesh-vault" && k.name === "broker"), JSON.stringify(served)); + assert.ok(!JSON.stringify(served).includes(onDisk), "secret_export returned a plaintext value"); +});