scenarios/vault-node.yml

@@ -0,0 +1,32 @@
+# One machine that becomes a mesh and then assigns itself mesh-vault and redis — a provider whose own
+# password is a `secret` the vault provides (novox/hq ADR 0085, design 24).
+#
+# redis-node proves a provider's runtime. This proves the vault: redis requires `secret`, the mesh
+# mints the pair credential, the host fills redis.conf from it, redis authenticates with it, and the
+# vault's ledger records its fingerprint. Then `rotate secret` moves both ends: the new password
+# works, the old one is refused, and the vault says it was rotated — design 13's three logins, for a
+# secret that had no owner before.
+scenario: vault-node
+
+segments:
+  hosting:
+    kind: public
+    cidr: [192.0.2.0/24]
+
+machines:
+  anchor:
+    at: { segment: hosting, address: [192.0.2.10] }
+    egress: true
+    inbound: allow
+    memory: 3GiB
+    cpus: 2
+
+images:
+  - mesh-controller:development
+  # Built by scripts/build-module-runtime.sh mesh-vault / redis into the local daemon; the machine holds
+  # each by its own image ID. redis's server image is pulled upstream by digest.
+  - mesh-runtime-mesh-vault:development
+  - mesh-runtime-redis:development
+
+place:
+  all: [host, runtime]

test/integration/assigned-vault.test.ts

@@ -0,0 +1,435 @@
+/**
+ * A module's own secret, provided by the vault — and rotated like any other credential.
+ *
+ * novox/hq ADR 0085: a secret a module needs for itself is a `secret` provision from a vault module,
+ * and the credential of that consumer↔vault pair IS the value. So everything design 13 built for a
+ * database password applies to it unchanged, and this bed checks exactly what 13 checks: three
+ * logins against the real software, from the machine that holds the secret —
+ *
+ *   1. the delivered password authenticates against redis
+ *   2. after `rotate secret`, the new one authenticates
+ *   3. the one that was rotated away does not
+ *
+ * — plus the half that makes the vault an owner rather than a bystander: its ledger records who
+ * holds the secret and its fingerprint, notices the rotation, and answers over the mesh without ever
+ * returning a value.
+ *
+ * And the amended half of ADR 0085: the mesh has an operator key, every secret a module holds for
+ * itself is sealed to it as well, the vault keeps that export on its disk, and the operator — with
+ * the private key that never entered the mesh — recovers exactly the value the node was given.
+ * Checked against the value the host unsealed on the machine, and with the store out of the loop.
+ *
+ * The manifests are the catalogue's own (../mesh-catalog/modules/{mesh-vault,redis}/module.json), with
+ * the runtime artifact named as the image the lab built, exactly as the other assigned-* beds do.
+ *
+ *   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
+ *   scripts/build-module-runtime.sh mesh-vault / redis build the two runtime images into the local
+ *   daemon; scenarios/vault-node.yml stocks them.
+ */
+
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec } from "../../src/lifecycle/operate.ts";
+import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
+import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
+import type { HeldImage } from "../../src/pinning.ts";
+
+const capability = await labIsUsable();
+const binary = hostBinaryPath();
+const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
+const catalogue = process.env["MESH_LAB_CATALOG"] ?? "../mesh-catalog";
+
+const skip = !capability.usable
+  ? `lab not usable: ${capability.why}`
+  : !binary || !existsSync(binary)
+    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
+    : !bundle || !existsSync(bundle)
+      ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
+      : !existsSync(join(catalogue, "modules/mesh-vault/module.json"))
+        ? `no catalogue at ${catalogue} (set MESH_LAB_CATALOG)`
+        : false;
+
+const SCENARIO = "vault-node";
+const MACHINE = "anchor";
+const SECRET_FILE = "/var/lib/redis-module/default.secret";
+const LEDGER = "/var/lib/mesh-vault/ledger";
+const ROOT = "/var/lib/mesh-vault/root";
+// Where the operator keeps things on this machine: outside every container, bind-mounted into a
+// throwaway one when the operator's own commands run. The controller's container is a scratch
+// image with no writable path — the private key could not be written there even by mistake.
+const OPERATOR_DIR = "/var/lib/lab-operator";
+
+let instanceId = "";
+let held: HeldImage[] = [];
+
+function quote(s: string): string {
+  return `'${s.replaceAll("'", `'\\''`)}'`;
+}
+
+async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
+  const { stdout } = await exec(instanceId, MACHINE, [
+    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
+  ], timeoutMs);
+  const marker = stdout.lastIndexOf("__exit=");
+  if (marker < 0) return { out: stdout, ok: false };
+  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
+}
+
+async function must(command: string, timeoutMs?: number): Promise<string> {
+  const { out, ok } = await on(command, timeoutMs);
+  if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
+  return out;
+}
+
+async function mesh(command: string, timeoutMs?: number): Promise<string> {
+  return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
+}
+
+function pinned(reference: string): string {
+  return onTheMachine(reference, held);
+}
+
+function bundleFor(images: HeldImage[]): string {
+  return foundationBundle(bundle, images);
+}
+
+function tokenFrom(said: string): string {
+  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
+  assert.ok(found, `no token in:\n${said}`);
+  return found;
+}
+
+/** sha256:<hex>, the only form in which a secret's value is ever spoken here. */
+function fingerprint(value: string): string {
+  return "sha256:" + createHash("sha256").update(value, "utf8").digest("hex");
+}
+
+/**
+ * The catalogue's own manifest, as the lab can run it: the runtime artifact becomes the image this
+ * scenario loaded, third-party images are pinned upstream, and `build` goes — nothing here builds.
+ */
+function catalogueManifest(name: string): string {
+  const m = JSON.parse(readFileSync(join(catalogue, `modules/${name}/module.json`), "utf8")) as {
+    build?: unknown;
+    resources: Record<string, unknown>[];
+  };
+  delete m.build;
+  for (const r of m.resources) {
+    if (r["artifact"] === "runtime") {
+      delete r["artifact"];
+      r["image"] = pinned(`mesh-runtime-${name}`);
+    } else if (typeof r["image"] === "string") {
+      r["image"] = pinned(r["image"]);
+    }
+  }
+  return JSON.stringify(m);
+}
+
+async function until<T>(what: string, ms: number, probe: () => Promise<T | undefined>): Promise<T> {
+  const deadline = Date.now() + ms;
+  let last: unknown;
+  while (Date.now() < deadline) {
+    try {
+      const got = await probe();
+      if (got !== undefined) return got;
+    } catch (err) {
+      last = err;
+    }
+    await new Promise((r) => setTimeout(r, 3000));
+  }
+  throw new Error(`${what} did not happen within ${Math.round(ms / 1000)}s${last ? `; last: ${String(last)}` : ""}`);
+}
+
+async function settled(withinMs = 480_000): Promise<void> {
+  const deadline = Date.now() + withinMs;
+  let last = "";
+  while (Date.now() < deadline) {
+    const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
+    if (asked.ok) {
+      try {
+        const state = JSON.parse(asked.out) as {
+          wrong: { node: string; outcome: string }[];
+          waiting: { node: string }[];
+          reported: { node: string; outcome: string; current: boolean }[];
+        };
+        const bad = state.wrong.find((w) => w.node === MACHINE);
+        if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
+        const word = state.reported.find((r) => r.node === MACHINE);
+        if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
+        last = asked.out;
+      } catch (err) {
+        if (err instanceof Error && err.message.includes("did not apply")) throw err;
+        last = asked.out;
+      }
+    }
+    await new Promise((r) => setTimeout(r, 5000));
+  }
+  throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
+}
+
+/** redis's answer to PING as `password`, from inside its own container — PONG, or why not. */
+async function pingAs(password: string): Promise<string> {
+  return (await on(`docker exec redis redis-cli --no-auth-warning -a ${quote(password)} ping`)).out.trim();
+}
+
+/** The operator's own commands: the controller binary in a throwaway container, with the operator's
+ *  directory at /work and nothing of the mesh in reach — no store, no broker. What an operator runs
+ *  on their own machine with the binary and a directory. */
+async function operator(command: string): Promise<string> {
+  return must(
+    `docker run --rm -v ${OPERATOR_DIR}:/work --entrypoint /mesh-controller ${pinned("mesh-controller")} ${command}`,
+    120_000,
+  );
+}
+
+/** Invoke one of the vault's tools over the mesh, the way a caller would, and parse its answer. */
+async function vaultTool(tool: string, args: Record<string, unknown> = {}): Promise<Record<string, unknown>> {
+  const said = await must(
+    `docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
+      `${pinned("mesh-runtime-mesh-vault")} invoke mesh-vault ${tool} ${quote(JSON.stringify(args))}`,
+    120_000,
+  );
+  const line = said.split("\n").map((l) => l.trim()).filter(Boolean).pop() ?? "";
+  return JSON.parse(line) as Record<string, unknown>;
+}
+
+interface Held {
+  as: string;
+  consumer: string;
+  fingerprint: string;
+  rotations: number;
+  history: { fingerprint: string; until: string }[];
+}
+
+before(async () => {
+  if (skip) return;
+
+  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
+    onProgress: (m) => console.log(`raise: ${m}`),
+  });
+  instanceId = raised.instanceId;
+  held = raised.images;
+
+  await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
+  await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
+  const up = await must(`docker ps --format '{{.Names}}'`);
+  for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
+    assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
+  }
+
+  await mesh(`node add ${MACHINE}`);
+  const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
+  await must(`${HOST_PATH} enrol --token ${quote(token)}`);
+  await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
+}, { timeout: 1_800_000 });
+
+after(async () => {
+  if (instanceId) await destroy(instanceId);
+  await destroyAll(`${SCENARIO}-`);
+}, { timeout: 600_000 });
+
+let delivered = "";
+let holder = "";
+
+test("redis's own password is a secret the vault provides: it authenticates, and the vault holds its fingerprint", {
+  skip, timeout: 900_000,
+}, async () => {
+  // The operator key first, so everything minted from here on is sealed to it as well. Made off
+  // the mesh — a throwaway container with the operator's directory mounted, no store in reach —
+  // and only the public half is given to the mesh.
+  await must(`mkdir -p ${OPERATOR_DIR} && chmod 777 ${OPERATOR_DIR}`);
+  const made = await operator(`operator key make --out /work/operator.key`);
+  assert.match(made, /operator key sha256:/, made);
+  const publicKey = made.split("\n").map((l) => l.trim()).map((l) => /^public ([A-Za-z0-9+\/]{40,}={0,2})$/.exec(l)?.[1]).find(Boolean);
+  assert.ok(publicKey, `no public key in:\n${made}`);
+  const privateKey = (await must(`cat ${OPERATOR_DIR}/operator.key`)).trim();
+  assert.ok(privateKey.length > 30 && !made.includes(privateKey), "the private key was printed");
+  const set = await mesh(`operator key set ${quote(publicKey)}`);
+  assert.match(set, /operator key sha256:/, set);
+
+  for (const name of ["mesh-vault", "redis"]) {
+    await must(`printf %s ${quote(catalogueManifest(name))} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
+    await mesh(`module add /${name}.json`);
+    const issued = await mesh(`module issue ${name} --node ${MACHINE}`);
+    assert.match(issued, /scoped to what it emits and consumes/, issued);
+    await mesh(`assign ${MACHINE} ${name}`);
+  }
+  await mesh(`push ${MACHINE}`);
+  await settled();
+
+  const running = await must(`docker ps --format '{{.Names}}'`);
+  for (const c of ["mesh-vault", "redis", "mesh-redis"]) {
+    assert.match(running, new RegExp(`(^|\\n)${c}(\\n|$)`),
+      `${c} is not running:\n${running}\n---host log---\n${(await on(`tail -40 /var/log/mesh-host.log`)).out}`);
+  }
+
+  // 1. The pair credential the mesh minted is on redis's machine as redis's own password — and
+  //    redis accepts it. Not because two files agree: because the login works.
+  delivered = (await must(`cat ${SECRET_FILE}`)).replace(/\n$/, "");
+  assert.ok(delivered.length >= 20, "the delivered secret is empty or implausibly short");
+  const mode = (await must(`stat -c %a ${SECRET_FILE}`)).trim();
+  assert.equal(mode, "600", `the secret file is readable by others (mode ${mode})`);
+  const pong = await until("redis accepting the delivered password", 120_000, async () => {
+    const said = await pingAs(delivered);
+    return said === "PONG" ? said : undefined;
+  });
+  assert.equal(pong, "PONG");
+  assert.notEqual(await pingAs("not-the-password"), "PONG", "redis accepted a wrong password — requirepass is not set");
+
+  // 2. The vault took custody: its ledger names the holder, with the fingerprint of exactly the
+  //    value redis holds — and no value.
+  const recorded = await until("the vault recording redis's secret", 90_000, async () => {
+    const ls = await on(`ls ${LEDGER}`);
+    const name = ls.ok ? ls.out.split("\n").map((l) => l.trim()).find((l) => l.endsWith(".json")) : undefined;
+    if (!name) return undefined;
+    return JSON.parse(await must(`cat ${LEDGER}/${name}`)) as Held;
+  });
+  holder = recorded.as;
+  assert.equal(recorded.consumer, MACHINE);
+  assert.equal(recorded.fingerprint, fingerprint(delivered), "the vault recorded a fingerprint of something other than what redis holds");
+  assert.equal(recorded.rotations, 0);
+  const ledgerRaw = await must(`cat ${LEDGER}/*.json`);
+  assert.ok(!ledgerRaw.includes(delivered), "the vault's ledger contains the secret's value");
+  // The holder is the login the mesh derived for redis on this machine, as the contributions file says.
+  const asked = JSON.parse(await must(`cat /var/lib/mesh-vault/grants/mesh.json`)) as { given: { as: string; from?: string }[] };
+  assert.ok(asked.given.some((g) => g.as === holder), `the vault was not asked to serve ${holder}:\n${JSON.stringify(asked)}`);
+
+  // 3. And it answers over the mesh — who holds what, verified by fingerprint, never by value.
+  const holders = await until("the vault serving its tools", 90_000, async () => {
+    const got = await vaultTool("secret_holders");
+    return Array.isArray(got["holders"]) && (got["holders"] as Held[]).length > 0 ? got : undefined;
+  });
+  const mine = (holders["holders"] as (Held & { module: string | null })[]).find((h) => h.as === holder);
+  assert.ok(mine, `secret_holders does not list ${holder}:\n${JSON.stringify(holders)}`);
+  assert.equal(mine.module, "redis");
+  assert.ok(!JSON.stringify(holders).includes(delivered), "secret_holders returned the value");
+
+  const verified = await vaultTool("secret_verify", { as: holder, fingerprint: fingerprint(delivered) });
+  assert.equal(verified["ok"], true, JSON.stringify(verified));
+  assert.equal(verified["deliveredMatchesRecorded"], true, JSON.stringify(verified));
+  assert.equal(verified["givenMatchesRecorded"], true, JSON.stringify(verified));
+
+  const vaultLog = (await on(`docker logs mesh-vault 2>&1`)).out;
+  assert.doesNotMatch(vaultLog, /emit .*failed/, `the vault's lifecycle event was refused:\n${vaultLog}`);
+  assert.match(vaultLog, /granted: /, `the vault never recorded the grant:\n${vaultLog}`);
+});
+
+test("rotating the secret moves both ends: the new password works, the old one is refused, and the vault says so", {
+  skip, timeout: 900_000,
+}, async () => {
+  assert.ok(delivered && holder, "the first test did not run");
+  const before = delivered;
+
+  const said = await mesh("rotate secret", 300_000);
+  assert.doesNotMatch(said, /nobody holds/i, said);
+
+  // The consumer's file moves — the rotation was delivered, not only recorded.
+  const after = await until("the rotated secret reaching redis's machine", 180_000, async () => {
+    const now = (await must(`cat ${SECRET_FILE}`)).replace(/\n$/, "");
+    return now !== before ? now : undefined;
+  });
+  assert.notEqual(after, before);
+
+  // Three logins. The new one works (redis was restarted on its config — `restart-on`), the old one
+  // does not: that third check is what makes it a rotation rather than an addition.
+  await until("redis accepting the rotated password", 180_000, async () => {
+    return (await pingAs(after)) === "PONG" ? true : undefined;
+  });
+  assert.notEqual(await pingAs(before), "PONG", "the rotated-away password still authenticates — redis kept the credential it started with");
+
+  // The vault noticed: same holder, one rotation, new fingerprint, old one in the history.
+  const recorded = await until("the vault recording the rotation", 90_000, async () => {
+    const h = JSON.parse(await must(`cat ${LEDGER}/${holder}.json`)) as Held;
+    return h.rotations >= 1 ? h : undefined;
+  });
+  assert.equal(recorded.rotations, 1);
+  assert.equal(recorded.fingerprint, fingerprint(after));
+  assert.deepEqual(recorded.history.map((h) => h.fingerprint), [fingerprint(before)]);
+
+  const stale = await vaultTool("secret_verify", { as: holder, fingerprint: fingerprint(before) });
+  assert.equal(stale["ok"], false, JSON.stringify(stale));
+  assert.equal(stale["givenIsAnEarlierValue"], true, JSON.stringify(stale));
+  const fresh = await vaultTool("secret_verify", { as: holder, fingerprint: fingerprint(after) });
+  assert.equal(fresh["ok"], true, JSON.stringify(fresh));
+
+  const vaultLog = (await on(`docker logs mesh-vault 2>&1`)).out;
+  assert.match(vaultLog, /rotated: /, `the vault never announced the rotation:\n${vaultLog}`);
+  assert.doesNotMatch(vaultLog, /emit .*failed/, vaultLog);
+});
+
+test("the operator recovers a root secret with a key the mesh never held, from the vault's copy and from the store", {
+  skip, timeout: 900_000,
+}, async () => {
+  assert.ok(holder, "the first test did not run");
+
+  // The vault keeps the export on its own disk, written by the mesh, ciphertext throughout.
+  const exported = await until("the vault holding the export", 90_000, async () => {
+    const got = await on(`cat ${ROOT}/export.json`);
+    return got.ok && /"kept"/.test(got.out) ? got.out : undefined;
+  });
+  const doc = JSON.parse(exported) as {
+    "operator-key": string;
+    kept: { node: string; module: string; name: string; origin: string; sealed: string }[];
+    unrecoverable?: unknown[];
+  };
+  const mode = (await must(`stat -c %a ${ROOT}/export.json`)).trim();
+  assert.equal(mode, "600", `the export is readable by others (mode ${mode})`);
+  // mesh-vault's own broker account was issued after the key existed, so it is in the export.
+  const ours = doc.kept.find((k) => k.node === MACHINE && k.module === "mesh-vault" && k.name === "broker");
+  assert.ok(ours, `the vault's own broker account is not in the export:\n${exported}`);
+  assert.equal(ours.origin, "accepted");
+  // The value the host unsealed on this machine — what the module actually runs with.
+  const onDisk = (await must(`cat /var/lib/mesh/mesh-vault/broker`)).replace(/\n$/, "");
+  assert.ok(onDisk.length > 20, "the node's own copy is empty");
+  assert.ok(!exported.includes(onDisk), "the export contains a plaintext value");
+
+  // 1. Recovered from the store, with the operator key: the same bytes the node holds. The key is
+  //    handed to the running controller for this one call, readable by its own account.
+  await must(`chown 65534:65534 ${OPERATOR_DIR}/operator.key && docker cp -a ${OPERATOR_DIR}/operator.key mesh-controller:/operator.key`);
+  const fromStore = (await mesh(`secret recover ${MACHINE} mesh-vault broker --key /operator.key --out -`)).replace(/\n$/, "");
+  assert.equal(fromStore, onDisk, "the operator recovered a different value from the one the node was given");
+
+  // 2. Recovered from the vault's export alone, off the mesh — no store, no broker, a directory and
+  //    the key — so a mesh whose store is gone is still a mesh whose root secrets a person can open.
+  await must(`cp ${ROOT}/export.json ${OPERATOR_DIR}/export.json && chmod 644 ${OPERATOR_DIR}/export.json`);
+  const said = await operator(`secret recover ${MACHINE} mesh-vault broker --key /work/operator.key --from-export /work/export.json --out /work/recovered`);
+  assert.match(said, /recovered to \/work\/recovered/, said);
+  assert.ok(!said.includes(onDisk), "recover printed the value");
+  const fromExport = (await must(`cat ${OPERATOR_DIR}/recovered`)).replace(/\n$/, "");
+  assert.equal(fromExport, onDisk);
+  const recoveredMode = (await must(`stat -c %a ${OPERATOR_DIR}/recovered`)).trim();
+  assert.equal(recoveredMode, "600");
+
+  // 3. Another key — made, never given to the mesh — opens nothing, and says so.
+  await operator(`operator key make --out /work/another.key`);
+  const wrong = await on(
+    `docker run --rm -v ${OPERATOR_DIR}:/work --entrypoint /mesh-controller ${pinned("mesh-controller")} ` +
+      `secret recover ${MACHINE} mesh-vault broker --key /work/another.key --from-export /work/export.json --out /work/nope`,
+  );
+  assert.equal(wrong.ok, false, `a different operator key opened the secret:\n${wrong.out}`);
+  assert.match(wrong.out, /does not open it/, wrong.out);
+
+  // 3b. And the secret the vault PROVIDES — redis's password, a pair credential — is recoverable
+  //     the same way, off the mesh: the export names it by the consumer and the provision.
+  const redisPassword = (await must(`cat ${SECRET_FILE}`)).replace(/\n$/, "");
+  const pairLine = doc.kept.find((k) => k.module === "redis" && k.name === "secret");
+  assert.ok(pairLine, `redis's vault-provided secret is not in the export:\n${exported}`);
+  await must(`cp ${ROOT}/export.json ${OPERATOR_DIR}/export.json && chmod 644 ${OPERATOR_DIR}/export.json`);
+  await operator(`secret recover ${MACHINE} redis secret --key /work/operator.key --from-export /work/export.json --out /work/redis.secret`);
+  const recoveredRedis = (await must(`cat ${OPERATOR_DIR}/redis.secret`)).replace(/\n$/, "");
+  assert.equal(recoveredRedis, redisPassword, "the recovered pair credential is not the password redis runs with");
+  assert.equal(await pingAs(recoveredRedis), "PONG", "the recovered password does not open redis");
+
+  // 4. The vault serves the export over the mesh — ciphertext, plus what is NOT recoverable.
+  const served = await vaultTool("secret_export", { sealed: false });
+  assert.equal(served["available"], true, JSON.stringify(served));
+  assert.equal(served["operator-key"], doc["operator-key"]);
+  const listed = served["kept"] as { module: string; name: string }[];
+  assert.ok(listed.some((k) => k.module === "mesh-vault" && k.name === "broker"), JSON.stringify(served));
+  assert.ok(!JSON.stringify(served).includes(onDisk), "secret_export returned a plaintext value");
+});

test/integration/one-node-mesh.test.ts

@@ -109,9 +109,9 @@ const CONTROL_PLANE = { module: "mesh-controller", repo: "mesh-controller", path
  * is not one.
  */
 const MUST_HOLD = ["mesh-controller", "distribution", "builder", "mesh-tools", "postgres",
-  "mesh-catalog", "lavinmq", "amqp-ping"];
+  "mesh-catalog", "lavinmq", "mesh-vault", "amqp-ping"];
 const MUST_RUN = ["mesh-controller", "mesh-registry", "mesh-broker", "mesh-store",
-  "mesh-postgres", "mesh-catalog", "mesh-lavinmq", "amqp-ping"];
+  "mesh-postgres", "mesh-catalog", "mesh-lavinmq", "mesh-vault", "amqp-ping"];
 /** Named once, because the step title is also how later steps say what they waited on. */
 const NEEDS = "the mesh runs a broker for that module to talk to";
 const GENESIS = "a bare machine becomes a mesh of one, raised by the installer";
@@ -133,6 +133,7 @@ const DESCRIBES = "the control plane can describe the mesh, and what it says is
 const CATALOGUED = "the catalogue holds every module this mesh built";
 const NETWORK = "the machine's networking is what the modules asked for";
 const DECLARED = "every resource the mesh declared is true on the machine";
+const ROOT_SECRETS = "the root secrets are the mesh's own, sealed to an operator key, and a person can recover them";
 const FOLLOWS = "a change to a module's source reaches the machine on its own";
 const STORE_UPGRADES = "the store is upgraded in place, and the controller reads it through the window";
 const BROKER_UPGRADES = "the broker is upgraded in place, and the mesh talks over the window";
@@ -230,7 +231,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
   for (;;) {
     const { out, ok } = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
     if (ok) return out;
-    if (/is not running|No such container/i.test(out) && Date.now() < deadline) {
+    // A push that has said "N node(s) told" has done its work: the declaration is on the bus. The
+    // exec carrying it can still die afterwards with no message when the controller itself is
+    // recreated under it — which is exactly what an in-place broker upgrade does, since the
+    // controller reconnects to the bus that returns (S2). The send is what the push is for.
+    if (/^push\b/.test(command) && /\d+ node\(s\) told/.test(out)) return out;
+    // "No such exec instance" and a daemon that cannot be reached are the same race one layer down:
+    // applying the packet filter restarts the container runtime itself, and every container with
+    // it, a few seconds after the installer's last push returns.
+    if (/is not running|No such container|No such exec instance|Cannot connect to the Docker daemon/i.test(out) && Date.now() < deadline) {
       await new Promise((r) => setTimeout(r, 2_000));
       continue;
     }
@@ -238,6 +247,34 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
   }
 }
 
+/** Until the anchor reports the last declaration genesis pushed as applied and current. */
+async function settledAfterGenesis(withinMs = 300_000): Promise<void> {
+  const deadline = Date.now() + withinMs;
+  let last = "";
+  while (Date.now() < deadline) {
+    const asked = await on(CONTROL, `docker exec mesh-controller /mesh-controller status --json`);
+    if (asked.ok) {
+      try {
+        const state = JSON.parse(asked.out) as {
+          wrong: { node: string; outcome: string }[];
+          waiting: { node: string }[];
+          reported: { node: string; outcome: string; current: boolean }[];
+        };
+        const bad = state.wrong.find((w) => w.node === CONTROL);
+        if (bad) throw new Error(`${CONTROL} did not apply what genesis sent: ${bad.outcome}\n${asked.out}`);
+        const word = state.reported.find((r) => r.node === CONTROL);
+        if (!state.waiting.some((w) => w.node === CONTROL) && word?.outcome === "applied" && word.current) return;
+        last = asked.out;
+      } catch (err) {
+        if (err instanceof Error && err.message.includes("did not apply")) throw err;
+        last = asked.out;
+      }
+    }
+    await new Promise((r) => setTimeout(r, 5_000));
+  }
+  throw new Error(`${CONTROL} never settled after genesis within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
+}
+
 /**
  * Stop the machine and start it again, the way a power cut or a kernel upgrade would.
  *
@@ -518,6 +555,13 @@ before(async () => {
   // saying it published an image and the registry serving one are different facts, and it is the
   // second that matters.
 
+  // **The machine is still applying what genesis pushed last.** The installer's final pushes — the
+  // private network, the packet filter — return when the mesh has sent them, not when the host has
+  // finished; applying the filter restarts the container runtime, and every container with it.
+  // Asked of the mesh rather than slept through: nothing below is sent until the node reports the
+  // declaration it was given as applied and current.
+  await settledAfterGenesis();
+
   await step("R2", FOUNDATION, GENESIS, async () => {
     await waitForContainer(CONTROL, "mesh-store", 120);
     await waitForContainer(CONTROL, "mesh-broker", 120);
@@ -798,10 +842,13 @@ before(async () => {
     // broker's loopback, reached by joining its network namespace.
     const image = (await on(CONTROL,
       `docker inspect -f '{{.Config.Image}}' mesh-catalog`)).out.trim();
+    // The administrator's password is the one genesis made, kept where the lavinmq module's own
+    // secret lives (novox/hq issue 071) — the image's default no longer opens the broker.
+    const adminPassword = (await must(CONTROL, `cat /var/lib/lavinmq-module/admin.secret`)).trim();
     const ask = async (tool: string, args = "{}") =>
       must(CONTROL,
         `docker run --rm --network container:mesh-broker ` +
-        `-e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
+        `-e MESH_BROKER_URL=amqp://guest:${encodeURIComponent(adminPassword)}@127.0.0.1:5672/ ` +
         `${image} invoke mesh-catalog ${tool} ${quote(args)}`,
         120_000);
 
@@ -961,6 +1008,73 @@ before(async () => {
     ].filter(Boolean).join("\n");
   });
 
+  // ---- V5. AND ITS ROOT SECRETS ARE ITS OWN ------------------------------------------------------
+  //
+  // novox/hq ADR 0085 (amended), issue 071. The template raises the store and broker with fixed
+  // credentials; the installer replaces them with values it made, seals every secret a module
+  // holds for itself to an operator key it made first, installs the vault to keep those copies,
+  // and writes the export beside the key. Checked from outside every container — a login over
+  // loopback inside the store's container is trusted and proves nothing (design 13).
+  await step("V5", ROOT_SECRETS, DECLARED, async () => {
+    const said: string[] = [];
+    const storeImage = (await must(CONTROL, `docker inspect -f '{{.Config.Image}}' mesh-store`)).trim();
+    const psql = async (password: string) =>
+      on(CONTROL,
+        `docker run --rm --network host ${storeImage} psql ` +
+        `${quote(`postgresql://postgres:${encodeURIComponent(password)}@127.0.0.1:5432/postgres?sslmode=disable`)} -tAc 'select 1'`,
+        60_000);
+
+    // 1. The template's password does not open the store.
+    const stale = await psql("bootstrap");
+    assert.ok(!stale.ok || !/^1$/m.test(stale.out), `the template's password still opens the store:\n${stale.out}`);
+    said.push(`  bootstrap         refused by the store`);
+
+    // 2. The operator key and the export are beside the bundle, and only root can read them.
+    for (const f of ["/var/lib/mesh-host/operator.key", "/var/lib/mesh-host/root-secrets.export.json", "/var/lib/mesh-host/foundation.lock"]) {
+      const mode = (await must(CONTROL, `stat -c %a ${f}`)).trim();
+      assert.equal(mode, "600", `${f} is mode ${mode}`);
+    }
+    const exported = await must(CONTROL, `cat /var/lib/mesh-host/root-secrets.export.json`);
+    const doc = JSON.parse(exported) as { kept: { node: string; module: string; name: string; origin: string }[]; unrecoverable?: unknown[] };
+    for (const want of [["postgres", "superuser"], ["lavinmq", "admin"], ["mesh-controller", "inventory"]]) {
+      assert.ok(doc.kept.some((k) => k.module === want[0] && k.name === want[1]),
+        `the export lacks ${want.join("/")}:\n${doc.kept.map((k) => `${k.module}/${k.name}`).join(" ")}`);
+    }
+    const superuser = (await must(CONTROL, `cat /var/lib/postgres/superuser.secret`)).trim();
+    const admin = (await must(CONTROL, `cat /var/lib/lavinmq-module/admin.secret`)).trim();
+    assert.ok(!exported.includes(superuser) && !exported.includes(admin), "the export holds a plaintext root secret");
+    said.push(`  exported          ${doc.kept.length} secret(s) sealed to the operator key; ${(doc.unrecoverable ?? []).length} not recoverable`);
+
+    // 3. The vault keeps the same export on its own disk.
+    const kept = await must(CONTROL, `cat /var/lib/mesh-vault/root/export.json`);
+    assert.ok(kept.includes('"kept"') && !kept.includes(superuser), "the vault's copy is missing or holds plaintext");
+    said.push(`  vault             keeps the export at /var/lib/mesh-vault/root/export.json`);
+
+    // 4. A person with the key recovers the store's superuser from the export alone — off the
+    //    mesh, in a throwaway container with no store or broker in reach — and it opens the store.
+    //    The copies are relaxed to a scratch directory for the test only: the operator's real
+    //    files stay root-owned at 0600 above.
+    const controllerImage = (await must(CONTROL, `docker inspect -f '{{.Config.Image}}' mesh-controller`)).trim();
+    await must(CONTROL, `rm -rf /tmp/operator && mkdir -p /tmp/operator && chmod 777 /tmp/operator && ` +
+      `cp /var/lib/mesh-host/operator.key /var/lib/mesh-host/root-secrets.export.json /tmp/operator/ && chmod 644 /tmp/operator/*`);
+    const recover = async (module: string, name: string) => {
+      await must(CONTROL,
+        `docker run --rm -v /tmp/operator:/work --entrypoint /mesh-controller ${controllerImage} ` +
+        `secret recover ${CONTROL} ${module} ${name} --key /work/operator.key --from-export /work/root-secrets.export.json --out /work/${module}.${name}`,
+        120_000);
+      return (await must(CONTROL, `cat /tmp/operator/${module}.${name}`)).replace(/\n$/, "");
+    };
+    const recoveredSuperuser = await recover("postgres", "superuser");
+    assert.equal(recoveredSuperuser, superuser, "the recovered superuser is not the one the store was raised with");
+    const opened = await psql(recoveredSuperuser);
+    assert.ok(opened.ok && /^1$/m.test(opened.out), `the recovered superuser does not open the store:\n${opened.out}`);
+    said.push(`  recovered         postgres/superuser with the operator key, and it opens the store`);
+    const recoveredAdmin = await recover("lavinmq", "admin");
+    assert.equal(recoveredAdmin, admin, "the recovered broker admin is not the one genesis made");
+    said.push(`  recovered         lavinmq/admin with the operator key — the broker's, as V2 dialled it`);
+    return said.join("\n");
+  });
+
   // ---- 11. A CHANGE REACHES THE MACHINE ON ITS OWN ----------------------------------------------
   //
   // The whole point of the mesh, and the capability the migration depends on: move a module's