diff --git a/scenarios/trust-anchor.yml b/scenarios/trust-anchor.yml new file mode 100644 index 0000000..49e74cf --- /dev/null +++ b/scenarios/trust-anchor.yml @@ -0,0 +1,40 @@ +# One machine that becomes a mesh, runs the mesh's own certificate authority, and is then given the +# module that makes it trust it — the bed for novox/hq ADR 0147 and issue 129. +# +# The question is narrow and the bed is shaped to answer only it: does a machine holding `ca-trust` +# verify a certificate from the mesh's own authority with no bundle argument and no `-k`, and does +# it stop verifying it when the module is taken away? The authority itself is what is dialled — +# step-ca serves its own API with a leaf it issued — so nothing else has to be right for the answer +# to mean something. No proxy, no routed name, no public issuance: those are the certificates and +# route-forwarding beds, and a trust bed that leaned on them would pass for their reasons. +# +# The negative half is not optional. It is asserted BEFORE the module is assigned and again AFTER it +# is unassigned, because an anchor bed that only ever checks the success is one that would pass on a +# machine that already trusted everything. +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock +# MESH_LAB_CATALOG=.../mesh-catalog/modules +# step-ca's image is upstream and pinned by the catalogue; the machine pulls it over its uplink. +# ca-trust carries no image at all — a script, a unit, and the machine's own systemd. +scenario: trust-anchor + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + +images: + - mesh-controller:development + +place: + # Only the host. The authority's image comes from the internet over the machine's uplink, and the + # trust module has nothing to place. + all: [host] diff --git a/test/integration/assigned-ca-trust.test.ts b/test/integration/assigned-ca-trust.test.ts new file mode 100644 index 0000000..8a8b215 --- /dev/null +++ b/test/integration/assigned-ca-trust.test.ts @@ -0,0 +1,221 @@ +/** + * A machine trusts the mesh's own authority because a module put its root there — and stops when + * the module is taken away (novox/hq ADR 0147, 04-ISSUES/129). + * + * What is dialled is the authority itself. step-ca serves its own API with a leaf it issued, so a + * plain client verifying that handshake — no `-k`, no `--cacert` — is verifying exactly one thing: + * that this machine's trust store now contains the mesh's root. No proxy, no routed name, no public + * issuance. A trust bed leaning on those would pass for their reasons, which is the failure this + * whole sequence keeps producing. + * + * The negative half runs twice, before the module is assigned and after it is unassigned. An anchor + * bed that only checks the success would pass on a machine that already trusted everything, and + * would say nothing at all about removal — which is the half issue 129 asked for by name. + * + * MESH_LAB_HOST_BINARY=.../mesh-host + * MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock + * MESH_LAB_CATALOG=.../mesh-catalog/modules + * step-ca's image is upstream, pinned by the catalogue, pulled by the machine over its uplink. + * ca-trust carries no image: a script, a unit, and the machine's own systemd. + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueIsPresent } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" + : catalogueIsPresent(); + +const SCENARIO = "trust-anchor"; +const MACHINE = "anchor"; +/** The authority's own API, which it serves with a certificate it issued itself. */ +const AUTHORITY = "https://127.0.0.1:9000/health"; +/** Where the module puts the root, and therefore what removal must take away. */ +const ANCHOR = "/etc/ca-certificates/trust-source/anchors/mesh-internal-ca.crt"; + +let instanceId = ""; +let held: HeldImage[] = []; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, MACHINE, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(command, timeoutMs); + if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`); + return out; +} + +/** The control plane, a container on the node. */ +async function mesh(command: string, timeoutMs?: number): Promise { + return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +async function settled(withinMs = 480_000): Promise { + const until = Date.now() + withinMs; + let last = ""; + while (Date.now() < until) { + const asked = await on(`docker exec mesh-controller /mesh-controller status --json`); + if (asked.ok) { + try { + const state = JSON.parse(asked.out) as { + wrong: { node: string; outcome: string }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + const bad = state.wrong.find((w) => w.node === MACHINE); + if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`); + const word = state.reported.find((r) => r.node === MACHINE); + if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return; + last = asked.out; + } catch (err) { + if (err instanceof Error && err.message.includes("did not apply")) throw err; + last = asked.out; + } + } + await new Promise((r) => setTimeout(r, 5000)); + } + throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`); +} + +/** Install a catalogue module's manifest into the control plane, read from the catalogue. */ +async function register(module: string): Promise { + const manifest = catalogueModule(module, held); + await must( + `printf %s ${quote(manifest)} > /tmp/${module}.json && ` + + `docker cp /tmp/${module}.json mesh-controller:/${module}.json`, + ); + await mesh(`module add /${module}.json`); +} + +/** Dial the authority the way anything on this machine would: verifying, with nothing handed to it. */ +async function verifying(): Promise<{ out: string; ok: boolean }> { + return on(`curl --silent --show-error --max-time 10 ${AUTHORITY}`); +} + +before(async () => { + if (skip) return; + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + held = raised.images; + + await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`); + await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000); + const up = await must(`docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { + assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`); + } + + // The control plane is a container, and a container that is restarting answers nothing. Asked + // until it answers rather than once, so a crash-looping control plane is reported as itself + // instead of as whatever command happened to be sent first. + let control = { out: "", ok: false }; + const answering = Date.now() + 120_000; + while (Date.now() < answering) { + control = await on(`docker exec mesh-controller /mesh-controller status`); + if (control.ok) break; + await new Promise((r) => setTimeout(r, 3000)); + } + assert.ok(control.ok, + `the control plane never answered:\n${control.out}\n` + + `${(await on(`docker logs mesh-controller 2>&1 | tail -40`)).out}`); + + await mesh(`node add ${MACHINE}`); + const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`)); + await must(`${HOST_PATH} enrol --token ${quote(token)}`); + await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); +}, { timeout: 1_800_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 600_000 }); + +test("the mesh's authority is verified on a machine holding ca-trust, and not on one that is not", { + skip, timeout: 1_800_000, +}, async () => { + // The authority first, on its own. Nothing about trust yet. + await register("step-ca"); + await mesh(`module issue step-ca --node ${MACHINE}`); + await mesh(`assign ${MACHINE} step-ca`); + await mesh(`push ${MACHINE}`); + await settled(); + + // It is up and answering — asked the way nothing else in this bed is allowed to ask, with + // verification off, because at this point in the bed no machine could verify it. + let answering = false; + const until = Date.now() + 180_000; + while (Date.now() < until && !answering) { + answering = (await on(`curl --silent --insecure --max-time 5 ${AUTHORITY}`)).ok; + if (!answering) await new Promise((r) => setTimeout(r, 3000)); + } + assert.ok(answering, `the authority never answered:\n${(await on(`docker logs step-ca 2>&1 | tail -30`)).out}`); + + // **The negative half, before anything is assigned.** If this passes, the bed is measuring + // something already in the machine's trust store and everything below it is worthless. + const before = await verifying(); + assert.equal(before.ok, false, `the authority verified before anything anchored its root:\n${before.out}`); + assert.match(before.out, /certificate|issuer/i, `it failed for some other reason:\n${before.out}`); + + // Now the module. + await register("ca-trust"); + await mesh(`assign ${MACHINE} ca-trust`); + await mesh(`push ${MACHINE}`); + await settled(); + + const unit = await on(`systemctl is-active mesh-ca-trust.service`); + assert.ok(unit.ok, `the unit is ${unit.out.trim()}:\n${(await on(`journalctl -u mesh-ca-trust --no-pager | tail -30`)).out}`); + await must(`test -s ${ANCHOR}`); + const anchors = await must(`trust list | grep -A2 -i 'Mesh Internal CA' || trust list`); + assert.match(anchors, /Mesh Internal CA/, `the mesh's authority is not among the machine's anchors:\n${anchors}`); + + // **The whole claim, in one command.** No -k, no --cacert: the machine's own trust store, and a + // certificate the mesh's authority issued. + const after = await verifying(); + assert.ok(after.ok, `the authority still does not verify with the module assigned:\n${after.out}`); + + // **And removal is the same unit's business.** Unassigned, the host stops it; stopping it takes + // the anchor away and refreshes the bundles, so the machine stops trusting the mesh. + await mesh(`unassign ${MACHINE} ca-trust`); + await mesh(`push ${MACHINE}`); + await settled(); + + const gone = await on(`test -e ${ANCHOR}`); + assert.equal(gone.ok, false, "the anchor is still on the machine after the module was unassigned"); + const afterwards = await verifying(); + assert.equal(afterwards.ok, false, `the machine still verifies the mesh's authority with nothing anchoring it:\n${afterwards.out}`); + assert.match(afterwards.out, /certificate|issuer/i, `it failed for some other reason:\n${afterwards.out}`); +}); diff --git a/test/integration/harness.ts b/test/integration/harness.ts index 6c3cded..466dbe9 100644 --- a/test/integration/harness.ts +++ b/test/integration/harness.ts @@ -40,6 +40,14 @@ const UPSTREAM_STORE = "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee"; const UPSTREAM_BROKER = "cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b"; +/** + * And the bus, for `foundation-first-node-nats.lock` — the bundle the mesh raises since it stopped + * speaking AMQP (novox/hq ADR 0131). The digest is the bundle's own: what the registry served was a + * copy of the upstream image, so the same digest resolves on Docker Hub, and this is a prefix being + * removed rather than a reference being replaced. + */ +const UPSTREAM_BUS = + "nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927"; /** * The foundation bundle as a machine should receive it. @@ -53,6 +61,7 @@ export function foundationBundle(path: string, held: HeldImage[]): string { text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE); text = text.replaceAll( /[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER); + text = text.replaceAll(/[A-Za-z0-9_.:-]+:[0-9]+\/nats@sha256:[0-9a-f]{64}/g, UPSTREAM_BUS); return pinnedInto(text, held); }