/** * A machine trusts the mesh's own authority because a module put its root there — and stops when * the module is taken away (novox/hq ADR 0147, 04-ISSUES/129). * * What is dialled is the authority itself. step-ca serves its own API with a leaf it issued, so a * plain client verifying that handshake — no `-k`, no `--cacert` — is verifying exactly one thing: * that this machine's trust store now contains the mesh's root. No proxy, no routed name, no public * issuance. A trust bed leaning on those would pass for their reasons, which is the failure this * whole sequence keeps producing. * * The negative half runs twice, before the module is assigned and after it is unassigned. An anchor * bed that only checks the success would pass on a machine that already trusted everything, and * would say nothing at all about removal — which is the half issue 129 asked for by name. * * MESH_LAB_HOST_BINARY=.../mesh-host * MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock * MESH_LAB_CATALOG=.../mesh-catalog/modules * step-ca's image is upstream, pinned by the catalogue, pulled by the machine over its uplink. * ca-trust carries no image: a script, a unit, and the machine's own systemd. */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { existsSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueIsPresent } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" : catalogueIsPresent(); const SCENARIO = "trust-anchor"; const MACHINE = "anchor"; /** The authority's own API, which it serves with a certificate it issued itself. */ const AUTHORITY = "https://127.0.0.1:9000/health"; /** Where the module puts the root, and therefore what removal must take away. */ const ANCHOR = "/etc/ca-certificates/trust-source/anchors/mesh-internal-ca.crt"; let instanceId = ""; let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { const { stdout } = await exec(instanceId, MACHINE, [ "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, ], timeoutMs); const marker = stdout.lastIndexOf("__exit="); if (marker < 0) return { out: stdout, ok: false }; return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; } async function must(command: string, timeoutMs?: number): Promise { const { out, ok } = await on(command, timeoutMs); if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`); return out; } /** The control plane, a container on the node. */ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); } function tokenFrom(said: string): string { const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); assert.ok(found, `no token in:\n${said}`); return found; } async function settled(withinMs = 480_000): Promise { const until = Date.now() + withinMs; let last = ""; while (Date.now() < until) { const asked = await on(`docker exec mesh-controller /mesh-controller status --json`); if (asked.ok) { try { const state = JSON.parse(asked.out) as { wrong: { node: string; outcome: string }[]; waiting: { node: string }[]; reported: { node: string; outcome: string; current: boolean }[]; }; const bad = state.wrong.find((w) => w.node === MACHINE); if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`); const word = state.reported.find((r) => r.node === MACHINE); if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return; last = asked.out; } catch (err) { if (err instanceof Error && err.message.includes("did not apply")) throw err; last = asked.out; } } await new Promise((r) => setTimeout(r, 5000)); } throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`); } /** Install a catalogue module's manifest into the control plane, read from the catalogue. */ async function register(module: string): Promise { const manifest = catalogueModule(module, held); await must( `printf %s ${quote(manifest)} > /tmp/${module}.json && ` + `docker cp /tmp/${module}.json mesh-controller:/${module}.json`, ); await mesh(`module add /${module}.json`); } /** Dial the authority the way anything on this machine would: verifying, with nothing handed to it. */ async function verifying(): Promise<{ out: string; ok: boolean }> { return on(`curl --silent --show-error --max-time 10 ${AUTHORITY}`); } before(async () => { if (skip) return; const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; held = raised.images; await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000); const up = await must(`docker ps --format '{{.Names}}'`); for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`); } // The control plane is a container, and a container that is restarting answers nothing. Asked // until it answers rather than once, so a crash-looping control plane is reported as itself // instead of as whatever command happened to be sent first. let control = { out: "", ok: false }; const answering = Date.now() + 120_000; while (Date.now() < answering) { control = await on(`docker exec mesh-controller /mesh-controller status`); if (control.ok) break; await new Promise((r) => setTimeout(r, 3000)); } assert.ok(control.ok, `the control plane never answered:\n${control.out}\n` + `${(await on(`docker logs mesh-controller 2>&1 | tail -40`)).out}`); await mesh(`node add ${MACHINE}`); const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`)); await must(`${HOST_PATH} enrol --token ${quote(token)}`); await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); }, { timeout: 1_800_000 }); after(async () => { if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 600_000 }); test("the mesh's authority is verified on a machine holding ca-trust, and not on one that is not", { skip, timeout: 1_800_000, }, async () => { // The authority first, on its own. Nothing about trust yet. await register("step-ca"); await mesh(`module issue step-ca --node ${MACHINE}`); await mesh(`assign ${MACHINE} step-ca`); await mesh(`push ${MACHINE}`); await settled(); // It is up and answering — asked the way nothing else in this bed is allowed to ask, with // verification off, because at this point in the bed no machine could verify it. let answering = false; const until = Date.now() + 180_000; while (Date.now() < until && !answering) { answering = (await on(`curl --silent --insecure --max-time 5 ${AUTHORITY}`)).ok; if (!answering) await new Promise((r) => setTimeout(r, 3000)); } assert.ok(answering, `the authority never answered:\n${(await on(`docker logs step-ca 2>&1 | tail -30`)).out}`); // **The negative half, before anything is assigned.** If this passes, the bed is measuring // something already in the machine's trust store and everything below it is worthless. const before = await verifying(); assert.equal(before.ok, false, `the authority verified before anything anchored its root:\n${before.out}`); assert.match(before.out, /certificate|issuer/i, `it failed for some other reason:\n${before.out}`); // Now the module. await register("ca-trust"); await mesh(`assign ${MACHINE} ca-trust`); await mesh(`push ${MACHINE}`); await settled(); const unit = await on(`systemctl is-active mesh-ca-trust.service`); assert.ok(unit.ok, `the unit is ${unit.out.trim()}:\n${(await on(`journalctl -u mesh-ca-trust --no-pager | tail -30`)).out}`); await must(`test -s ${ANCHOR}`); const anchors = await must(`trust list | grep -A2 -i 'Mesh Internal CA' || trust list`); assert.match(anchors, /Mesh Internal CA/, `the mesh's authority is not among the machine's anchors:\n${anchors}`); // **The whole claim, in one command.** No -k, no --cacert: the machine's own trust store, and a // certificate the mesh's authority issued. const after = await verifying(); assert.ok(after.ok, `the authority still does not verify with the module assigned:\n${after.out}`); // **And removal is the same unit's business.** Unassigned, the host stops it; stopping it takes // the anchor away and refreshes the bundles, so the machine stops trusting the mesh. await mesh(`unassign ${MACHINE} ca-trust`); await mesh(`push ${MACHINE}`); await settled(); const gone = await on(`test -e ${ANCHOR}`); assert.equal(gone.ok, false, "the anchor is still on the machine after the module was unassigned"); const afterwards = await verifying(); assert.equal(afterwards.ok, false, `the machine still verifies the mesh's authority with nothing anchoring it:\n${afterwards.out}`); assert.match(afterwards.out, /certificate|issuer/i, `it failed for some other reason:\n${afterwards.out}`); });