diff --git a/test/integration/whole-mesh-full.test.ts b/test/integration/whole-mesh-full.test.ts index b41694d..f5233a5 100644 --- a/test/integration/whole-mesh-full.test.ts +++ b/test/integration/whole-mesh-full.test.ts @@ -64,6 +64,20 @@ const CONTROL = "novox"; const NODES = ["novox", "ace", "shanks", "g14"]; const HOME_NODES = ["ace", "shanks", "g14"]; +/** + * ADR 0056 — the domain each public-facing node composes its routed names under. + * + * **The bed had none, so the ADR was untested by construction.** A module now contributes a `label` + * to `route` and nothing else; the mesh joins it to the node's public domain and the join is the + * whole feature. On a node with no public domain a labelled contribution composes to nothing — no + * host, no route — so every routed module on this bed was silently unreachable and the bed still + * went green. Two nodes face outward here; the workstations do not and get none, which is also part + * of the design being exercised. + * + * `.incus` rather than the real domains: this repository's beds name nothing routable. + */ +const PUBLIC_DOMAIN: Record = { novox: "novox.incus", ace: "zurag.incus" }; + /** Keep the instance standing and browsable rather than tearing it down. */ const KEEP = !!process.env["MESH_LAB_KEEP"]; const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined); @@ -95,6 +109,10 @@ const NOVOX: Mod[] = [ { name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, { name: "mssql", containers: ["mssql", "mesh-mssql"] }, { name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] }, + // ADR 0056: the proxy now REQUIRES an `acme-ca`, so the bed must assign a provider of one or + // route-proxy is unresolvable and takes every routed module down with it. step-ca is that + // provider, on the anchor, at mesh scope. + { name: "step-ca", containers: ["step-ca"] }, { name: "route-proxy", containers: ["route-proxy"] }, { name: "keycloak", containers: ["keycloak", "mesh-keycloak"] }, { name: "gitea", containers: ["gitea", "mesh-gitea"] }, @@ -127,6 +145,10 @@ const CORE_NOVOX = new Set([ ]); const GAPS_NOVOX = new Set([ "umami", "mailu", "firewall", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder", + // step-ca is reported, not gated: the internal-CA ISSUANCE path is still being fixed in + // mesh-control, and this bed is not the place to discover that a fix has not landed yet. What is + // gated is the half that is decided and cheap — see the ADR 0056 section at the end. + "step-ca", ]); /** The ace media/home set. */ @@ -329,6 +351,57 @@ async function psMapOf(node: string): Promise> { return map; } +/** + * ADR 0056: the internal CA is initialised FROM AN OPERATOR'S ROOT — it does not mint its own. + * + * So the bed has to be an operator. The material is made on the anchor with openssl and handed to + * the mesh through the real `secret accept` path, exactly as a person would: the mesh cannot invent + * a PEM, and the random 32 bytes it makes for an own-secret nobody supplied would leave step-ca + * crash-looping on a root key that is not a key. + */ +async function deliverCaRoot(): Promise { + const made = await on(CONTROL, [ + "set -e", + "mkdir -p /tmp/ca && cd /tmp/ca", + // No trailing newline on a password file: step-ca reads the file as the password itself. + "openssl rand -hex 16 | tr -d '\\n' > key-password", + "openssl ecparam -genkey -name prime256v1 -out root.unenc", + "openssl ec -in root.unenc -aes256 -passout file:key-password -out root.key", + "rm -f root.unenc", + "openssl req -x509 -new -key root.key -passin file:key-password -sha256 -days 3650" + + ` -out root.crt -subj "/CN=Mesh Internal CA/O=Novox Mesh Lab"`, + "docker cp /tmp/ca/root.crt mesh-control:/ca-root-cert", + "docker cp /tmp/ca/root.key mesh-control:/ca-root-key", + "docker cp /tmp/ca/key-password mesh-control:/ca-root-key-password", + ].join("\n"), 180_000); + if (!made.ok) { + console.log(`CA ROOT NOT MADE on ${CONTROL}:\n${made.out.split("\n").slice(-8).join("\n")}`); + return false; + } + for (const [name, file] of [ + ["root-cert", "/ca-root-cert"], + ["root-key", "/ca-root-key"], + ["root-key-password", "/ca-root-key-password"], + ] as const) { + try { + await mesh(`secret accept ${CONTROL} step-ca ${name} --from ${file}`); + } catch (err) { + console.log(`CA ROOT ACCEPT FAILED (${name}): ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`); + return false; + } + } + return true; +} + +/** What a module's manifest says its route label is, or "" if it contributes no route. */ +function routeLabelOf(name: string): string { + const path = resolve(catalogDir, name, "module.json"); + const m = JSON.parse(readFileSync(path, "utf8")) as { + contributes?: { route?: { label?: string } }; + }; + return m.contributes?.route?.label ?? ""; +} + /** A node's overlay (mesh0) address, or "" if it has none yet. */ async function overlayAddr(node: string): Promise { const out = (await on(node, `ip -4 -o addr show mesh0 2>/dev/null | awk '{print $4}' | cut -d/ -f1`)).out; @@ -356,7 +429,27 @@ before(async () => { // fingerprint, not hostname, so only the address needs correcting. const bundleText = bundleFor(raised.images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671"); await must(CONTROL, `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleText}\nMESHBUNDLE`); - await must(CONTROL, `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); + + // **Belt as well as braces on the registry.** `raise` now refuses to return until every machine + // can fetch a manifest from the scenario registry, so the first attempt should be the only one. + // This retry is here because of what the failure looked like when the guarantee was missing: the + // apply died on a pull, `before` threw, and the instance was left a bare shell — VMs and a + // registry, no substrate, no enrolment, nothing to read. A pull is the one step here that can + // fail for a reason that goes away by itself, so it is the one step worth attempting twice. + { + let applied = false; + let said = ""; + for (let attempt = 1; attempt <= 3 && !applied; attempt++) { + const tried = await on(CONTROL, `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); + applied = tried.ok; + said = tried.out; + if (!applied && attempt < 3) { + console.log(`substrate apply attempt ${attempt} failed; retrying in 30s:\n${said.split("\n").slice(-8).join("\n")}`); + await new Promise((r) => setTimeout(r, 30_000)); + } + } + assert.ok(applied, `the substrate did not apply on novox after three attempts:\n${said}`); + } const up = await must(CONTROL, `docker ps --format '{{.Names}}'`); for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); @@ -367,6 +460,10 @@ before(async () => { // home→public works. novox enrols too: substrate host and service node at once. for (const machine of NODES) { await mesh(`node add ${machine}`); + // ADR 0056: said as soon as the record exists, because everything routed is composed from it. + // A node that faces the outside has one; the workstations do not, and are given none. + const domain = PUBLIC_DOMAIN[machine]; + if (domain) await mesh(`node public-domain ${machine} ${domain}`); const token = tokenFrom(await mesh(`token issue --node ${machine}`)); const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); assert.match(said, new RegExp(`enrolled as ${machine}`), said); @@ -511,6 +608,10 @@ test("the full mesh forms across the access point and both server sets converge" } } + // ADR 0056: the CA's root, before the push that would otherwise deliver a random 32 bytes for it. + const caRootDelivered = assigned["novox"]!.has("step-ca") ? await deliverCaRoot() : false; + if (!caRootDelivered) console.log("ADR 0056: no operator root delivered; step-ca cannot initialise."); + // ONE push per node (workstations first — cheap — then the heavy service nodes). const pushError: Record = {}; for (const node of ["shanks", "g14", "novox", "ace"]) { @@ -600,6 +701,55 @@ test("the full mesh forms across the access point and both server sets converge" } } + // ================================================================================================ + // ADR 0056 — ROUTE NAMES AND THE INTERNAL CA. Additive, and deliberately only the cheap half. + // + // What is checked here is the part that is DECIDED and costs one file read: a module contributes a + // LABEL, the node carries a PUBLIC DOMAIN, and the mesh joins them — `