/** * The FULL mesh in its REAL production shape: two segments, one access point, one overlay — and the * first multi-segment whole-mesh bed. It rewrites the flat three-node whole-mesh-full (separate * anchor, everything on one public segment) into what production actually is: * * hosting (public) home (private, behind a NAT access point) * novox 192.0.2.20 — the ANCHOR: ace 192.168.1.10 the home server, media/IoT set * substrate (store/broker/ shanks 192.168.1.20 workstation (light: portainer only) * control) + the whole novox g14 192.168.1.30 workstation (light: portainer only) * set + overlay hub + ingress * * There is NO separate anchor: novox IS the anchor. The substrate runs on novox, and novox also * enrols as a node and receives its own service set — the substrate host and a service node at once. * * THE THING THIS BED EXISTS TO PROVE (the flat beds never could): does the WireGuard overlay tunnel * FORM across the access point? A home node (ace/shanks/g14) dials novox's PUBLIC hub endpoint * 192.0.2.20:51820/udp OUT through the household gateway's masquerade; the handshake has to complete * through that NAT and the keepalive has to hold the hole open. Phase A drives exactly this and * verifies it — WireGuard handshake state AND a ping over the overlay from a home node to novox — * BEFORE any heavy module lands, so the cross-segment-overlay verdict survives whatever the module * convergence then does. Phase B converges the full node sets and reports per node. * * SUBSTRATE-ON-NOVOX PORT COLLISIONS (a real consequence of collapsing the anchor onto novox that the * separate-anchor beds never hit): the substrate store binds 127.0.0.1:5432 and novox's postgres * provider publishes 5432; the substrate broker binds 5671 + 127.0.0.1:5672 and novox's lavinmq * provider publishes 5672. The two provider host publishes are REMAPPED off the substrate's ports * (REMAP below); consumers reach the providers over the mesh network on the container port, so the * host side is free to move. Reported as a topology finding. * * PERSISTENT RAISE. With MESH_LAB_KEEP set the instance is raised under a fixed id * (whole-mesh-full-live) and NOT torn down — it is left standing and browsable. Without it the bed * behaves like every other: raise in before(), destroy in after(). * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { existsSync, readFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" : false; const SCENARIO = "whole-mesh-full"; /** novox hosts the substrate and the control plane; it is where `mesh` commands run. */ const CONTROL = "novox"; /** Every node that enrols. novox is on hosting; the rest are behind the home gateway. */ const NODES = ["novox", "ace", "shanks", "g14"]; const HOME_NODES = ["ace", "shanks", "g14"]; /** * ADR 0066 — the domain each public-facing node composes its routed names under. * * **The bed had none, so the ADR was untested by construction.** A module now contributes a `label` * to `route` and nothing else; the mesh joins it to the node's public domain and the join is the * whole feature. On a node with no public domain a labelled contribution composes to nothing — no * host, no route — so every routed module on this bed was silently unreachable and the bed still * went green. Two nodes face outward here; the workstations do not and get none, which is also part * of the design being exercised. * * `.incus` rather than the real domains: this repository's beds name nothing routable. */ const PUBLIC_DOMAIN: Record = { novox: "novox.incus", ace: "zurag.incus" }; /** Keep the instance standing and browsable rather than tearing it down. */ const KEEP = !!process.env["MESH_LAB_KEEP"]; const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined); const catalogDir = process.env["MESH_LAB_CATALOG"] ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); const MEDIA_DIRS = [ "/services/media/series", "/services/media/anime", "/services/media/movies", "/services/media/music", "/services/media/audiobooks", "/services/media/downloads", "/services/media/books", ]; type Mod = { name: string; containers: string[]; node?: boolean; runOnce?: string[] }; /** * The novox set (feat/novox-conversions @ 431310f). The slug fix means only-office/de-spiegel/ * amqp-email-forwarder now resolve (their minted login was over the 20-char cap before), so they * are INCLUDED. CORE gates; the rest are reported gaps (documented in the whole-mesh-novox bed): * umami (provisioner url/admin unset), mailu (nox-schema gaps), only-office/de-spiegel (new plain * apps, boot secondary), amqp-email-forwarder (hard-coded AMQP vhost authz), firewall/fail2ban * (offline lab cannot fetch the package). */ const NOVOX: Mod[] = [ { name: "postgres", containers: ["postgres", "mesh-postgres"] }, { name: "redis", containers: ["redis", "mesh-redis"] }, { name: "minio", containers: ["minio", "mesh-minio"] }, { name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, { name: "mssql", containers: ["mssql", "mesh-mssql"] }, { name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] }, // ADR 0066: the proxy now REQUIRES an `acme-ca`, so the bed must assign a provider of one or // route-proxy is unresolvable and takes every routed module down with it. step-ca is that // provider, on the anchor, at mesh scope. { name: "step-ca", containers: ["step-ca"] }, { name: "route-proxy", containers: ["route-proxy"] }, { name: "keycloak", containers: ["keycloak", "mesh-keycloak"] }, { name: "gitea", containers: ["gitea", "mesh-gitea"] }, { name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] }, { name: "umami", containers: ["umami", "mesh-umami"] }, { name: "photos", containers: ["photos-server", "photos-admin-client", "photos-client-eef", "photos-client-filip"] }, { name: "invoicing", containers: ["invoicing-app", "invoicing-api"] }, { name: "novox.be", containers: ["novox-be"] }, { name: "only-office", containers: ["office-novox-be"] }, { name: "de-spiegel", containers: ["de-spiegel-novox-be"] }, { name: "amqp-email-forwarder", containers: ["amqp-email-forwarder"] }, { name: "portainer", containers: ["portainer", "mesh-portainer"] }, { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] }, { name: "registry", containers: ["mesh-registry"] }, { name: "mailu", containers: [ "mailu-resolver", "mailu-redis", "mailu-admin", "mailu-imap", "mailu-smtp", "mailu-antispam", "mailu-antivirus", "mailu-webmail", "mailu-webdav", "mailu-fetchmail", "mailu-front", "mesh-mailu", ], }, { name: "firewall", containers: [], node: true }, { name: "fail2ban", containers: [], node: true }, ]; const CORE_NOVOX = new Set([ "postgres", "redis", "minio", "mongodb", "mssql", "lavinmq", "route-proxy", "keycloak", "gitea", "nextcloud", "invoicing", "photos", "novox.be", "portainer", "verdaccio", "registry", ]); const GAPS_NOVOX = new Set([ "umami", "mailu", "firewall", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder", // step-ca is reported, not gated: the internal-CA ISSUANCE path is still being fixed in // mesh-control, and this bed is not the place to discover that a fix has not landed yet. What is // gated is the half that is decided and cheap — see the ADR 0066 section at the end. "step-ca", ]); /** The ace media/home set. */ const ACE: Mod[] = [ { name: "postgres", containers: ["postgres", "mesh-postgres"] }, { name: "redis", containers: ["redis", "mesh-redis"] }, { name: "mssql", containers: ["mssql", "mesh-mssql"] }, { name: "sonarr", containers: ["sonarr", "mesh-sonarr"] }, { name: "radarr", containers: ["radarr", "mesh-radarr"] }, { name: "lidarr", containers: ["lidarr", "mesh-lidarr"] }, { name: "plex", containers: ["plex", "mesh-plex"] }, { name: "bazarr", containers: ["bazarr", "mesh-bazarr"] }, { name: "nzbget", containers: ["nzbget", "mesh-nzbget"] }, { name: "qbittorrent", containers: ["qbittorrent", "mesh-qbittorrent"] }, { name: "jackett", containers: ["jackett", "mesh-jackett"] }, { name: "ombi", containers: ["ombi", "mesh-ombi"] }, { name: "tautulli", containers: ["tautulli", "mesh-tautulli"] }, { name: "bookshelf", containers: ["bookshelf", "mesh-bookshelf"] }, { name: "home-assistant", containers: ["home-assistant", "mesh-home-assistant"] }, { name: "mosquitto", containers: ["mosquitto", "mesh-mosquitto"], runOnce: ["mosquitto-bootstrap"] }, { name: "influxdb", containers: ["influxdb", "mesh-influxdb"] }, { name: "grafana", containers: ["grafana", "mesh-grafana"] }, { name: "baserow", containers: ["baserow", "mesh-baserow"] }, { name: "letta", containers: ["letta", "mesh-letta"] }, { name: "nodered", containers: ["nodered", "mesh-nodered"] }, { name: "searxng", containers: ["valkey", "searxng", "mesh-searxng"] }, { name: "unifi", containers: ["unifi-controller", "mesh-unifi"] }, { name: "portainer", containers: ["portainer", "mesh-portainer"] }, ]; const CORE_ACE = new Set([ "postgres", "redis", "mssql", "sonarr", "radarr", "lidarr", "jackett", "tautulli", "bookshelf", "mosquitto", "influxdb", "grafana", "baserow", "nodered", "searxng", "unifi", "portainer", ]); const GAPS_ACE = new Set(["plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta"]); /** The two workstations run one light module each, to prove a real module converges and joins the overlay. */ const LIGHT: Mod[] = [{ name: "portainer", containers: ["portainer", "mesh-portainer"] }]; const CORE_LIGHT = new Set(["portainer"]); const GAPS_LIGHT = new Set(); const PLAN: { node: string; mods: Mod[]; core: Set; gaps: Set }[] = [ { node: "novox", mods: NOVOX, core: CORE_NOVOX, gaps: GAPS_NOVOX }, { node: "ace", mods: ACE, core: CORE_ACE, gaps: GAPS_ACE }, { node: "shanks", mods: LIGHT, core: CORE_LIGHT, gaps: GAPS_LIGHT }, { node: "g14", mods: LIGHT, core: CORE_LIGHT, gaps: GAPS_LIGHT }, ]; /** * Host-port remaps (per module; host ports are per-VM so novox's and ace's never clash across nodes). * The two SUBSTRATE collisions are the new ones: postgres 5432 and lavinmq 5672 are moved off the * substrate store/broker's host ports, which only exist on novox because that is where the substrate * runs. The rest break the novox web/app host-port collisions (route-proxy fronts 80/443). */ const REMAP: Record> = { postgres: { "5432": "127.0.0.1:15432:5432" }, lavinmq: { "5672": "127.0.0.1:15673:5672" }, nextcloud: { "80": "8090:80" }, umami: { "3000": "3090:3000" }, invoicing: { "80": "8091:80", "9000": "9091:9000" }, qbittorrent: { "8080": "8090:8080" }, searxng: { "8080": "8092:8080" }, nzbget: { "6789": "6790:6789" }, }; /** Operator-provided app credentials, delivered as fake values through the real `secret accept` path. */ const CREDENTIALS: { node: string; module: string; name: string; crash: string }[] = [ { node: "ace", module: "plex", name: "token", crash: "no Plex token" }, { node: "ace", module: "bazarr", name: "api-key", crash: "no Bazarr API key" }, { node: "ace", module: "ombi", name: "api-key", crash: "no Ombi API key" }, { node: "ace", module: "home-assistant", name: "token", crash: "no Home Assistant token" }, { node: "ace", module: "nzbget", name: "password", crash: "NZBGet not configured" }, { node: "ace", module: "qbittorrent", name: "password", crash: "qBittorrent not configured" }, { node: "novox", module: "umami", name: "admin", crash: "admin password is not set" }, ]; /** Operator secrets for the credential modules that own-secret their whole app (mailu, de-spiegel). */ const OPERATOR_SECRETS: { node: string; module: string; name: string; value: string }[] = [ { node: "novox", module: "mailu", name: "secret-key", value: "0123456789abcdef0123456789abcdef" }, { node: "novox", module: "mailu", name: "admin", value: "MailuAdminFakePass123" }, { node: "novox", module: "mailu", name: "api-token", value: "mailuapitokenfake0123456789abcd" }, { node: "novox", module: "de-spiegel", name: "smtp-user", value: "despiegel-smtp-fake" }, { node: "novox", module: "de-spiegel", name: "smtp-pass", value: "despiegel-pass-fake" }, { node: "novox", module: "amqp-email-forwarder", name: "smtp-user", value: "eef-smtp-fake" }, { node: "novox", module: "amqp-email-forwarder", name: "smtp-pass", value: "eef-pass-fake" }, ]; let instanceId = ""; let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { const { stdout } = await exec(instanceId, machine, [ "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, ], timeoutMs); const marker = stdout.lastIndexOf("__exit="); if (marker < 0) return { out: stdout, ok: false }; return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; } async function must(machine: string, command: string, timeoutMs?: number): Promise { const { out, ok } = await on(machine, command, timeoutMs); if (!ok) throw new Error(`${machine}: ${command}\n${out}`); return out; } /** The control plane, a container on novox (the anchor). */ async function mesh(command: string, timeoutMs?: number): Promise { return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } /** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ function pinned(reference: string): string { return onTheMachine(reference, held); } function bundleFor(images: HeldImage[]): string { return substrateBundle(bundle, images); } function loadManifest(name: string): { manifest: string; broker: boolean } { const path = resolve(catalogDir, name, "module.json"); const m = JSON.parse(readFileSync(path, "utf8")) as { resources?: { type: string; image?: string; ports?: string[] }[]; }; const remap = REMAP[name] ?? {}; for (const r of m.resources ?? []) { if (r.type !== "container") continue; if (typeof r.image === "string") r.image = pinned(r.image); if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); } const manifest = JSON.stringify(m); return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; } function tokenFrom(said: string): string { const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); assert.ok(found, `no token in:\n${said}`); return found; } interface NodeState { reached: boolean; applied: boolean; current: boolean; waiting: boolean; wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined; raw: string; } async function nodeState(node: string): Promise { const asked = await on(CONTROL, `docker exec mesh-control /mesh-control status --json`); if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; let state: { wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[]; waiting: { node: string }[]; reported: { node: string; outcome: string; current: boolean }[]; }; try { state = JSON.parse(asked.out); } catch { return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; } const word = state.reported.find((r) => r.node === node); const bad = state.wrong.find((w) => w.node === node); return { reached: true, applied: word?.outcome === "applied", current: !!word?.current, waiting: state.waiting.some((w) => w.node === node), wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined, raw: asked.out, }; } async function psMapOf(node: string): Promise> { const out = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; const map = new Map(); for (const line of out.split("\n")) { const [n, ...rest] = line.split("\t"); if (n) map.set(n.trim(), rest.join("\t").trim()); } return map; } /** * ADR 0066: the internal CA is initialised FROM AN OPERATOR'S ROOT — it does not mint its own. * * So the bed has to be an operator. The material is made on the anchor with openssl and handed to * the mesh through the real `secret accept` path, exactly as a person would: the mesh cannot invent * a PEM, and the random 32 bytes it makes for an own-secret nobody supplied would leave step-ca * crash-looping on a root key that is not a key. */ async function deliverCaRoot(): Promise { const made = await on(CONTROL, [ "set -e", "mkdir -p /tmp/ca && cd /tmp/ca", // No trailing newline on a password file: step-ca reads the file as the password itself. "openssl rand -hex 16 | tr -d '\\n' > key-password", "openssl ecparam -genkey -name prime256v1 -out root.unenc", "openssl ec -in root.unenc -aes256 -passout file:key-password -out root.key", "rm -f root.unenc", "openssl req -x509 -new -key root.key -passin file:key-password -sha256 -days 3650" + ` -out root.crt -subj "/CN=Mesh Internal CA/O=Novox Mesh Lab"`, // Readable by the control plane, which is not root. Its image is FROM scratch and runs as // 65534, and `docker cp` keeps the ownership and mode a file had outside — openssl writes a // private key 0600 root-owned, so the copy landed unreadable and `secret accept` failed with // `open /ca-root-key: permission denied`. The CA then crash-looped on a root it never got. // Chowning it inside the container is not available: there is no shell in there to do it with. // // Safe here and nowhere else: these three exist for the seconds between being written and // being sealed to the machine, on a lab node, for a CA thrown away with the scenario. "chmod 0644 /tmp/ca/root.crt /tmp/ca/root.key /tmp/ca/key-password", "docker cp /tmp/ca/root.crt mesh-control:/ca-root-cert", "docker cp /tmp/ca/root.key mesh-control:/ca-root-key", "docker cp /tmp/ca/key-password mesh-control:/ca-root-key-password", ].join("\n"), 180_000); if (!made.ok) { console.log(`CA ROOT NOT MADE on ${CONTROL}:\n${made.out.split("\n").slice(-8).join("\n")}`); return false; } for (const [name, file] of [ ["root-cert", "/ca-root-cert"], ["root-key", "/ca-root-key"], ["root-key-password", "/ca-root-key-password"], ] as const) { try { await mesh(`secret accept ${CONTROL} step-ca ${name} --from ${file}`); } catch (err) { console.log(`CA ROOT ACCEPT FAILED (${name}): ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`); return false; } } return true; } /** What a module's manifest says its route label is, or "" if it contributes no route. */ function routeLabelOf(name: string): string { const path = resolve(catalogDir, name, "module.json"); const m = JSON.parse(readFileSync(path, "utf8")) as { contributes?: { route?: { label?: string } }; }; return m.contributes?.route?.label ?? ""; } /** A node's overlay (mesh0) address, or "" if it has none yet. */ async function overlayAddr(node: string): Promise { const out = (await on(node, `ip -4 -o addr show mesh0 2>/dev/null | awk '{print $4}' | cut -d/ -f1`)).out; return out.split("\n").map((l) => l.trim()).find(Boolean) ?? ""; } before(async () => { if (skip) return; assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), }); instanceId = raised.instanceId; held = raised.images; console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); // novox raises the substrate from its bundle. The store and the broker keep upstream references // and novox PULLS them, over its uplink, the way any first node does; mesh-control exists in no // registry, so it becomes the ID novox holds it under — the whole of what changed here. // // The rest is the collapse: the substrate rides novox, not a separate anchor. The bundle hardcodes the // broker's advertised address as 192.0.2.10:5671 (the OLD separate-anchor address) — and a token // carries MESH_BROKER_ADDRESS verbatim as the endpoint an enrolling node dials. With the substrate // on novox that endpoint must be novox's own public address, or every node (novox included) would // enrol against a dead address. The broker serves its cert on all interfaces and the token pins by // fingerprint, not hostname, so only the address needs correcting. const bundleText = bundleFor(raised.images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671"); await must(CONTROL, `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleText}\nMESHBUNDLE`); // **Belt as well as braces on fetching.** `raise` refuses to return until every machine with // egress has resolved a name and reached the internet, so the first attempt should be the only // one. This retry is here because of what the failure looked like when there was no such // guarantee: the apply died on a pull, `before` threw, and the instance was left a bare shell — // VMs, no substrate, no enrolment, nothing to read. // // And a pull is now genuinely the one step that can fail for a reason which goes away by itself: // the store and the broker come from the internet, through a household gateway's masquerade, and // a registry elsewhere having a bad minute is not this mesh's fault. That is the trade this bed // accepts — it is no longer hermetic, because a real node is not either, and the faults it was // hiding were exactly the ones that only appear when a machine has to fetch for itself. { let applied = false; let said = ""; for (let attempt = 1; attempt <= 3 && !applied; attempt++) { const tried = await on(CONTROL, `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); applied = tried.ok; said = tried.out; if (!applied && attempt < 3) { console.log(`substrate apply attempt ${attempt} failed; retrying in 30s:\n${said.split("\n").slice(-8).join("\n")}`); await new Promise((r) => setTimeout(r, 30_000)); } } assert.ok(applied, `the substrate did not apply on novox after three attempts:\n${said}`); } const up = await must(CONTROL, `docker ps --format '{{.Names}}'`); for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); } // Every node joins the one mesh and runs a host. The home nodes reach novox's public 192.0.2.20:5671 // by dialling OUT through the household gateway — the enrol itself is the first proof that outbound // home→public works. novox enrols too: substrate host and service node at once. for (const machine of NODES) { await mesh(`node add ${machine}`); // ADR 0066: said as soon as the record exists, because everything routed is composed from it. // A node that faces the outside has one; the workstations do not, and are given none. const domain = PUBLIC_DOMAIN[machine]; if (domain) await mesh(`node public-domain ${machine} ${domain}`); const token = tokenFrom(await mesh(`token issue --node ${machine}`)); const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); assert.match(said, new RegExp(`enrolled as ${machine}`), said); await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); } // The operator provides ace's media library (ADR 0051 accesses confirm the paths, create nothing). await must("ace", `mkdir -p ${MEDIA_DIRS.join(" ")}`); }, { timeout: 3_600_000 }); after(async () => { if (KEEP) { console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`); return; } if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 900_000 }); test("the full mesh forms across the access point and both server sets converge", { skip, timeout: 5_400_000, }, async () => { // ================================================================================================ // PHASE A — THE HEADLINE. Place the overlay (hub on novox at its public endpoint; the home nodes // dial out, no endpoint of their own), assign networking to every node, push, and VERIFY the tunnel // forms ACROSS the gateway. This runs BEFORE any heavy module, so the cross-segment-overlay verdict // is captured whatever the module convergence then does. // ================================================================================================ await mesh("overlay place novox --hub --endpoint 192.0.2.20:51820 --site hosting"); for (const node of HOME_NODES) await mesh(`overlay place ${node} --site home`); for (const node of NODES) await mesh(`assign ${node} networking`); for (const node of NODES) { try { await mesh(`push ${node}`, 180_000); } catch (err) { console.log(`networking push rejected (${node}): ${(err as Error).message.split("\n").slice(0, 4).join(" | ")}`); } } // Give the home nodes time to dial the hub and complete a handshake through the NAT. const overlay: Record = {}; const deadline = Date.now() + 300_000; while (Date.now() < deadline) { for (const node of NODES) if (!overlay[node]) overlay[node] = await overlayAddr(node); if (NODES.every((n) => overlay[n])) break; await new Promise((r) => setTimeout(r, 8000)); } // A little longer for handshakes to settle (keepalive interval). await new Promise((r) => setTimeout(r, 30000)); const overlayReport: string[] = ["================ CROSS-SEGMENT OVERLAY (the headline) ================"]; for (const node of NODES) overlayReport.push(` ${node.padEnd(8)} mesh0 = ${overlay[node] || "NONE"}`); // The hub's WireGuard peers and their handshakes, from novox. const hubWg = (await on("novox", `wg show 2>&1 || echo 'wg tool absent'`)).out; overlayReport.push(`\n---- novox (hub) wg show ----\n${hubWg}`); // From each home node: its wg peer state (endpoint should be 192.0.2.20:51820, with a recent // handshake) AND a ping to novox's overlay address — the functional proof the tunnel carries // traffic across the gateway. const overlayFormed: Record = {}; const novoxOverlay = overlay["novox"] ?? ""; for (const node of HOME_NODES) { const wg = (await on(node, `wg show 2>&1 || echo 'wg tool absent'`)).out; const handshake = (await on(node, `wg show all latest-handshakes 2>/dev/null | awk '{print $2}' | sort -rn | head -1`)).out.trim(); const ping = novoxOverlay ? await on(node, `ping -c 3 -W 2 ${novoxOverlay} 2>&1 | tail -3`) : { out: "novox has no overlay address to ping", ok: false }; const handshakeSecs = Number(handshake) || 0; // Formed = we can reach novox over the overlay from this home node (traffic across the NAT). overlayFormed[node] = ping.ok; overlayReport.push(`\n---- ${node} (home) ----`); overlayReport.push(wg.split("\n").map((l) => ` ${l}`).join("\n")); overlayReport.push(` latest-handshake epoch: ${handshake || "none"}${handshakeSecs ? "" : " (no handshake recorded)"}`); overlayReport.push(` ping novox(${novoxOverlay}) over overlay: ${ping.ok ? "REPLIES" : "NO REPLY"}`); overlayReport.push(ping.out.split("\n").map((l) => ` ${l}`).join("\n")); } const anyHomeFormed = HOME_NODES.some((n) => overlayFormed[n]); const allHomeFormed = HOME_NODES.every((n) => overlayFormed[n]); overlayReport.push(`\nVERDICT: overlay across the access point ${allHomeFormed ? "FORMED for all home nodes" : anyHomeFormed ? "FORMED for some home nodes" : "DID NOT FORM"}.`); const overlaySummary = overlayReport.join("\n"); console.log(overlaySummary); // ================================================================================================ // PHASE B — converge the full node sets on top of the overlay. // ================================================================================================ const added = new Map(); async function ensureAdded(name: string): Promise { const known = added.get(name); if (known !== undefined) return known; const { manifest, broker } = loadManifest(name); await must(CONTROL, `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); await mesh(`module add /${name}.json`); added.set(name, broker); return broker; } const assigned: Record> = { novox: new Set(), ace: new Set(), shanks: new Set(), g14: new Set() }; const refused: Record = { novox: [], ace: [], shanks: [], g14: [] }; for (const { node, mods } of PLAN) { for (const { name } of mods) { try { const broker = await ensureAdded(name); if (broker) await mesh(`module issue ${name} --node ${node}`); await mesh(`assign ${node} ${name}`); assigned[node]!.add(name); } catch (err) { const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | "); refused[node]!.push({ name, why }); console.log(`NOT ASSIGNED ${node}/${name}: ${why}`); } } } // Operator-provided app credentials (own-secrets), delivered as fake values through `secret accept`. const credentialDelivered = new Map(); for (const name of new Set(CREDENTIALS.map((c) => c.name))) { await must(CONTROL, `printf %s ${quote(`fake-${name}-value`)} > /tmp/fake-${name} && docker cp /tmp/fake-${name} mesh-control:/fake-${name}`); } for (const c of CREDENTIALS) { if (!assigned[c.node]!.has(c.module)) { credentialDelivered.set(`${c.node}/${c.module}`, false); continue; } try { await mesh(`secret accept ${c.node} ${c.module} ${c.name} --from /fake-${c.name}`); credentialDelivered.set(`${c.node}/${c.module}`, true); } catch (err) { credentialDelivered.set(`${c.node}/${c.module}`, false); console.log(`CREDENTIAL ACCEPT FAILED ${c.node}/${c.module}: ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`); } } // Whole-app own-secrets (mailu/de-spiegel/amqp-email-forwarder). for (const s of OPERATOR_SECRETS) { if (!assigned[s.node]!.has(s.module)) continue; try { const inControl = `/secret-${s.module}-${s.name}`; await must(CONTROL, `printf %s ${quote(s.value)} > /tmp${inControl} && docker cp /tmp${inControl} mesh-control:${inControl}`); await mesh(`secret accept ${s.node} ${s.module} ${s.name} --from ${inControl}`); } catch (err) { console.log(`OPERATOR SECRET FAILED ${s.node}/${s.module}/${s.name}: ${(err as Error).message.split("\n").slice(0, 2).join(" | ")}`); } } // ADR 0066: the CA's root, before the push that would otherwise deliver a random 32 bytes for it. const caRootDelivered = assigned["novox"]!.has("step-ca") ? await deliverCaRoot() : false; if (!caRootDelivered) console.log("ADR 0066: no operator root delivered; step-ca cannot initialise."); // ONE push per node (workstations first — cheap — then the heavy service nodes). const pushError: Record = {}; for (const node of ["shanks", "g14", "novox", "ace"]) { try { await mesh(`push ${node}`, 300_000); } catch (err) { pushError[node] = (err as Error).message; console.log(`PUSH REJECTED (${node}):\n${pushError[node]!.split("\n").slice(0, 6).join("\n")}`); } } // Wait for each node's CORE containers to come up (all nodes pull concurrently from the one registry). const psMaps: Record> = { novox: new Map(), ace: new Map(), shanks: new Map(), g14: new Map() }; for (const { node, mods, core } of PLAN) { if (pushError[node]) continue; const coreContainers = mods.filter((m) => core.has(m.name) && assigned[node]!.has(m.name)).flatMap((m) => m.containers); const until = Date.now() + 3_000_000; while (Date.now() < until) { psMaps[node] = await psMapOf(node); if (coreContainers.every((c) => (psMaps[node]!.get(c) ?? "").startsWith("Up"))) break; await new Promise((r) => setTimeout(r, 10000)); } } await new Promise((r) => setTimeout(r, 20000)); // ================================================================================================ // Per-node convergence report. // ================================================================================================ const users = (await on("novox", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; const allProblems: string[] = []; const report: string[] = ["================ FULL MESH CONVERGENCE ================"]; for (const { node, mods, core, gaps } of PLAN) { const psMap = psMaps[node] = await psMapOf(node); const st = await nodeState(node); const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up"); const ranOnce = (name: string): boolean => !psMap.has(name) || /^(Up|Exited \(0\))/.test(psMap.get(name) ?? ""); const failedResources = st.wrong?.failed ?? []; report.push(`\n---- node ${node}: reached=${st.reached} applied=${st.applied} current=${st.current} waiting=${st.waiting} ----`); if (pushError[node]) report.push(` PUSH REJECTED: ${pushError[node]!.split("\n").slice(0, 6).join("\n ")}`); if (st.wrong) { report.push(` NODE WRONG: outcome=${st.wrong.outcome}`); for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`); } for (const r of refused[node]!) report.push(` REFUSED ${r.name}: ${r.why}`); const coreFailures: string[] = []; for (const mod of mods) { if (!assigned[node]!.has(mod.name)) continue; if (mod.node) { report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(20)} ${core.has(mod.name) ? "CORE" : "gap "} node-service`); continue; } const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`); const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce); const tag = core.has(mod.name) ? (ok ? "OK " : "FAIL") : (ok ? "ok " : "GAP "); report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(20)} ${tag} ${states.join(" ")}`); if (core.has(mod.name) && !ok) coreFailures.push(mod.name); } const issuedHere = mods.filter((m) => new RegExp(`${node}-${m.name}\\b`).test(users)).length; report.push(` broker accounts: ${issuedHere} present for ${node}`); const gapOwnerOf = (f: { id: string; error: string }): string => { const m = f.error.match(/applying "([^".]+)\./); return m?.[1] ?? (f.id.split(".")[0] ?? ""); }; if (pushError[node]) allProblems.push(`${node}: push rejected`); if (coreFailures.length) allProblems.push(`${node}: CORE not converged: ${coreFailures.join(", ")}`); const nonGapFailed = failedResources.filter((f) => !gaps.has(gapOwnerOf(f))); if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`); } const summary = report.join("\n"); console.log(summary); // Diagnostics for any CORE container that did not come up. for (const { node, mods, core } of PLAN) { const psMap = psMaps[node]!; for (const mod of mods) { if (!core.has(mod.name) || !assigned[node]!.has(mod.name)) continue; for (const c of mod.containers) { if (psMap.has(c) && !(psMap.get(c) ?? "").startsWith("Up")) { console.log(`\n---- ${node} logs: ${c} (${psMap.get(c)}) ----\n${(await on(node, `docker logs ${c} 2>&1 | tail -25`)).out}`); } } } } // ================================================================================================ // ADR 0066 — ROUTE NAMES AND THE INTERNAL CA. Additive, and deliberately only the cheap half. // // What is checked here is the part that is DECIDED and costs one file read: a module contributes a // LABEL, the node carries a PUBLIC DOMAIN, and the mesh joins them — `