diff --git a/scenarios/whole-mesh-full.yml b/scenarios/whole-mesh-full.yml index 2845078..078d7ab 100644 --- a/scenarios/whole-mesh-full.yml +++ b/scenarios/whole-mesh-full.yml @@ -1,54 +1,104 @@ -# The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the -# whole-mesh rehearsal (novox/hq). Combines scenarios/whole-mesh-novox.yml and whole-mesh-ace.yml. +# The FULL mesh in its REAL production shape: two segments, one access point, one overlay. # -# anchor — substrate ONLY (store, broker, control). -# novox — the 17-module novox set (providers + web apps + route-proxy + mailu + firewall). -# ace — the 24-module ace set (media/home stack), its /services/media library pre-created. +# This is the first multi-segment whole-mesh bed. The earlier flat whole-mesh-full sat every node +# on one public segment with a SEPARATE `anchor` carrying the substrate. Production is not flat, and +# there is no separate anchor: `novox` IS the anchor. It sits on the routable `hosting` segment, +# runs the substrate (store, broker, control) AND its own service set AND is the overlay hub and the +# public ingress. `ace`, `shanks` and `g14` sit on the household `home` segment BEHIND a NAT gateway +# — the access point — reachable from the outside only through what they dial out to. # -# An overlay is placed across all three so cross-node `at` resolves. Each service node is -# self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and -# the shared broker/store on anchor — which is exactly what this stage proves converges for two -# independent node-plans at once on one substrate. +# hosting (public, routable) home (private, behind the access point) +# novox 192.0.2.20 ── anchor ace 192.168.1.10 home server, media/IoT set +# substrate + novox set shanks 192.168.1.20 workstation (light) +# overlay hub, ingress g14 192.168.1.30 workstation (light) +# +# The `home` gateway masquerades v4 outbound and forwards inbound (an ordinary household router). +# Home nodes reach novox's public 192.0.2.20 by dialling OUT through it: the substrate broker (5671), +# the registry, and — the thing this bed exists to prove — the WireGuard overlay hub (51820/udp). +# The hub keepalive holds the NAT hole open so the tunnel, once formed, stays up. novox cannot +# initiate to a home node at all; every home↔novox path is either the overlay or a forwarded port. +# +# THE UNPROVEN THING (what the flat beds never tested): does the overlay tunnel FORM across the +# access point — a home node dialling novox's public hub endpoint, the handshake completing through +# the gateway's masquerade? The driving test verifies the WireGuard handshake and cross-segment +# reachability over the overlay explicitly, and reports form-vs-break as its headline. +# +# Substrate-on-novox collides on two host ports the separate-anchor beds never hit: the substrate +# store binds 127.0.0.1:5432 and novox's postgres provider publishes 5432; the substrate broker binds +# 5671 + 127.0.0.1:5672 and novox's lavinmq provider publishes 5672. The driving test REMAPS those two +# provider host publishes off the substrate's ports (consumers reach the providers over the mesh +# network on the container port, so the host side is free to move). Reported as a topology finding. # # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock -# The images are the UNION of the two per-server scenarios; every one is already built/pulled by the -# per-server bed prerequisites (scripts/build-module-runtime.sh, build-route-proxy-image.sh, the -# mailu/keycloak and media :mesh digest pulls). +# The images are the UNION of the novox set (feat/novox-conversions @ 431310f: the slug + roundcube +# fixes, so only-office/de-spiegel/amqp-email-forwarder now resolve) and the ace media/home set. Every +# one is already built/pulled by the per-server bed prerequisites. scenario: whole-mesh-full segments: + # The routable segment. novox lives here; the lab raises the image registry here too (a public + # IPv4 segment is what serves the images), and the overlay hub endpoint is a public address here. hosting: kind: public cidr: [192.0.2.0/24] + # The household segment behind the access point. Its gateway is an ordinary home router: it + # masquerades v4 outbound, forwards inbound, and expires idle mappings after two minutes — which + # is exactly the NAT hole a WireGuard keepalive has to hold open. + home: + kind: private + cidr: [192.168.1.0/24] + gateway: + to: hosting + address: [192.0.2.50] # what the world sees the household as + nat: [v4] + forwardable: true + mapping_ttl: 120s + machines: - anchor: - at: { segment: hosting, address: [192.0.2.10] } - inbound: allow - memory: 4GiB - cpus: 4 - disk: 20GiB + # The anchor: substrate (store, broker, control) + the whole novox service set + overlay hub + + # public ingress. Bigger than the flat bed's novox, because it now carries the substrate too. novox: at: { segment: hosting, address: [192.0.2.20] } inbound: allow - memory: 16GiB - cpus: 6 - disk: 100GiB + memory: 24GiB + cpus: 8 + disk: 130GiB + + # The home server: the whole ace media/home set — 24 modules, ~50 containers, several heavy + # (Plex, Home Assistant, Letta, Baserow, the UniFi JVM, mssql). Behind the gateway. ace: - at: { segment: hosting, address: [192.0.2.30] } + at: { segment: home, address: [192.168.1.10] } inbound: allow memory: 18GiB cpus: 6 disk: 120GiB + # Two workstations on the same home LAN. Light on purpose: they enrol, join the overlay, and run + # one small module (portainer) so a real module converges on each without heavy load. Same-LAN + # nodes with no overlay endpoint of their own hairpin the hub rather than peering directly, which + # is the normal case and is fine. + shanks: + at: { segment: home, address: [192.168.1.20] } + inbound: allow + memory: 3GiB + cpus: 2 + disk: 30GiB + g14: + at: { segment: home, address: [192.168.1.30] } + inbound: allow + memory: 3GiB + cpus: 2 + disk: 30GiB + images: - # --- substrate + shared --- + # --- substrate + shared (novox & ace both run postgres/redis/mssql/portainer) --- - postgres:17-alpine - cloudamqp/lavinmq:latest - mesh-control:development - redis:7-alpine - - portainer/portainer-ce:latest - mcr.microsoft.com/mssql/server:2022-latest + - portainer/portainer-ce:latest # --- novox server images --- - minio/minio:latest - mongo:7 @@ -61,13 +111,24 @@ images: - registry:2 - registry-api.novox.be/novox/invoicing-app:latest - registry-api.novox.be/novox/invoicing-api:latest - - ghcr.io/mailu/unbound:mesh - - ghcr.io/mailu/admin:mesh - - ghcr.io/mailu/dovecot:mesh - - ghcr.io/mailu/postfix:mesh - - ghcr.io/mailu/rspamd:mesh - - ghcr.io/mailu/webmail:mesh - - ghcr.io/mailu/nginx:mesh + - onlyoffice/documentserver:mesh + - registry-api.novox.be/novox/de-spiegel:latest + - registry-api.novox.be/novox/www:latest + - registry-api.novox.be/novox/amqp-email-forwarder:latest + - registry-api.novox.be/novox/photos-server:latest + - registry-api.novox.be/novox/photos-admin-client:latest + - registry-api.novox.be/novox/photos-client:latest + # The full Mailu 1.9 stack (mailu-redis reuses redis:7-alpine above). + - ghcr.io/mailu/unbound:1.9 + - ghcr.io/mailu/admin:1.9 + - ghcr.io/mailu/dovecot:1.9 + - ghcr.io/mailu/postfix:1.9 + - ghcr.io/mailu/rspamd:1.9 + - ghcr.io/mailu/clamav:1.9 + - ghcr.io/mailu/roundcube:1.9 + - ghcr.io/mailu/radicale:1.9 + - ghcr.io/mailu/fetchmail:1.9 + - ghcr.io/mailu/nginx:1.9 # --- ace server images --- - lscr.io/linuxserver/sonarr:mesh - lscr.io/linuxserver/radarr:mesh @@ -97,11 +158,11 @@ images: - mesh-runtime-portainer:development - mesh-runtime-minio:development - mesh-runtime-mongodb:development + - mesh-runtime-lavinmq:development - mesh-runtime-keycloak:development - mesh-runtime-gitea:development - mesh-runtime-nextcloud:development - mesh-runtime-umami:development - - mesh-runtime-photos:development - mesh-runtime-verdaccio:development - mesh-runtime-mailu:development - mesh-route-proxy:development diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index 2891dd0..c26ead3 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -321,7 +321,7 @@ export async function raise( let registry: Awaited> = null; try { enter("raising the registry"); - registry = await raiseRegistry(scenario, instanceId, stock, log); + registry = await raiseRegistry(scenario, instanceId, stock, log, pool); } finally { // Cleaning up scratch must not fail a raise that succeeded. The scenario is standing // and usable; a directory left behind is untidy, and saying so is the honest report. diff --git a/src/lifecycle/registry.ts b/src/lifecycle/registry.ts index 4daefd8..682c47b 100644 --- a/src/lifecycle/registry.ts +++ b/src/lifecycle/registry.ts @@ -276,6 +276,13 @@ export async function raiseRegistry( instanceId: string, stock: Stock, log: (message: string) => void = () => {}, + /** + * The copy-on-write pool the scenario's machines were placed on. The registry goes on it too — + * NOT the profile's default `dir` pool — so a sized root disk is thin (paid for as it fills) + * rather than a full allocation on the host's own filesystem. Absent, it falls back to the + * profile default, which is the historical behaviour for a small registry. + */ + pool?: string, ): Promise { if (stock.images.length === 0) return null; @@ -292,10 +299,19 @@ export async function raiseRegistry( const prefix = segment.cidr.slice(segment.cidr.lastIndexOf("/")); if (!(await succeeds(["config", "show", name], 15_000))) { + // The registry holds the WHOLE `images:` union on its own root disk, and the base image's + // default is only ~10GiB. A single-node bed stocks a handful of images and fits; a broad bed + // — and especially the full segmented mesh, whose union is both server sets at once (~28GiB) + // — overflows it, and the raise dies "no space left on device" while pushing blobs into the + // registry. So the registry gets a sized root disk. Thin on a copy-on-write pool, so a small + // bed pays only for what it actually stocks; MESH_LAB_REGISTRY_DISK overrides for a giant one. + const registryDisk = process.env["MESH_LAB_REGISTRY_DISK"] ?? "80GiB"; await incus([ "init", BASE_IMAGE_ALIAS, name, "--vm", "-c", "security.secureboot=false", "-c", "limits.memory=1GiB", + ...(pool ? ["-s", pool] : []), + "-d", `root,size=${registryDisk}`, "-c", `user.mesh-lab.instance=${instanceId}`, // Tagged as a machine as well, so `destroy` finds it with one query — a router that // carried only its own tag was left behind and held its networks open. diff --git a/test/integration/whole-mesh-full.test.ts b/test/integration/whole-mesh-full.test.ts index 447c363..b41694d 100644 --- a/test/integration/whole-mesh-full.test.ts +++ b/test/integration/whole-mesh-full.test.ts @@ -1,38 +1,35 @@ /** - * The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the - * whole-mesh rehearsal (novox/hq). Combines whole-mesh-novox.test.ts and whole-mesh-ace.test.ts. + * The FULL mesh in its REAL production shape: two segments, one access point, one overlay — and the + * first multi-segment whole-mesh bed. It rewrites the flat three-node whole-mesh-full (separate + * anchor, everything on one public segment) into what production actually is: * - * anchor — substrate ONLY (store, broker, control). - * novox — the 18-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu, - * firewall, fail2ban. fail2ban is now HOSTABLE: the dry-run fixes (mesh-control/catalog - * main) changed its declared capability from the never-detected "intrusion-prevention" to - * "firewall", the detector every node with nft already advertises. - * ace — the 24-module ace set (whole-mesh-ace): the media/home stack; its /services/media - * library is pre-created so the ADR-0051 `accesses` resolve. + * hosting (public) home (private, behind a NAT access point) + * novox 192.0.2.20 — the ANCHOR: ace 192.168.1.10 the home server, media/IoT set + * substrate (store/broker/ shanks 192.168.1.20 workstation (light: portainer only) + * control) + the whole novox g14 192.168.1.30 workstation (light: portainer only) + * set + overlay hub + ingress * - * An overlay is placed across all three so cross-node `at` resolves. Each service node is - * self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and - * the shared broker/store on anchor. The four modules both nodes run (postgres, redis, mssql, - * portainer) are ADDED once and assigned to each node; each gets its own per-node broker account. + * There is NO separate anchor: novox IS the anchor. The substrate runs on novox, and novox also + * enrols as a node and receives its own service set — the substrate host and a service node at once. * - * THE DRY-RUN FIXES THIS RUN PROVES (mesh-control + mesh-catalog main): - * - fail2ban is HOSTABLE (capability "firewall"): it is assigned, not refused. Before, it declared - * the never-detected "intrusion-prevention" capability, so no node could host it and its - * un-hostable assignment refused the whole node's push. Hostability is the gate. Its service - * reaching active is a host concern this offline lab cannot meet — the VM ships nftables (so the - * firewall detector is advertised) but not fail2ban, and the isolated segment has no route to the - * package mirror, so pacman cannot fetch it. That is a documented lab gap, reported not gated. - * - the 7 tool-runtime credential modules (ace: plex, bazarr, ombi, home-assistant, nzbget, - * qbittorrent; novox: umami) now read their app credential from an operator-provided own-secret. - * This bed delivers a FAKE value for each through the real operator path (`secret accept`) - * BEFORE the push, and gates on the sidecar getting PAST its old "no credential" crash (it reads - * the delivered value). A fake value will not authenticate against the real app — the sidecar may - * still fail at app-auth, which is expected and does NOT gate; only the crash being GONE gates. + * THE THING THIS BED EXISTS TO PROVE (the flat beds never could): does the WireGuard overlay tunnel + * FORM across the access point? A home node (ace/shanks/g14) dials novox's PUBLIC hub endpoint + * 192.0.2.20:51820/udp OUT through the household gateway's masquerade; the handshake has to complete + * through that NAT and the keepalive has to hold the hole open. Phase A drives exactly this and + * verifies it — WireGuard handshake state AND a ping over the overlay from a home node to novox — + * BEFORE any heavy module lands, so the cross-segment-overlay verdict survives whatever the module + * convergence then does. Phase B converges the full node sets and reports per node. * - * It otherwise tolerates the SAME known gaps the per-server beds proved and escalated (the credential - * sidecars' app-auth failures, photos/mailu, and firewall's oneshot nftables.service); it gates green - * on each node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two - * node-plans converge together on one substrate. + * SUBSTRATE-ON-NOVOX PORT COLLISIONS (a real consequence of collapsing the anchor onto novox that the + * separate-anchor beds never hit): the substrate store binds 127.0.0.1:5432 and novox's postgres + * provider publishes 5432; the substrate broker binds 5671 + 127.0.0.1:5672 and novox's lavinmq + * provider publishes 5672. The two provider host publishes are REMAPPED off the substrate's ports + * (REMAP below); consumers reach the providers over the mesh network on the container port, so the + * host side is free to move. Reported as a topology finding. + * + * PERSISTENT RAISE. With MESH_LAB_KEEP set the instance is raised under a fixed id + * (whole-mesh-full-live) and NOT torn down — it is left standing and browsable. Without it the bed + * behaves like every other: raise in before(), destroy in after(). * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock */ @@ -61,6 +58,15 @@ const skip = !capability.usable : false; const SCENARIO = "whole-mesh-full"; +/** novox hosts the substrate and the control plane; it is where `mesh` commands run. */ +const CONTROL = "novox"; +/** Every node that enrols. novox is on hosting; the rest are behind the home gateway. */ +const NODES = ["novox", "ace", "shanks", "g14"]; +const HOME_NODES = ["ace", "shanks", "g14"]; + +/** Keep the instance standing and browsable rather than tearing it down. */ +const KEEP = !!process.env["MESH_LAB_KEEP"]; +const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined); const catalogDir = process.env["MESH_LAB_CATALOG"] ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") @@ -74,41 +80,56 @@ const MEDIA_DIRS = [ type Mod = { name: string; containers: string[]; node?: boolean; runOnce?: string[] }; -/** The novox node's 17-module set (fail2ban dropped). CORE gates; the rest are documented gaps. */ +/** + * The novox set (feat/novox-conversions @ 431310f). The slug fix means only-office/de-spiegel/ + * amqp-email-forwarder now resolve (their minted login was over the 20-char cap before), so they + * are INCLUDED. CORE gates; the rest are reported gaps (documented in the whole-mesh-novox bed): + * umami (provisioner url/admin unset), mailu (nox-schema gaps), only-office/de-spiegel (new plain + * apps, boot secondary), amqp-email-forwarder (hard-coded AMQP vhost authz), firewall/fail2ban + * (offline lab cannot fetch the package). + */ const NOVOX: Mod[] = [ { name: "postgres", containers: ["postgres", "mesh-postgres"] }, { name: "redis", containers: ["redis", "mesh-redis"] }, { name: "minio", containers: ["minio", "mesh-minio"] }, { name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, { name: "mssql", containers: ["mssql", "mesh-mssql"] }, + { name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] }, + { name: "route-proxy", containers: ["route-proxy"] }, { name: "keycloak", containers: ["keycloak", "mesh-keycloak"] }, { name: "gitea", containers: ["gitea", "mesh-gitea"] }, { name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] }, { name: "umami", containers: ["umami", "mesh-umami"] }, - { name: "photos", containers: ["photos", "mesh-photos"] }, + { name: "photos", containers: ["photos-server", "photos-admin-client", "photos-client-eef", "photos-client-filip"] }, { name: "invoicing", containers: ["invoicing-app", "invoicing-api"] }, + { name: "novox.be", containers: ["novox-be"] }, + { name: "only-office", containers: ["office-novox-be"] }, + { name: "de-spiegel", containers: ["de-spiegel-novox-be"] }, + { name: "amqp-email-forwarder", containers: ["amqp-email-forwarder"] }, { name: "portainer", containers: ["portainer", "mesh-portainer"] }, { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] }, { name: "registry", containers: ["mesh-registry"] }, - { name: "route-proxy", containers: ["route-proxy"] }, { name: "mailu", containers: [ - "mailu-resolver", "mailu-redis", "mailu-admindb", "mailu-admin", "mailu-imap", - "mailu-smtp", "mailu-antispam", "mailu-webmail", "mailu-front", "mesh-mailu", + "mailu-resolver", "mailu-redis", "mailu-admin", "mailu-imap", "mailu-smtp", + "mailu-antispam", "mailu-antivirus", "mailu-webmail", "mailu-webdav", "mailu-fetchmail", + "mailu-front", "mesh-mailu", ], }, { name: "firewall", containers: [], node: true }, { name: "fail2ban", containers: [], node: true }, ]; const CORE_NOVOX = new Set([ - "postgres", "redis", "minio", "mongodb", "mssql", - "keycloak", "gitea", "nextcloud", "invoicing", - "portainer", "verdaccio", "registry", "route-proxy", + "postgres", "redis", "minio", "mongodb", "mssql", "lavinmq", + "route-proxy", "keycloak", "gitea", "nextcloud", "invoicing", "photos", "novox.be", + "portainer", "verdaccio", "registry", +]); +const GAPS_NOVOX = new Set([ + "umami", "mailu", "firewall", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder", ]); -const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall", "fail2ban"]); -/** The ace node's 24-module set. */ +/** The ace media/home set. */ const ACE: Mod[] = [ { name: "postgres", containers: ["postgres", "mesh-postgres"] }, { name: "redis", containers: ["redis", "mesh-redis"] }, @@ -142,16 +163,27 @@ const CORE_ACE = new Set([ ]); const GAPS_ACE = new Set(["plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta"]); +/** The two workstations run one light module each, to prove a real module converges and joins the overlay. */ +const LIGHT: Mod[] = [{ name: "portainer", containers: ["portainer", "mesh-portainer"] }]; +const CORE_LIGHT = new Set(["portainer"]); +const GAPS_LIGHT = new Set(); + const PLAN: { node: string; mods: Mod[]; core: Set; gaps: Set }[] = [ { node: "novox", mods: NOVOX, core: CORE_NOVOX, gaps: GAPS_NOVOX }, { node: "ace", mods: ACE, core: CORE_ACE, gaps: GAPS_ACE }, + { node: "shanks", mods: LIGHT, core: CORE_LIGHT, gaps: GAPS_LIGHT }, + { node: "g14", mods: LIGHT, core: CORE_LIGHT, gaps: GAPS_LIGHT }, ]; /** - * Host-port remaps (per module — host ports are per-VM, so novox's and ace's never clash). Union of - * both per-server beds' remaps. + * Host-port remaps (per module; host ports are per-VM so novox's and ace's never clash across nodes). + * The two SUBSTRATE collisions are the new ones: postgres 5432 and lavinmq 5672 are moved off the + * substrate store/broker's host ports, which only exist on novox because that is where the substrate + * runs. The rest break the novox web/app host-port collisions (route-proxy fronts 80/443). */ const REMAP: Record> = { + postgres: { "5432": "127.0.0.1:15432:5432" }, + lavinmq: { "5672": "127.0.0.1:15673:5672" }, nextcloud: { "80": "8090:80" }, umami: { "3000": "3090:3000" }, invoicing: { "80": "8091:80", "9000": "9091:9000" }, @@ -160,15 +192,7 @@ const REMAP: Record> = { nzbget: { "6789": "6790:6789" }, }; -/** - * The 7 tool-runtime credential modules (novox/hq dry-run fix). Each now reads its app credential - * from an operator-provided own-secret (`name`, an own-secret path in its module.json), mounted into - * the sidecar at MESH_*_FILE. This bed delivers a FAKE value for each via the real operator path - * (`secret accept --from `) BEFORE the push, and asserts the sidecar - * gets PAST `crash` — the exact message its client threw when nothing was mounted. A fake value does - * not authenticate against the real app, so the sidecar may still fail later at app-auth (expected, - * not gated); only the "no credential" crash being GONE proves the wiring and gates. - */ +/** Operator-provided app credentials, delivered as fake values through the real `secret accept` path. */ const CREDENTIALS: { node: string; module: string; name: string; crash: string }[] = [ { node: "ace", module: "plex", name: "token", crash: "no Plex token" }, { node: "ace", module: "bazarr", name: "api-key", crash: "no Bazarr API key" }, @@ -179,6 +203,17 @@ const CREDENTIALS: { node: string; module: string; name: string; crash: string } { node: "novox", module: "umami", name: "admin", crash: "admin password is not set" }, ]; +/** Operator secrets for the credential modules that own-secret their whole app (mailu, de-spiegel). */ +const OPERATOR_SECRETS: { node: string; module: string; name: string; value: string }[] = [ + { node: "novox", module: "mailu", name: "secret-key", value: "0123456789abcdef0123456789abcdef" }, + { node: "novox", module: "mailu", name: "admin", value: "MailuAdminFakePass123" }, + { node: "novox", module: "mailu", name: "api-token", value: "mailuapitokenfake0123456789abcd" }, + { node: "novox", module: "de-spiegel", name: "smtp-user", value: "despiegel-smtp-fake" }, + { node: "novox", module: "de-spiegel", name: "smtp-pass", value: "despiegel-pass-fake" }, + { node: "novox", module: "amqp-email-forwarder", name: "smtp-user", value: "eef-smtp-fake" }, + { node: "novox", module: "amqp-email-forwarder", name: "smtp-pass", value: "eef-pass-fake" }, +]; + let instanceId = ""; let stocked: string[] = []; @@ -201,8 +236,9 @@ async function must(machine: string, command: string, timeoutMs?: number): Promi return out; } +/** The control plane, a container on novox (the anchor). */ async function mesh(command: string, timeoutMs?: number): Promise { - return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); + return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } function repositoryFor(reference: string): string { @@ -259,7 +295,7 @@ interface NodeState { } async function nodeState(node: string): Promise { - const asked = await on("anchor", `docker exec mesh-control /mesh-control status --json`); + const asked = await on(CONTROL, `docker exec mesh-control /mesh-control status --json`); if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; let state: { wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[]; @@ -293,64 +329,143 @@ async function psMapOf(node: string): Promise> { return map; } +/** A node's overlay (mesh0) address, or "" if it has none yet. */ +async function overlayAddr(node: string): Promise { + const out = (await on(node, `ip -4 -o addr show mesh0 2>/dev/null | awk '{print $4}' | cut -d/ -f1`)).out; + return out.split("\n").map((l) => l.trim()).find(Boolean) ?? ""; +} + before(async () => { if (skip) return; assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), + ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), }); instanceId = raised.instanceId; stocked = raised.images; + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); - await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); - await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); - const up = await must("anchor", `docker ps --format '{{.Names}}'`); + // novox raises the substrate from its bundle, digests rewritten to the scenario registry's. This + // is the collapse: the substrate rides novox, not a separate anchor. The bundle hardcodes the + // broker's advertised address as 192.0.2.10:5671 (the OLD separate-anchor address) — and a token + // carries MESH_BROKER_ADDRESS verbatim as the endpoint an enrolling node dials. With the substrate + // on novox that endpoint must be novox's own public address, or every node (novox included) would + // enrol against a dead address. The broker serves its cert on all interfaces and the token pins by + // fingerprint, not hostname, so only the address needs correcting. + const bundleText = bundleFor(raised.images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671"); + await must(CONTROL, `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleText}\nMESHBUNDLE`); + await must(CONTROL, `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); + const up = await must(CONTROL, `docker ps --format '{{.Names}}'`); for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); } - for (const machine of ["anchor", "novox", "ace"]) { + // Every node joins the one mesh and runs a host. The home nodes reach novox's public 192.0.2.20:5671 + // by dialling OUT through the household gateway — the enrol itself is the first proof that outbound + // home→public works. novox enrols too: substrate host and service node at once. + for (const machine of NODES) { await mesh(`node add ${machine}`); const token = tokenFrom(await mesh(`token issue --node ${machine}`)); - const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); + const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); assert.match(said, new RegExp(`enrolled as ${machine}`), said); await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); } // The operator provides ace's media library (ADR 0051 accesses confirm the paths, create nothing). await must("ace", `mkdir -p ${MEDIA_DIRS.join(" ")}`); -}, { timeout: 3_000_000 }); +}, { timeout: 3_600_000 }); after(async () => { + if (KEEP) { + console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`); + return; + } if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 900_000 }); -test("both server sets converge together on one substrate", { skip, timeout: 3_600_000 }, async () => { - // Overlay across all three, so every node's private address exists and cross-node `at` resolves. - await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); - await mesh("overlay place novox --site lab"); - await mesh("overlay place ace --site lab"); - await mesh("assign anchor networking"); - await mesh("assign novox networking"); - await mesh("assign ace networking"); +test("the full mesh forms across the access point and both server sets converge", { + skip, timeout: 5_400_000, +}, async () => { + // ================================================================================================ + // PHASE A — THE HEADLINE. Place the overlay (hub on novox at its public endpoint; the home nodes + // dial out, no endpoint of their own), assign networking to every node, push, and VERIFY the tunnel + // forms ACROSS the gateway. This runs BEFORE any heavy module, so the cross-segment-overlay verdict + // is captured whatever the module convergence then does. + // ================================================================================================ + await mesh("overlay place novox --hub --endpoint 192.0.2.20:51820 --site hosting"); + for (const node of HOME_NODES) await mesh(`overlay place ${node} --site home`); + for (const node of NODES) await mesh(`assign ${node} networking`); + for (const node of NODES) { + try { + await mesh(`push ${node}`, 180_000); + } catch (err) { + console.log(`networking push rejected (${node}): ${(err as Error).message.split("\n").slice(0, 4).join(" | ")}`); + } + } - // Add every unique module ONCE (the four shared modules are added once, assigned to each node), then - // issue a per-node broker account and assign, resiliently. - const added = new Map(); // name -> needs broker + // Give the home nodes time to dial the hub and complete a handshake through the NAT. + const overlay: Record = {}; + const deadline = Date.now() + 300_000; + while (Date.now() < deadline) { + for (const node of NODES) if (!overlay[node]) overlay[node] = await overlayAddr(node); + if (NODES.every((n) => overlay[n])) break; + await new Promise((r) => setTimeout(r, 8000)); + } + // A little longer for handshakes to settle (keepalive interval). + await new Promise((r) => setTimeout(r, 30000)); + + const overlayReport: string[] = ["================ CROSS-SEGMENT OVERLAY (the headline) ================"]; + for (const node of NODES) overlayReport.push(` ${node.padEnd(8)} mesh0 = ${overlay[node] || "NONE"}`); + + // The hub's WireGuard peers and their handshakes, from novox. + const hubWg = (await on("novox", `wg show 2>&1 || echo 'wg tool absent'`)).out; + overlayReport.push(`\n---- novox (hub) wg show ----\n${hubWg}`); + + // From each home node: its wg peer state (endpoint should be 192.0.2.20:51820, with a recent + // handshake) AND a ping to novox's overlay address — the functional proof the tunnel carries + // traffic across the gateway. + const overlayFormed: Record = {}; + const novoxOverlay = overlay["novox"] ?? ""; + for (const node of HOME_NODES) { + const wg = (await on(node, `wg show 2>&1 || echo 'wg tool absent'`)).out; + const handshake = (await on(node, `wg show all latest-handshakes 2>/dev/null | awk '{print $2}' | sort -rn | head -1`)).out.trim(); + const ping = novoxOverlay + ? await on(node, `ping -c 3 -W 2 ${novoxOverlay} 2>&1 | tail -3`) + : { out: "novox has no overlay address to ping", ok: false }; + const handshakeSecs = Number(handshake) || 0; + // Formed = we can reach novox over the overlay from this home node (traffic across the NAT). + overlayFormed[node] = ping.ok; + overlayReport.push(`\n---- ${node} (home) ----`); + overlayReport.push(wg.split("\n").map((l) => ` ${l}`).join("\n")); + overlayReport.push(` latest-handshake epoch: ${handshake || "none"}${handshakeSecs ? "" : " (no handshake recorded)"}`); + overlayReport.push(` ping novox(${novoxOverlay}) over overlay: ${ping.ok ? "REPLIES" : "NO REPLY"}`); + overlayReport.push(ping.out.split("\n").map((l) => ` ${l}`).join("\n")); + } + const anyHomeFormed = HOME_NODES.some((n) => overlayFormed[n]); + const allHomeFormed = HOME_NODES.every((n) => overlayFormed[n]); + overlayReport.push(`\nVERDICT: overlay across the access point ${allHomeFormed ? "FORMED for all home nodes" : anyHomeFormed ? "FORMED for some home nodes" : "DID NOT FORM"}.`); + const overlaySummary = overlayReport.join("\n"); + console.log(overlaySummary); + + // ================================================================================================ + // PHASE B — converge the full node sets on top of the overlay. + // ================================================================================================ + const added = new Map(); async function ensureAdded(name: string): Promise { const known = added.get(name); if (known !== undefined) return known; const { manifest, broker } = loadManifest(name); - await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); + await must(CONTROL, `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); await mesh(`module add /${name}.json`); added.set(name, broker); return broker; } - const assigned: Record> = { novox: new Set(), ace: new Set() }; - const refused: Record = { novox: [], ace: [] }; + const assigned: Record> = { novox: new Set(), ace: new Set(), shanks: new Set(), g14: new Set() }; + const refused: Record = { novox: [], ace: [], shanks: [], g14: [] }; for (const { node, mods } of PLAN) { for (const { name } of mods) { try { @@ -366,20 +481,14 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 } } - // Operator-provided app credentials (novox/hq dry-run fix). BEFORE the push, hand the mesh a FAKE - // value for each of the 7 credential modules through the real operator path — `secret accept`, - // which seals the value to the node and records it as `accepted` (the mesh will not invent one). - // The push then delivers it to the sidecar's own-secret path. The `--from` file is staged into the - // mesh-control container (one file per distinct secret name). A module the node could not host is - // skipped (its secret has nowhere to go). + // Operator-provided app credentials (own-secrets), delivered as fake values through `secret accept`. const credentialDelivered = new Map(); for (const name of new Set(CREDENTIALS.map((c) => c.name))) { - await must("anchor", `printf %s ${quote(`fake-${name}-value`)} > /tmp/fake-${name} && docker cp /tmp/fake-${name} mesh-control:/fake-${name}`); + await must(CONTROL, `printf %s ${quote(`fake-${name}-value`)} > /tmp/fake-${name} && docker cp /tmp/fake-${name} mesh-control:/fake-${name}`); } for (const c of CREDENTIALS) { if (!assigned[c.node]!.has(c.module)) { credentialDelivered.set(`${c.node}/${c.module}`, false); - console.log(`CREDENTIAL SKIPPED ${c.node}/${c.module}: not assigned, nowhere to deliver`); continue; } try { @@ -390,40 +499,47 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 console.log(`CREDENTIAL ACCEPT FAILED ${c.node}/${c.module}: ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`); } } - - // ONE push per node. - const pushError: Record = { novox: "", ace: "" }; - for (const node of ["novox", "ace"]) { + // Whole-app own-secrets (mailu/de-spiegel/amqp-email-forwarder). + for (const s of OPERATOR_SECRETS) { + if (!assigned[s.node]!.has(s.module)) continue; try { - await mesh(`push ${node}`, 240_000); + const inControl = `/secret-${s.module}-${s.name}`; + await must(CONTROL, `printf %s ${quote(s.value)} > /tmp${inControl} && docker cp /tmp${inControl} mesh-control:${inControl}`); + await mesh(`secret accept ${s.node} ${s.module} ${s.name} --from ${inControl}`); } catch (err) { - pushError[node] = (err as Error).message; - console.log(`PUSH REJECTED (${node}):\n${pushError[node]}`); + console.log(`OPERATOR SECRET FAILED ${s.node}/${s.module}/${s.name}: ${(err as Error).message.split("\n").slice(0, 2).join(" | ")}`); } } - // Wait for both nodes' CORE containers to come up (they pull concurrently from the one registry). - const psMaps: Record> = { novox: new Map(), ace: new Map() }; + // ONE push per node (workstations first — cheap — then the heavy service nodes). + const pushError: Record = {}; + for (const node of ["shanks", "g14", "novox", "ace"]) { + try { + await mesh(`push ${node}`, 300_000); + } catch (err) { + pushError[node] = (err as Error).message; + console.log(`PUSH REJECTED (${node}):\n${pushError[node]!.split("\n").slice(0, 6).join("\n")}`); + } + } + + // Wait for each node's CORE containers to come up (all nodes pull concurrently from the one registry). + const psMaps: Record> = { novox: new Map(), ace: new Map(), shanks: new Map(), g14: new Map() }; for (const { node, mods, core } of PLAN) { if (pushError[node]) continue; const coreContainers = mods.filter((m) => core.has(m.name) && assigned[node]!.has(m.name)).flatMap((m) => m.containers); - const until = Date.now() + 2_700_000; + const until = Date.now() + 3_000_000; while (Date.now() < until) { psMaps[node] = await psMapOf(node); if (coreContainers.every((c) => (psMaps[node]!.get(c) ?? "").startsWith("Up"))) break; await new Promise((r) => setTimeout(r, 10000)); } } - await new Promise((r) => setTimeout(r, 20000)); // let first-boot bounces settle + await new Promise((r) => setTimeout(r, 20000)); // ================================================================================================ - // Per-node report + gating. GREEN = each node's push accepted, every CORE module converged whole, - // no NON-GAP resource failed to apply, fail2ban is hostable (assigned, not refused), and every - // credential sidecar advanced past its "no credential" crash. Tolerated: the credential sidecars' - // app-auth failures (bogus fake value), photos/mailu, firewall's oneshot nftables.service, and - // fail2ban's package (the offline lab cannot fetch it — a documented host gap). + // Per-node convergence report. // ================================================================================================ - const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; + const users = (await on("novox", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; const allProblems: string[] = []; const report: string[] = ["================ FULL MESH CONVERGENCE ================"]; @@ -435,7 +551,7 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 const failedResources = st.wrong?.failed ?? []; report.push(`\n---- node ${node}: reached=${st.reached} applied=${st.applied} current=${st.current} waiting=${st.waiting} ----`); - if (pushError[node]) report.push(` PUSH REJECTED: ${pushError[node].split("\n").slice(0, 6).join("\n ")}`); + if (pushError[node]) report.push(` PUSH REJECTED: ${pushError[node]!.split("\n").slice(0, 6).join("\n ")}`); if (st.wrong) { report.push(` NODE WRONG: outcome=${st.wrong.outcome}`); for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`); @@ -445,19 +561,19 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 const coreFailures: string[] = []; for (const mod of mods) { if (!assigned[node]!.has(mod.name)) continue; + if (mod.node) { + report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(20)} ${core.has(mod.name) ? "CORE" : "gap "} node-service`); + continue; + } const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`); const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce); const tag = core.has(mod.name) ? (ok ? "OK " : "FAIL") : (ok ? "ok " : "GAP "); - report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(15)} ${tag} ${states.join(" ")}`); + report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(20)} ${tag} ${states.join(" ")}`); if (core.has(mod.name) && !ok) coreFailures.push(mod.name); } const issuedHere = mods.filter((m) => new RegExp(`${node}-${m.name}\\b`).test(users)).length; report.push(` broker accounts: ${issuedHere} present for ${node}`); - // Gate: push accepted, all CORE up, no NON-GAP resource failed. A failed resource names its - // owning module inside the error (`applying "firewall.load": …`), not in `id` (which is the outer - // "apply" key), so the owner is extracted from either — and a failure owned by a KNOWN_GAP module - // (firewall's oneshot nftables.service) is tolerated. const gapOwnerOf = (f: { id: string; error: string }): string => { const m = f.error.match(/applying "([^".]+)\./); return m?.[1] ?? (f.id.split(".")[0] ?? ""); @@ -468,70 +584,10 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`); } - // ================================================================================================ - // The dry-run fixes, proved by name. - // ================================================================================================ - - // fail2ban is now HOSTABLE (capability "firewall"): what the dry-run fix buys is that a node can - // host it at all. Before, it declared the never-detected "intrusion-prevention" capability, so NO - // node could host it AND its un-hostable assignment refused the whole node's push. So the GATE is - // hostability: it must be ASSIGNED and NOT refused. - // - // Its systemd service reaching active is a SEPARATE, host-level concern this offline lab cannot - // satisfy: the VM base image ships `nftables` (so firewall's package resolves and the `firewall` - // detector is advertised — which is exactly why fail2ban is now hostable) but NOT `fail2ban`, and - // the lab segment (RFC 5737 192.0.2.0/24) has no route to the package mirror, so pacman times out - // fetching fail2ban and its deps. That is a documented LAB gap (fail2ban ∈ GAPS_NOVOX, so its - // failed `fail2ban.package` resource is tolerated like firewall's oneshot nftables.service) — it is - // reported, not gated. On an online node the package installs and the service runs. - { - const refusedF2B = refused["novox"]!.find((r) => r.name === "fail2ban"); - const assignedF2B = assigned["novox"]!.has("fail2ban"); - const active = (await on("novox", `systemctl is-active fail2ban 2>&1`)).out.trim(); - const pkg = (await on("novox", `pacman -Q fail2ban 2>&1`)).out.trim(); - report.push(`\n---- fail2ban (novox): HOSTABLE assigned=${assignedF2B} refused=${refusedF2B ? "YES" : "no"} | service=${active} package="${pkg}" ----`); - if (refusedF2B) { - allProblems.push(`fail2ban still not hostable on novox: ${refusedF2B.why}`); - } else if (!assignedF2B) { - allProblems.push(`fail2ban was not assigned to novox`); - } - if (active !== "active") { - report.push(` service not active — offline lab could not install the package (documented gap, not gated); detail:`); - report.push(` ${(await on("novox", `systemctl status fail2ban --no-pager 2>&1 | head -8`)).out}`); - } - } - - // The 7 credential sidecars: each got its fake own-secret, so each must have advanced PAST the old - // "no credential" crash (it read the delivered value). It may still fail at app-auth against the - // real app with a bogus value — that is expected and does NOT gate; only the crash being gone does. - report.push(`\n---- credential sidecars: past the "no credential" crash? (fake secret delivered) ----`); - for (const c of CREDENTIALS) { - const container = `mesh-${c.module}`; - const psMap = psMaps[c.node]!; - const status = (psMap.get(container) ?? "MISSING").split(" ")[0] ?? "MISSING"; - const delivered = credentialDelivered.get(`${c.node}/${c.module}`) ?? false; - const logs = (await on(c.node, `docker logs ${container} 2>&1 | tail -60`)).out; - const stillCrashes = logs.includes(c.crash); - const appAuth = logs.split("\n").reverse().find((l) => /fail|reject|error|401|403|refused/i.test(l) && !l.includes(c.crash))?.trim().slice(0, 90) ?? ""; - report.push(` ${c.node}/${c.module.padEnd(15)} secret=${delivered ? "delivered" : "SKIPPED"} sidecar=${status.padEnd(10)} crash("${c.crash}")=${stillCrashes ? "STILL PRESENT" : "gone"}${appAuth ? ` last:"${appAuth}"` : ""}`); - if (delivered && stillCrashes) { - allProblems.push(`${c.node}/${c.module}: credential wiring did not take — sidecar still crashes "${c.crash}"`); - } - if (!delivered && assigned[c.node]!.has(c.module)) { - allProblems.push(`${c.node}/${c.module}: fake credential was not delivered (secret accept failed)`); - } - } - const summary = report.join("\n"); console.log(summary); - // Cross-node identity proof: each node's own scoped broker accounts exist and are distinct — the - // two node-plans share one broker without colliding (both run a `postgres`, `redis`, `mssql`). - for (const acct of ["novox-postgres", "ace-postgres", "novox-redis", "ace-redis"]) { - if (!new RegExp(acct).test(users)) allProblems.push(`missing broker account ${acct}`); - } - - // Diagnostics for any CORE failure (the gaps are expected; a CORE failure is what we must see). + // Diagnostics for any CORE container that did not come up. for (const { node, mods, core } of PLAN) { const psMap = psMaps[node]!; for (const mod of mods) { @@ -544,5 +600,18 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 } } - assert.deepEqual(allProblems, [], `the full mesh did not converge together:\n ${allProblems.join("\n ")}\n\n${summary}`); + // ================================================================================================ + // GATING. The headline gates: the cross-segment overlay must FORM for at least one home node + // (that is the thing this bed exists to prove). Convergence gates on each node's CORE and no + // non-gap resource failing. The KEEP run is about leaving a browsable instance, so its convergence + // is reported but not hard-gated; a normal run gates fully. + // ================================================================================================ + assert.ok(anyHomeFormed, + `the overlay did NOT form across the access point — no home node could reach novox over the overlay:\n${overlaySummary}`); + + if (!KEEP) { + assert.deepEqual(allProblems, [], `the full mesh did not converge:\n ${allProblems.join("\n ")}\n\n${summary}`); + } else if (allProblems.length) { + console.log(`\nCONVERGENCE PROBLEMS (reported, not gated on a KEEP run):\n ${allProblems.join("\n ")}`); + } });