From fcdcd338c027b0688edad9894f01ab0c9f0a076c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 8 Sep 2026 18:22:56 +0200 Subject: [PATCH] Add whole-mesh full three-node bed (stage 3: both server sets, one substrate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Combine the novox (17-module) and ace (24-module) sets on ONE substrate and prove both node-plans converge together. anchor runs the substrate only; novox and ace each run their own self-contained set (own postgres/redis), so nothing crosses a node boundary except enrolment and the shared broker/store. The four modules both nodes run (postgres, redis, mssql, portainer) are added once and assigned to each node, each getting its own per-node broker account. An overlay is placed across all three nodes. Proven green: both nodes converge together on the one substrate. ace reaches applied+current with all 17 of its CORE up (and letta too this run); novox reaches all 13 CORE up with its only failed resource the known firewall.load oneshot gap. The two node-plans share one broker without collision — distinct novox- and ace- accounts for the modules both run. No new cross-node bug (overlay/DNS/identity/port) surfaced; ports are per-VM and the sets are node-self-contained. Tolerates the same nine credential-sidecar gaps and firewall's nftables.service oneshot documented in the per-server beds. Resource envelope: 3 VMs (anchor 4GiB, novox 16GiB, ace 18GiB) + registry scenery, ~79 union images (~35GB) stocked to one registry VM and pulled concurrently by both nodes; fit within 125GiB host RAM and the 180GiB lab pool. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- scenarios/whole-mesh-full.yml | 130 +++++++ test/integration/whole-mesh-full.test.ts | 431 +++++++++++++++++++++++ 2 files changed, 561 insertions(+) create mode 100644 scenarios/whole-mesh-full.yml create mode 100644 test/integration/whole-mesh-full.test.ts diff --git a/scenarios/whole-mesh-full.yml b/scenarios/whole-mesh-full.yml new file mode 100644 index 0000000..2845078 --- /dev/null +++ b/scenarios/whole-mesh-full.yml @@ -0,0 +1,130 @@ +# The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the +# whole-mesh rehearsal (novox/hq). Combines scenarios/whole-mesh-novox.yml and whole-mesh-ace.yml. +# +# anchor — substrate ONLY (store, broker, control). +# novox — the 17-module novox set (providers + web apps + route-proxy + mailu + firewall). +# ace — the 24-module ace set (media/home stack), its /services/media library pre-created. +# +# An overlay is placed across all three so cross-node `at` resolves. Each service node is +# self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and +# the shared broker/store on anchor — which is exactly what this stage proves converges for two +# independent node-plans at once on one substrate. +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock +# The images are the UNION of the two per-server scenarios; every one is already built/pulled by the +# per-server bed prerequisites (scripts/build-module-runtime.sh, build-route-proxy-image.sh, the +# mailu/keycloak and media :mesh digest pulls). +scenario: whole-mesh-full + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + memory: 4GiB + cpus: 4 + disk: 20GiB + novox: + at: { segment: hosting, address: [192.0.2.20] } + inbound: allow + memory: 16GiB + cpus: 6 + disk: 100GiB + ace: + at: { segment: hosting, address: [192.0.2.30] } + inbound: allow + memory: 18GiB + cpus: 6 + disk: 120GiB + +images: + # --- substrate + shared --- + - postgres:17-alpine + - cloudamqp/lavinmq:latest + - mesh-control:development + - redis:7-alpine + - portainer/portainer-ce:latest + - mcr.microsoft.com/mssql/server:2022-latest + # --- novox server images --- + - minio/minio:latest + - mongo:7 + - quay.io/keycloak/keycloak:mesh + - gitea/gitea:1.22 + - nextcloud:stable + - ghcr.io/umami-software/umami:postgresql-latest + - alpine:latest + - verdaccio/verdaccio:6 + - registry:2 + - registry-api.novox.be/novox/invoicing-app:latest + - registry-api.novox.be/novox/invoicing-api:latest + - ghcr.io/mailu/unbound:mesh + - ghcr.io/mailu/admin:mesh + - ghcr.io/mailu/dovecot:mesh + - ghcr.io/mailu/postfix:mesh + - ghcr.io/mailu/rspamd:mesh + - ghcr.io/mailu/webmail:mesh + - ghcr.io/mailu/nginx:mesh + # --- ace server images --- + - lscr.io/linuxserver/sonarr:mesh + - lscr.io/linuxserver/radarr:mesh + - lscr.io/linuxserver/lidarr:mesh + - lscr.io/linuxserver/bazarr:mesh + - lscr.io/linuxserver/nzbget:mesh + - lscr.io/linuxserver/qbittorrent:mesh + - lscr.io/linuxserver/jackett:mesh + - lscr.io/linuxserver/ombi:mesh + - lscr.io/linuxserver/tautulli:mesh + - lscr.io/linuxserver/unifi-controller:mesh + - plexinc/pms-docker:mesh + - ghcr.io/pennydreadful/bookshelf:mesh + - ghcr.io/home-assistant/home-assistant:mesh + - eclipse-mosquitto:mesh + - influxdb:mesh + - grafana/grafana:mesh + - baserow/baserow:mesh + - letta/letta:mesh + - nodered/node-red:mesh + - searxng/searxng:mesh + - valkey/valkey:mesh + # --- per-module runtimes (union) --- + - mesh-runtime-postgres:development + - mesh-runtime-redis:development + - mesh-runtime-mssql:development + - mesh-runtime-portainer:development + - mesh-runtime-minio:development + - mesh-runtime-mongodb:development + - mesh-runtime-keycloak:development + - mesh-runtime-gitea:development + - mesh-runtime-nextcloud:development + - mesh-runtime-umami:development + - mesh-runtime-photos:development + - mesh-runtime-verdaccio:development + - mesh-runtime-mailu:development + - mesh-route-proxy:development + - mesh-runtime-sonarr:development + - mesh-runtime-radarr:development + - mesh-runtime-lidarr:development + - mesh-runtime-plex:development + - mesh-runtime-bazarr:development + - mesh-runtime-nzbget:development + - mesh-runtime-qbittorrent:development + - mesh-runtime-jackett:development + - mesh-runtime-ombi:development + - mesh-runtime-tautulli:development + - mesh-runtime-bookshelf:development + - mesh-runtime-home-assistant:development + - mesh-runtime-mosquitto:development + - mesh-runtime-influxdb:development + - mesh-runtime-grafana:development + - mesh-runtime-baserow:development + - mesh-runtime-letta:development + - mesh-runtime-nodered:development + - mesh-runtime-searxng:development + - mesh-runtime-unifi:development + +place: + all: [host, runtime] diff --git a/test/integration/whole-mesh-full.test.ts b/test/integration/whole-mesh-full.test.ts new file mode 100644 index 0000000..51b2a32 --- /dev/null +++ b/test/integration/whole-mesh-full.test.ts @@ -0,0 +1,431 @@ +/** + * The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the + * whole-mesh rehearsal (novox/hq). Combines whole-mesh-novox.test.ts and whole-mesh-ace.test.ts. + * + * anchor — substrate ONLY (store, broker, control). + * novox — the 17-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu, + * firewall. fail2ban is dropped (no `intrusion-prevention` detector — see that bed). + * ace — the 24-module ace set (whole-mesh-ace): the media/home stack; its /services/media + * library is pre-created so the ADR-0051 `accesses` resolve. + * + * An overlay is placed across all three so cross-node `at` resolves. Each service node is + * self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and + * the shared broker/store on anchor. The four modules both nodes run (postgres, redis, mssql, + * portainer) are ADDED once and assigned to each node; each gets its own per-node broker account. + * + * This bed tolerates the SAME known gaps the per-server beds proved and escalated (nine + * credential-sidecar crash-loops and firewall's oneshot nftables.service); it gates green on each + * node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two node-plans + * converge together on one substrate. + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll } from "./harness.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" + : false; + +const SCENARIO = "whole-mesh-full"; + +const catalogDir = process.env["MESH_LAB_CATALOG"] + ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") + ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + +const MEDIA_DIRS = [ + "/services/media/series", "/services/media/anime", "/services/media/movies", + "/services/media/music", "/services/media/audiobooks", "/services/media/downloads", + "/services/media/books", +]; + +type Mod = { name: string; containers: string[]; node?: boolean; runOnce?: string[] }; + +/** The novox node's 17-module set (fail2ban dropped). CORE gates; the rest are documented gaps. */ +const NOVOX: Mod[] = [ + { name: "postgres", containers: ["postgres", "mesh-postgres"] }, + { name: "redis", containers: ["redis", "mesh-redis"] }, + { name: "minio", containers: ["minio", "mesh-minio"] }, + { name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, + { name: "mssql", containers: ["mssql", "mesh-mssql"] }, + { name: "keycloak", containers: ["keycloak", "mesh-keycloak"] }, + { name: "gitea", containers: ["gitea", "mesh-gitea"] }, + { name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] }, + { name: "umami", containers: ["umami", "mesh-umami"] }, + { name: "photos", containers: ["photos", "mesh-photos"] }, + { name: "invoicing", containers: ["invoicing-app", "invoicing-api"] }, + { name: "portainer", containers: ["portainer", "mesh-portainer"] }, + { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] }, + { name: "registry", containers: ["mesh-registry"] }, + { name: "route-proxy", containers: ["route-proxy"] }, + { + name: "mailu", + containers: [ + "mailu-resolver", "mailu-redis", "mailu-admindb", "mailu-admin", "mailu-imap", + "mailu-smtp", "mailu-antispam", "mailu-webmail", "mailu-front", "mesh-mailu", + ], + }, + { name: "firewall", containers: [], node: true }, +]; +const CORE_NOVOX = new Set([ + "postgres", "redis", "minio", "mongodb", "mssql", + "keycloak", "gitea", "nextcloud", "invoicing", + "portainer", "verdaccio", "registry", "route-proxy", +]); +const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall"]); + +/** The ace node's 24-module set. */ +const ACE: Mod[] = [ + { name: "postgres", containers: ["postgres", "mesh-postgres"] }, + { name: "redis", containers: ["redis", "mesh-redis"] }, + { name: "mssql", containers: ["mssql", "mesh-mssql"] }, + { name: "sonarr", containers: ["sonarr", "mesh-sonarr"] }, + { name: "radarr", containers: ["radarr", "mesh-radarr"] }, + { name: "lidarr", containers: ["lidarr", "mesh-lidarr"] }, + { name: "plex", containers: ["plex", "mesh-plex"] }, + { name: "bazarr", containers: ["bazarr", "mesh-bazarr"] }, + { name: "nzbget", containers: ["nzbget", "mesh-nzbget"] }, + { name: "qbittorrent", containers: ["qbittorrent", "mesh-qbittorrent"] }, + { name: "jackett", containers: ["jackett", "mesh-jackett"] }, + { name: "ombi", containers: ["ombi", "mesh-ombi"] }, + { name: "tautulli", containers: ["tautulli", "mesh-tautulli"] }, + { name: "bookshelf", containers: ["bookshelf", "mesh-bookshelf"] }, + { name: "home-assistant", containers: ["home-assistant", "mesh-home-assistant"] }, + { name: "mosquitto", containers: ["mosquitto", "mesh-mosquitto"], runOnce: ["mosquitto-bootstrap"] }, + { name: "influxdb", containers: ["influxdb", "mesh-influxdb"] }, + { name: "grafana", containers: ["grafana", "mesh-grafana"] }, + { name: "baserow", containers: ["baserow", "mesh-baserow"] }, + { name: "letta", containers: ["letta", "mesh-letta"] }, + { name: "nodered", containers: ["nodered", "mesh-nodered"] }, + { name: "searxng", containers: ["valkey", "searxng", "mesh-searxng"] }, + { name: "unifi", containers: ["unifi-controller", "mesh-unifi"] }, + { name: "portainer", containers: ["portainer", "mesh-portainer"] }, +]; +const CORE_ACE = new Set([ + "postgres", "redis", "mssql", + "sonarr", "radarr", "lidarr", "jackett", "tautulli", "bookshelf", + "mosquitto", "influxdb", "grafana", "baserow", "nodered", "searxng", "unifi", "portainer", +]); +const GAPS_ACE = new Set(["plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta"]); + +const PLAN: { node: string; mods: Mod[]; core: Set; gaps: Set }[] = [ + { node: "novox", mods: NOVOX, core: CORE_NOVOX, gaps: GAPS_NOVOX }, + { node: "ace", mods: ACE, core: CORE_ACE, gaps: GAPS_ACE }, +]; + +/** + * Host-port remaps (per module — host ports are per-VM, so novox's and ace's never clash). Union of + * both per-server beds' remaps. + */ +const REMAP: Record> = { + nextcloud: { "80": "8090:80" }, + umami: { "3000": "3090:3000" }, + invoicing: { "80": "8091:80", "9000": "9091:9000" }, + qbittorrent: { "8080": "8090:8080" }, + searxng: { "8080": "8092:8080" }, + nzbget: { "6789": "6790:6789" }, +}; + +let instanceId = ""; +let stocked: string[] = []; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} + +async function mesh(command: string, timeoutMs?: number): Promise { + return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); +} + +function repositoryFor(reference: string): string { + const withoutDigest = reference.split("@")[0] ?? reference; + const lastColon = withoutDigest.lastIndexOf(":"); + const lastSlash = withoutDigest.lastIndexOf("/"); + return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; +} + +function pinned(repository: string): string { + const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); + assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`); + return found; +} + +function bundleFor(images: string[]): string { + let text = readFileSync(bundle, "utf8"); + for (const ref of images) { + const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); + const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); + text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); + } + return text; +} + +function loadManifest(name: string): { manifest: string; broker: boolean } { + const path = resolve(catalogDir, name, "module.json"); + const m = JSON.parse(readFileSync(path, "utf8")) as { + resources?: { type: string; image?: string; ports?: string[] }[]; + }; + const remap = REMAP[name] ?? {}; + for (const r of m.resources ?? []) { + if (r.type !== "container") continue; + if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image)); + if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); + } + const manifest = JSON.stringify(m); + return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +interface NodeState { + reached: boolean; + applied: boolean; + current: boolean; + waiting: boolean; + wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined; + raw: string; +} + +async function nodeState(node: string): Promise { + const asked = await on("anchor", `docker exec mesh-control /mesh-control status --json`); + if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + let state: { + wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + try { + state = JSON.parse(asked.out); + } catch { + return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + } + const word = state.reported.find((r) => r.node === node); + const bad = state.wrong.find((w) => w.node === node); + return { + reached: true, + applied: word?.outcome === "applied", + current: !!word?.current, + waiting: state.waiting.some((w) => w.node === node), + wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined, + raw: asked.out, + }; +} + +async function psMapOf(node: string): Promise> { + const out = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + const map = new Map(); + for (const line of out.split("\n")) { + const [n, ...rest] = line.split("\t"); + if (n) map.set(n.trim(), rest.join("\t").trim()); + } + return map; +} + +before(async () => { + if (skip) return; + assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + stocked = raised.images; + + await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); + await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); + const up = await must("anchor", `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { + assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); + } + + for (const machine of ["anchor", "novox", "ace"]) { + await mesh(`node add ${machine}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, new RegExp(`enrolled as ${machine}`), said); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + } + + // The operator provides ace's media library (ADR 0051 accesses confirm the paths, create nothing). + await must("ace", `mkdir -p ${MEDIA_DIRS.join(" ")}`); +}, { timeout: 3_000_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("both server sets converge together on one substrate", { skip, timeout: 3_600_000 }, async () => { + // Overlay across all three, so every node's private address exists and cross-node `at` resolves. + await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); + await mesh("overlay place novox --site lab"); + await mesh("overlay place ace --site lab"); + await mesh("assign anchor networking"); + await mesh("assign novox networking"); + await mesh("assign ace networking"); + + // Add every unique module ONCE (the four shared modules are added once, assigned to each node), then + // issue a per-node broker account and assign, resiliently. + const added = new Map(); // name -> needs broker + async function ensureAdded(name: string): Promise { + const known = added.get(name); + if (known !== undefined) return known; + const { manifest, broker } = loadManifest(name); + await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); + await mesh(`module add /${name}.json`); + added.set(name, broker); + return broker; + } + + const assigned: Record> = { novox: new Set(), ace: new Set() }; + const refused: Record = { novox: [], ace: [] }; + for (const { node, mods } of PLAN) { + for (const { name } of mods) { + try { + const broker = await ensureAdded(name); + if (broker) await mesh(`module issue ${name} --node ${node}`); + await mesh(`assign ${node} ${name}`); + assigned[node]!.add(name); + } catch (err) { + const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | "); + refused[node]!.push({ name, why }); + console.log(`NOT ASSIGNED ${node}/${name}: ${why}`); + } + } + } + + // ONE push per node. + const pushError: Record = { novox: "", ace: "" }; + for (const node of ["novox", "ace"]) { + try { + await mesh(`push ${node}`, 240_000); + } catch (err) { + pushError[node] = (err as Error).message; + console.log(`PUSH REJECTED (${node}):\n${pushError[node]}`); + } + } + + // Wait for both nodes' CORE containers to come up (they pull concurrently from the one registry). + const psMaps: Record> = { novox: new Map(), ace: new Map() }; + for (const { node, mods, core } of PLAN) { + if (pushError[node]) continue; + const coreContainers = mods.filter((m) => core.has(m.name) && assigned[node]!.has(m.name)).flatMap((m) => m.containers); + const until = Date.now() + 2_700_000; + while (Date.now() < until) { + psMaps[node] = await psMapOf(node); + if (coreContainers.every((c) => (psMaps[node]!.get(c) ?? "").startsWith("Up"))) break; + await new Promise((r) => setTimeout(r, 10000)); + } + } + await new Promise((r) => setTimeout(r, 20000)); // let first-boot bounces settle + + // ================================================================================================ + // Per-node report + gating. GREEN = each node's push accepted, every CORE module converged whole, + // and no NON-GAP resource failed to apply. The nine credential-sidecar gaps and firewall's oneshot + // are tolerated (documented + escalated in the per-server beds). + // ================================================================================================ + const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; + const allProblems: string[] = []; + const report: string[] = ["================ FULL MESH CONVERGENCE ================"]; + + for (const { node, mods, core, gaps } of PLAN) { + const psMap = psMaps[node] = await psMapOf(node); + const st = await nodeState(node); + const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up"); + const ranOnce = (name: string): boolean => !psMap.has(name) || /^(Up|Exited \(0\))/.test(psMap.get(name) ?? ""); + const failedResources = st.wrong?.failed ?? []; + + report.push(`\n---- node ${node}: reached=${st.reached} applied=${st.applied} current=${st.current} waiting=${st.waiting} ----`); + if (pushError[node]) report.push(` PUSH REJECTED: ${pushError[node].split("\n").slice(0, 6).join("\n ")}`); + if (st.wrong) { + report.push(` NODE WRONG: outcome=${st.wrong.outcome}`); + for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`); + } + for (const r of refused[node]!) report.push(` REFUSED ${r.name}: ${r.why}`); + + const coreFailures: string[] = []; + for (const mod of mods) { + if (!assigned[node]!.has(mod.name)) continue; + const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`); + const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce); + const tag = core.has(mod.name) ? (ok ? "OK " : "FAIL") : (ok ? "ok " : "GAP "); + report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(15)} ${tag} ${states.join(" ")}`); + if (core.has(mod.name) && !ok) coreFailures.push(mod.name); + } + const issuedHere = mods.filter((m) => new RegExp(`${node}-${m.name}\\b`).test(users)).length; + report.push(` broker accounts: ${issuedHere} present for ${node}`); + + // Gate: push accepted, all CORE up, no NON-GAP resource failed. A failed resource names its + // owning module inside the error (`applying "firewall.load": …`), not in `id` (which is the outer + // "apply" key), so the owner is extracted from either — and a failure owned by a KNOWN_GAP module + // (firewall's oneshot nftables.service) is tolerated. + const gapOwnerOf = (f: { id: string; error: string }): string => { + const m = f.error.match(/applying "([^".]+)\./); + return m?.[1] ?? (f.id.split(".")[0] ?? ""); + }; + if (pushError[node]) allProblems.push(`${node}: push rejected`); + if (coreFailures.length) allProblems.push(`${node}: CORE not converged: ${coreFailures.join(", ")}`); + const nonGapFailed = failedResources.filter((f) => !gaps.has(gapOwnerOf(f))); + if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`); + } + + const summary = report.join("\n"); + console.log(summary); + + // Cross-node identity proof: each node's own scoped broker accounts exist and are distinct — the + // two node-plans share one broker without colliding (both run a `postgres`, `redis`, `mssql`). + for (const acct of ["novox-postgres", "ace-postgres", "novox-redis", "ace-redis"]) { + if (!new RegExp(acct).test(users)) allProblems.push(`missing broker account ${acct}`); + } + + // Diagnostics for any CORE failure (the gaps are expected; a CORE failure is what we must see). + for (const { node, mods, core } of PLAN) { + const psMap = psMaps[node]!; + for (const mod of mods) { + if (!core.has(mod.name) || !assigned[node]!.has(mod.name)) continue; + for (const c of mod.containers) { + if (psMap.has(c) && !(psMap.get(c) ?? "").startsWith("Up")) { + console.log(`\n---- ${node} logs: ${c} (${psMap.get(c)}) ----\n${(await on(node, `docker logs ${c} 2>&1 | tail -25`)).out}`); + } + } + } + } + + assert.deepEqual(allProblems, [], `the full mesh did not converge together:\n ${allProblems.join("\n ")}\n\n${summary}`); +});