From 581fd6da770a6be07e730b26e4ce762ada4ea280 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 7 Sep 2026 22:54:10 +0200 Subject: [PATCH] Add whole-mesh novox dry-run bed (stage 1 of whole-mesh rehearsal) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Install the real novox server's converted service set together on one node behind the substrate — the whole-catalogue install this rebuild never ran. The bed loads each committed module.json from mesh-catalog (no hand-written manifests), rewrites image refs to the scenario registry's digests, and remaps the co-located host-port collisions (nextcloud/invoicing/route-proxy :80, minio/invoicing :9000, gitea/umami :3000). Proven green: the whole set of 17 modules RESOLVES and applies (191 resources); the CORE 13 converge whole — all five providers (postgres, redis, minio, mongodb, mssql) plus keycloak, gitea, nextcloud and invoicing reaching their providers and staying up, plus portainer, verdaccio, registry and route-proxy. Reported as escalated gaps (do not gate green): fail2ban (declares capability intrusion-prevention that no host detector provides, and an unappliable assignment blocks whole-node resolution), umami/photos/mailu (catalog manifests do not wire the runtime/app env the images need; photos' server image is an alpine placeholder), and firewall (nftables.service is a oneshot that exits, but the module declares state running so mesh-host marks it failed). Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- scenarios/whole-mesh-novox.yml | 101 +++++ test/integration/whole-mesh-novox.test.ts | 466 ++++++++++++++++++++++ 2 files changed, 567 insertions(+) create mode 100644 scenarios/whole-mesh-novox.yml create mode 100644 test/integration/whole-mesh-novox.test.ts diff --git a/scenarios/whole-mesh-novox.yml b/scenarios/whole-mesh-novox.yml new file mode 100644 index 0000000..e4cdcca --- /dev/null +++ b/scenarios/whole-mesh-novox.yml @@ -0,0 +1,101 @@ +# The whole `novox` server's converted service set, installed together on ONE node behind the mesh +# substrate — the whole-catalogue install the rebuild has never actually run. First stage of a +# whole-mesh rehearsal (novox/hq). +# +# Topology, proven by test/integration/assigned-two-node-db.test.ts: the substrate (store, broker, +# control) rides `anchor` and NOTHING else; ALL of novox's services ride the `novox` node — its own +# postgres provider owns 5432 there, so it cannot co-locate with the substrate store on 5432. Both +# machines sit on one public segment and enrol into the one mesh; an overlay is placed so a +# consumer's binding `at` resolves to novox's private address and every consumer reaches the +# providers co-located with it. +# +# The service SET (novox/hq ADR 0039/0048/0052, all converted in mesh-catalog/modules/): +# providers postgres redis minio mongodb mssql +# consumers keycloak gitea nextcloud umami photos invoicing +# apps portainer verdaccio registry route-proxy mailu +# node-level firewall fail2ban +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock +# The runtimes are built by scripts/build-module-runtime.sh (one per module that has code) and the +# route-proxy image by scripts/build-route-proxy-image.sh; every server image must be in the local +# daemon to be stocked. The test loads each committed module.json from mesh-catalog and rewrites its +# image references to what this scenario's own registry serves by digest. +scenario: whole-mesh-novox + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + # The substrate ONLY: store, broker, control. Nothing else lands here. + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + memory: 4GiB + cpus: 4 + disk: 20GiB + # The whole novox service set — ~38 containers (five providers with runtimes, six consumers with + # runtimes, portainer/verdaccio/registry/route-proxy, the nine-container Mailu stack and its + # runtime) plus two node-level modules. mssql alone wants ~2GiB; Mailu, Nextcloud and Keycloak are + # each heavy. Sized well past the two-node-db bed's second node. + novox: + at: { segment: hosting, address: [192.0.2.20] } + inbound: allow + memory: 16GiB + cpus: 8 + # ~14GiB of images are pulled from the scenario's own registry by digest, several of them large + # (mssql 1.7GiB, invoicing-api 1.9GiB, nextcloud 1.5GiB, umami/mongo ~0.9GiB), plus writable + # layers and the runtimes. A hundred gigabytes holds the whole set without exhausting the disk + # mid-apply. + disk: 100GiB + +images: + # The first-node substrate: store, broker, control. postgres:17-alpine doubles as the postgres + # provider's own service image (and Mailu's internal admin DB). + - postgres:17-alpine + - cloudamqp/lavinmq:latest + - mesh-control:development + # The module server images. Each is stocked under the repository path its module.json names, so the + # test's rewrite (pinned(repositoryFor(image))) finds it. + - redis:7-alpine + - minio/minio:latest + - mongo:7 + - mcr.microsoft.com/mssql/server:2022-latest + - quay.io/keycloak/keycloak:mesh + - gitea/gitea:1.22 + - nextcloud:stable + - ghcr.io/umami-software/umami:postgresql-latest + - alpine:latest + - portainer/portainer-ce:latest + - verdaccio/verdaccio:6 + - registry:2 + - registry-api.novox.be/novox/invoicing-app:latest + - registry-api.novox.be/novox/invoicing-api:latest + # The Mailu stack (pulled by digest, tagged :mesh so repositoryFor matches the module.json paths). + - ghcr.io/mailu/unbound:mesh + - ghcr.io/mailu/admin:mesh + - ghcr.io/mailu/dovecot:mesh + - ghcr.io/mailu/postfix:mesh + - ghcr.io/mailu/rspamd:mesh + - ghcr.io/mailu/webmail:mesh + - ghcr.io/mailu/nginx:mesh + # The per-module runtimes (built by scripts/build-module-runtime.sh). registry, route-proxy, + # invoicing, firewall and fail2ban carry no mesh-runtime image; route-proxy ships its own. + - mesh-runtime-postgres:development + - mesh-runtime-redis:development + - mesh-runtime-minio:development + - mesh-runtime-mongodb:development + - mesh-runtime-mssql:development + - mesh-runtime-keycloak:development + - mesh-runtime-gitea:development + - mesh-runtime-nextcloud:development + - mesh-runtime-umami:development + - mesh-runtime-photos:development + - mesh-runtime-portainer:development + - mesh-runtime-verdaccio:development + - mesh-runtime-mailu:development + - mesh-route-proxy:development + +place: + all: [host, runtime] diff --git a/test/integration/whole-mesh-novox.test.ts b/test/integration/whole-mesh-novox.test.ts new file mode 100644 index 0000000..3aa189f --- /dev/null +++ b/test/integration/whole-mesh-novox.test.ts @@ -0,0 +1,466 @@ +/** + * The whole `novox` server's converted service set, installed together on ONE node behind the + * substrate — the whole-catalogue install this rebuild has never actually run. First stage of a + * whole-mesh rehearsal (novox/hq). + * + * Topology (proven by assigned-two-node-db.test.ts): the substrate (store, broker, control) rides + * `anchor` and nothing else; ALL of novox's services ride the `novox` node. novox's own postgres + * provider owns 5432 there, so it cannot co-locate with the substrate store. An overlay is placed so + * each consumer's binding `at` resolves to novox's private address and reaches the providers + * co-located with it. + * + * The SET (18 modules, all converted in mesh-catalog/modules/): + * providers postgres redis minio mongodb mssql + * consumers keycloak gitea nextcloud umami photos invoicing + * apps portainer verdaccio registry route-proxy mailu + * node-level firewall fail2ban + * + * Each committed module.json is LOADED from mesh-catalog — not hand-written — and its container + * image references are rewritten to what this scenario's own registry serves by digest (the same + * pinned(repositoryFor(image)) rule the two-node-db bed applies by hand). Two things this bed + * discovered about the co-located set are handled at load time and RECORDED as findings: + * + * HOST-PORT COLLISIONS. When the whole set lands on one node with its committed host publishes, + * several servers claim the same host port: nextcloud, invoicing-app and route-proxy all want 80; + * minio and invoicing-api both want 9000; gitea and umami both want 3000. route-proxy is meant to + * FRONT the web apps on 80/443, so the web apps' own host publishes are only for direct access. + * To let the whole set converge, the colliding web/app host publishes are remapped to distinct + * host ports here (container ports unchanged); the provider ports the consumers actually connect to + * (postgres 5432, minio 9000, mongodb 27017, mssql 1433) are left as-is. See REMAP below. + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock + * scripts/build-module-runtime.sh builds one runtime per module that has code; the route-proxy image + * is built by scripts/build-route-proxy-image.sh; scenarios/whole-mesh-novox.yml stocks them all + * alongside every server image. + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll } from "./harness.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" + : false; + +const SCENARIO = "whole-mesh-novox"; +const NODE = "novox"; + +/** Where the committed module.json files live: the mesh-catalog beside mesh-control. */ +const catalogDir = process.env["MESH_LAB_CATALOG"] + ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") + ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + +/** + * The set, in dependency-reading order (the resolver accepts any order). Each row: the module, and + * the container names it should bring up on the node. Node-level modules (firewall, fail2ban) bring + * up no container — they install a package and run a service, checked separately. + */ +const MODULES: { name: string; containers: string[]; node?: boolean }[] = [ + { name: "postgres", containers: ["postgres", "mesh-postgres"] }, + { name: "redis", containers: ["redis", "mesh-redis"] }, + { name: "minio", containers: ["minio", "mesh-minio"] }, + { name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, + { name: "mssql", containers: ["mssql", "mesh-mssql"] }, + { name: "keycloak", containers: ["keycloak", "mesh-keycloak"] }, + { name: "gitea", containers: ["gitea", "mesh-gitea"] }, + { name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] }, + { name: "umami", containers: ["umami", "mesh-umami"] }, + { name: "photos", containers: ["photos", "mesh-photos"] }, + { name: "invoicing", containers: ["invoicing-app", "invoicing-api"] }, + { name: "portainer", containers: ["portainer", "mesh-portainer"] }, + { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] }, + { name: "registry", containers: ["mesh-registry"] }, + { name: "route-proxy", containers: ["route-proxy"] }, + { + name: "mailu", + containers: [ + "mailu-resolver", "mailu-redis", "mailu-admindb", "mailu-admin", "mailu-imap", + "mailu-smtp", "mailu-antispam", "mailu-webmail", "mailu-front", "mesh-mailu", + ], + }, + { name: "firewall", containers: [], node: true }, +]; + +/** + * Dropped from the converging set, with cause — recorded as a finding rather than silently omitted. + * + * fail2ban declares `capabilities: ["intrusion-prevention"]`, but mesh-host advertises no such + * capability: profile/detectors.go defines container-runtime, package-manager, service-manager, + * firewall, overlay, seat, privileged and graphical-session — nothing for intrusion-prevention. So + * NO node can ever host fail2ban. Worse, `mesh-control assign` records the assignment even while + * reporting it "cannot be applied", and the whole-node `push` then refuses to resolve the ENTIRE node + * ("nothing was sent") over that one un-hostable assignment — one bad module blocks every other. It + * is therefore left unassigned here so the rest of the set can be proven. (novox/hq — escalated.) + */ +const DROPPED: { name: string; why: string }[] = [ + { + name: "fail2ban", + why: 'declares capability "intrusion-prevention", which mesh-host has no detector for, so no node ' + + "can host it; and an unappliable assignment blocks whole-node resolution (nothing sent).", + }, +]; + +/** + * The provable CORE: modules that converge WHOLE on this node (every container up and stable) once + * the substrate resolves and applies the set. This bed gates green on the CORE — a regression in any + * of these turns it red. It is the substrate + all five providers + the four consumers that reach + * their providers and stay up + the four standalone apps. + */ +const CORE = new Set([ + "postgres", "redis", "minio", "mongodb", "mssql", + "keycloak", "gitea", "nextcloud", "invoicing", + "portainer", "verdaccio", "registry", "route-proxy", +]); + +/** + * KNOWN GAPS: modules that RESOLVE and are placed, but whose containers do not stay up because the + * committed catalog manifest is incomplete (or, for firewall, a node-service expectation is unmet). + * They are reported every run with the exact failure, and escalated (novox/hq) — but they do not gate + * green, because the gap is in the catalog/host, not in this bed or the mesh substrate. + * + * umami — the mesh-umami provisioner needs the umami server URL and admin password in its + * provisioner.env; the manifest wires neither, so it dies "UMAMI url or admin password + * is not set". The umami SERVER itself comes up. + * photos — the server image is a bare `alpine` placeholder (a real deployment runs immich at + * :2283, where the runtime's MESH_PHOTOS_URL points); alpine has no long-running command + * so it exits, and the runtime dies "no photos API key". Not genuinely converted. + * mailu — the manifest generates only secret/database/admin env; the Mailu images need their full + * configuration env (HOSTNAMES, DOMAIN, …), so every Mailu container dies rendering its + * template: "jinja2 UndefinedError: 'HOSTNAMES' is undefined" (and the resolver's + * unbound.conf is malformed). mailu-redis/admindb/admin/antispam do come up. + * firewall — resolves and applies its package and ruleset, but nftables.service does not stay + * running, so the node reports firewall.load failed. Diagnosed live in the report below. + */ +const KNOWN_GAPS = new Set(["umami", "photos", "mailu", "firewall"]); + +/** + * Host-port remaps applied at load time to break the co-located host-port collisions (see the file + * header). Keyed by module, then by the module.json port entry to replace. Container ports are + * preserved; only the host side changes. + */ +const REMAP: Record> = { + nextcloud: { "80": "8090:80" }, + umami: { "3000": "3090:3000" }, + invoicing: { "80": "8091:80", "9000": "9091:9000" }, +}; + +let instanceId = ""; +/** What the scenario's registry serves, by digest. */ +let stocked: string[] = []; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} + +/** The control plane, a container on the first node. */ +async function mesh(command: string, timeoutMs?: number): Promise { + return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); +} + +/** The repository path a reference serves under — registry.ts's repositoryFor, mirrored. */ +function repositoryFor(reference: string): string { + const withoutDigest = reference.split("@")[0] ?? reference; + const lastColon = withoutDigest.lastIndexOf(":"); + const lastSlash = withoutDigest.lastIndexOf("/"); + return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; +} + +/** The pinned reference this scenario's registry serves for a repository. */ +function pinned(repository: string): string { + const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); + assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`); + return found; +} + +/** The substrate bundle, its image references pointed at this scenario's own registry. */ +function bundleFor(images: string[]): string { + let text = readFileSync(bundle, "utf8"); + for (const ref of images) { + const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); + const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); + text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); + } + return text; +} + +/** + * Load a committed module.json, rewrite every container image to the scenario's pinned digest, and + * apply the host-port remaps. Returns the manifest as a string and whether it needs a broker account + * (a runtime that reads MESH_BROKER_FILE — providers and tooled apps do; plain/node modules do not). + */ +function loadManifest(name: string): { manifest: string; broker: boolean } { + const path = resolve(catalogDir, name, "module.json"); + const m = JSON.parse(readFileSync(path, "utf8")) as { + resources?: { type: string; image?: string; ports?: string[] }[]; + }; + const remap = REMAP[name] ?? {}; + for (const r of m.resources ?? []) { + if (r.type !== "container") continue; + if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image)); + if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); + } + const manifest = JSON.stringify(m); + return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +interface NodeState { + reached: boolean; + applied: boolean; + current: boolean; + waiting: boolean; + wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined; + raw: string; +} + +/** Ask the mesh, in its own terms, what a node has done with what it was sent. Never throws. */ +async function nodeState(node: string): Promise { + const asked = await on("anchor", `docker exec mesh-control /mesh-control status --json`); + if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + let state: { + wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + try { + state = JSON.parse(asked.out); + } catch { + return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + } + const word = state.reported.find((r) => r.node === node); + const bad = state.wrong.find((w) => w.node === node); + return { + reached: true, + applied: word?.outcome === "applied", + current: !!word?.current, + waiting: state.waiting.some((w) => w.node === node), + wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined, + raw: asked.out, + }; +} + +before(async () => { + if (skip) return; + assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + stocked = raised.images; + + // anchor raises the substrate from its bundle, digests rewritten to the scenario registry's. + await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); + await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); + const up = await must("anchor", `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { + assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); + } + + // Both machines join the one mesh and run a host so they apply what they are pushed. + for (const machine of ["anchor", NODE]) { + await mesh(`node add ${machine}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, new RegExp(`enrolled as ${machine}`), said); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + } +}, { timeout: 2_700_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("the whole novox service set resolves, installs and converges on one node in one push", { + skip, timeout: 3_300_000, +}, async () => { + for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`); + + // The overlay, so a consumer's binding `at` (the provider's private-network address) is non-empty. + // Provider and consumers are co-located on novox, but the address the mesh writes into a consumer's + // grant is the overlay address, so the overlay is placed on both nodes first (as two-node-db does). + await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); + await mesh(`overlay place ${NODE} --site lab`); + await mesh("assign anchor networking"); + await mesh(`assign ${NODE} networking`); + + // Add every module from its committed catalog manifest, issue the ones with a broker runtime, and + // assign all to novox. The resolver resolves the whole set at push time regardless of order. The + // loop is resilient: a module the node cannot host (a capability it does not advertise) is recorded + // and skipped rather than aborting the whole run, so ONE run yields the full per-module picture. + const issued: string[] = []; + const assigned = new Set(); + const refused: { name: string; why: string }[] = []; + for (const { name } of MODULES) { + try { + const { manifest, broker } = loadManifest(name); + await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); + await mesh(`module add /${name}.json`); + if (broker) { + await mesh(`module issue ${name} --node ${NODE}`); + issued.push(name); + } + await mesh(`assign ${NODE} ${name}`); + assigned.add(name); + } catch (err) { + const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | "); + refused.push({ name, why }); + console.log(`NOT ASSIGNED ${name}: ${why}`); + } + } + console.log(`issued broker accounts for: ${issued.join(", ")}`); + if (refused.length) console.log(`refused (node cannot host): ${refused.map((r) => r.name).join(", ")}`); + + // ONE push. Resolution happens here; a resolver rejection surfaces as a non-zero push. + let pushError = ""; + try { + await mesh(`push ${NODE}`, 120_000); + } catch (err) { + pushError = (err as Error).message; + console.log(`PUSH REJECTED:\n${pushError}`); + } + + // The node cannot reach applied+current while a KNOWN_GAP node-service (firewall.load) keeps + // failing, so convergence is measured directly: wait until every CORE container is up (the node + // still pulls ~14GiB first), bounded. `settle` is used only to read the node's own verdict for the + // report — the wait is on the containers. + const coreContainers = MODULES.filter((m) => CORE.has(m.name) && assigned.has(m.name)) + .flatMap((m) => m.containers); + const psNames = async (): Promise> => { + const out = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + const map = new Map(); + for (const line of out.split("\n")) { + const [n, ...rest] = line.split("\t"); + if (n) map.set(n.trim(), rest.join("\t").trim()); + } + return map; + }; + let psMap = new Map(); + if (!pushError) { + const until = Date.now() + 2_400_000; + while (Date.now() < until) { + psMap = await psNames(); + if (coreContainers.every((c) => (psMap.get(c) ?? "").startsWith("Up"))) break; + await new Promise((r) => setTimeout(r, 8000)); + } + // A moment for first-boot bounces to settle before the crash-loop check below. + await new Promise((r) => setTimeout(r, 15000)); + } + psMap = await psNames(); + const final = await nodeState(NODE); + const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up"); + + const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; + const nft = (await on(NODE, `systemctl is-active nftables 2>&1`)).out; + + // ================================================================================================ + // The per-module report — this run's deliverable. + // ================================================================================================ + const report: string[] = []; + report.push("================ WHOLE-MESH novox CONVERGENCE ================"); + report.push(`node reached=${final.reached} applied=${final.applied} current=${final.current} waiting=${final.waiting}`); + if (pushError) report.push(`PUSH REJECTED (resolver): ${pushError.split("\n").slice(0, 6).join("\n ")}`); + const failedResources = final.wrong?.failed ?? []; + if (final.wrong) { + report.push(`NODE WRONG: outcome=${final.wrong.outcome} refused=${final.wrong.refused ?? "-"}`); + for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`); + } + if (DROPPED.length) { + report.push("---- DROPPED (not assignable on any node) ----"); + for (const d of DROPPED) report.push(` ${d.name.padEnd(14)} ${d.why}`); + } + if (refused.length) { + report.push("---- REFUSED at assign ----"); + for (const r of refused) report.push(` ${r.name.padEnd(14)} ${r.why}`); + } + report.push("---- CORE (gates green) ----"); + const coreFailures: string[] = []; + const gapStatus: string[] = []; + for (const mod of MODULES) { + if (!assigned.has(mod.name)) continue; + const line = mod.node + ? `${mod.name.padEnd(14)} node-service nftables=${nft.trim()}` + : (() => { + const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`); + const allUp = mod.containers.every(running); + return `${mod.name.padEnd(14)} ${allUp ? "OK " : "GAP "} ${states.join(" ")}`; + })(); + if (CORE.has(mod.name)) { + const ok = !mod.node && mod.containers.every(running); + report.push(` ${line}`); + if (!ok) coreFailures.push(mod.name); + } else { + gapStatus.push(` ${line}`); + } + } + report.push("---- KNOWN GAPS (reported, escalated, do NOT gate green) ----"); + for (const l of gapStatus) report.push(l); + report.push("---- broker accounts (issued modules) ----"); + for (const name of issued) { + const present = new RegExp(`${NODE}-${name}\\b`).test(users); + report.push(` ${name.padEnd(14)} account ${present ? "present" : "MISSING"}`); + } + const summary = report.join("\n"); + console.log(summary); + + // Live diagnostics for the KNOWN_GAP failures, so the report carries the exact cause each run. + console.log(`\n---- firewall diagnostics ----\n${(await on(NODE, `systemctl status nftables --no-pager 2>&1 | head -12; echo '--- nftables.conf ---'; sed -n '1,20p' /etc/nftables.conf 2>&1; echo '--- journal ---'; journalctl -u nftables --no-pager -n 15 2>&1`)).out}`); + for (const mod of MODULES.filter((m) => KNOWN_GAPS.has(m.name) && !m.node && assigned.has(m.name))) { + for (const c of mod.containers) { + if (psMap.has(c) && !running(c)) { + console.log(`\n---- logs: ${c} (${psMap.get(c)}) ----\n${(await on(NODE, `docker logs ${c} 2>&1 | tail -20`)).out}`); + } + } + } + + // ================================================================================================ + // GREEN = the whole set RESOLVED (push accepted, resources applied), every CORE module converged + // whole, and NO core resource failed to apply. The KNOWN_GAPS (umami, photos, mailu, firewall) and + // DROPPED (fail2ban) are reported and escalated but do not gate — the gap is in the catalog/host. + // ================================================================================================ + assert.equal(pushError, "", `the whole set did not resolve — push was rejected:\n${pushError}`); + const coreResourceFailures = failedResources.filter((f) => { + const mod = f.id.split(".")[0] ?? ""; + return CORE.has(mod); + }); + assert.deepEqual(coreResourceFailures, [], + `a CORE resource failed to apply:\n${coreResourceFailures.map((f) => `${f.id}: ${f.error}`).join("\n")}\n${summary}`); + assert.deepEqual(coreFailures, [], + `these CORE modules did not converge whole: ${coreFailures.join(", ")}\n${summary}`); +});