From 581fd6da770a6be07e730b26e4ce762ada4ea280 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 7 Sep 2026 22:54:10 +0200 Subject: [PATCH 1/4] Add whole-mesh novox dry-run bed (stage 1 of whole-mesh rehearsal) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Install the real novox server's converted service set together on one node behind the substrate — the whole-catalogue install this rebuild never ran. The bed loads each committed module.json from mesh-catalog (no hand-written manifests), rewrites image refs to the scenario registry's digests, and remaps the co-located host-port collisions (nextcloud/invoicing/route-proxy :80, minio/invoicing :9000, gitea/umami :3000). Proven green: the whole set of 17 modules RESOLVES and applies (191 resources); the CORE 13 converge whole — all five providers (postgres, redis, minio, mongodb, mssql) plus keycloak, gitea, nextcloud and invoicing reaching their providers and staying up, plus portainer, verdaccio, registry and route-proxy. Reported as escalated gaps (do not gate green): fail2ban (declares capability intrusion-prevention that no host detector provides, and an unappliable assignment blocks whole-node resolution), umami/photos/mailu (catalog manifests do not wire the runtime/app env the images need; photos' server image is an alpine placeholder), and firewall (nftables.service is a oneshot that exits, but the module declares state running so mesh-host marks it failed). Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- scenarios/whole-mesh-novox.yml | 101 +++++ test/integration/whole-mesh-novox.test.ts | 466 ++++++++++++++++++++++ 2 files changed, 567 insertions(+) create mode 100644 scenarios/whole-mesh-novox.yml create mode 100644 test/integration/whole-mesh-novox.test.ts diff --git a/scenarios/whole-mesh-novox.yml b/scenarios/whole-mesh-novox.yml new file mode 100644 index 0000000..e4cdcca --- /dev/null +++ b/scenarios/whole-mesh-novox.yml @@ -0,0 +1,101 @@ +# The whole `novox` server's converted service set, installed together on ONE node behind the mesh +# substrate — the whole-catalogue install the rebuild has never actually run. First stage of a +# whole-mesh rehearsal (novox/hq). +# +# Topology, proven by test/integration/assigned-two-node-db.test.ts: the substrate (store, broker, +# control) rides `anchor` and NOTHING else; ALL of novox's services ride the `novox` node — its own +# postgres provider owns 5432 there, so it cannot co-locate with the substrate store on 5432. Both +# machines sit on one public segment and enrol into the one mesh; an overlay is placed so a +# consumer's binding `at` resolves to novox's private address and every consumer reaches the +# providers co-located with it. +# +# The service SET (novox/hq ADR 0039/0048/0052, all converted in mesh-catalog/modules/): +# providers postgres redis minio mongodb mssql +# consumers keycloak gitea nextcloud umami photos invoicing +# apps portainer verdaccio registry route-proxy mailu +# node-level firewall fail2ban +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock +# The runtimes are built by scripts/build-module-runtime.sh (one per module that has code) and the +# route-proxy image by scripts/build-route-proxy-image.sh; every server image must be in the local +# daemon to be stocked. The test loads each committed module.json from mesh-catalog and rewrites its +# image references to what this scenario's own registry serves by digest. +scenario: whole-mesh-novox + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + # The substrate ONLY: store, broker, control. Nothing else lands here. + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + memory: 4GiB + cpus: 4 + disk: 20GiB + # The whole novox service set — ~38 containers (five providers with runtimes, six consumers with + # runtimes, portainer/verdaccio/registry/route-proxy, the nine-container Mailu stack and its + # runtime) plus two node-level modules. mssql alone wants ~2GiB; Mailu, Nextcloud and Keycloak are + # each heavy. Sized well past the two-node-db bed's second node. + novox: + at: { segment: hosting, address: [192.0.2.20] } + inbound: allow + memory: 16GiB + cpus: 8 + # ~14GiB of images are pulled from the scenario's own registry by digest, several of them large + # (mssql 1.7GiB, invoicing-api 1.9GiB, nextcloud 1.5GiB, umami/mongo ~0.9GiB), plus writable + # layers and the runtimes. A hundred gigabytes holds the whole set without exhausting the disk + # mid-apply. + disk: 100GiB + +images: + # The first-node substrate: store, broker, control. postgres:17-alpine doubles as the postgres + # provider's own service image (and Mailu's internal admin DB). + - postgres:17-alpine + - cloudamqp/lavinmq:latest + - mesh-control:development + # The module server images. Each is stocked under the repository path its module.json names, so the + # test's rewrite (pinned(repositoryFor(image))) finds it. + - redis:7-alpine + - minio/minio:latest + - mongo:7 + - mcr.microsoft.com/mssql/server:2022-latest + - quay.io/keycloak/keycloak:mesh + - gitea/gitea:1.22 + - nextcloud:stable + - ghcr.io/umami-software/umami:postgresql-latest + - alpine:latest + - portainer/portainer-ce:latest + - verdaccio/verdaccio:6 + - registry:2 + - registry-api.novox.be/novox/invoicing-app:latest + - registry-api.novox.be/novox/invoicing-api:latest + # The Mailu stack (pulled by digest, tagged :mesh so repositoryFor matches the module.json paths). + - ghcr.io/mailu/unbound:mesh + - ghcr.io/mailu/admin:mesh + - ghcr.io/mailu/dovecot:mesh + - ghcr.io/mailu/postfix:mesh + - ghcr.io/mailu/rspamd:mesh + - ghcr.io/mailu/webmail:mesh + - ghcr.io/mailu/nginx:mesh + # The per-module runtimes (built by scripts/build-module-runtime.sh). registry, route-proxy, + # invoicing, firewall and fail2ban carry no mesh-runtime image; route-proxy ships its own. + - mesh-runtime-postgres:development + - mesh-runtime-redis:development + - mesh-runtime-minio:development + - mesh-runtime-mongodb:development + - mesh-runtime-mssql:development + - mesh-runtime-keycloak:development + - mesh-runtime-gitea:development + - mesh-runtime-nextcloud:development + - mesh-runtime-umami:development + - mesh-runtime-photos:development + - mesh-runtime-portainer:development + - mesh-runtime-verdaccio:development + - mesh-runtime-mailu:development + - mesh-route-proxy:development + +place: + all: [host, runtime] diff --git a/test/integration/whole-mesh-novox.test.ts b/test/integration/whole-mesh-novox.test.ts new file mode 100644 index 0000000..3aa189f --- /dev/null +++ b/test/integration/whole-mesh-novox.test.ts @@ -0,0 +1,466 @@ +/** + * The whole `novox` server's converted service set, installed together on ONE node behind the + * substrate — the whole-catalogue install this rebuild has never actually run. First stage of a + * whole-mesh rehearsal (novox/hq). + * + * Topology (proven by assigned-two-node-db.test.ts): the substrate (store, broker, control) rides + * `anchor` and nothing else; ALL of novox's services ride the `novox` node. novox's own postgres + * provider owns 5432 there, so it cannot co-locate with the substrate store. An overlay is placed so + * each consumer's binding `at` resolves to novox's private address and reaches the providers + * co-located with it. + * + * The SET (18 modules, all converted in mesh-catalog/modules/): + * providers postgres redis minio mongodb mssql + * consumers keycloak gitea nextcloud umami photos invoicing + * apps portainer verdaccio registry route-proxy mailu + * node-level firewall fail2ban + * + * Each committed module.json is LOADED from mesh-catalog — not hand-written — and its container + * image references are rewritten to what this scenario's own registry serves by digest (the same + * pinned(repositoryFor(image)) rule the two-node-db bed applies by hand). Two things this bed + * discovered about the co-located set are handled at load time and RECORDED as findings: + * + * HOST-PORT COLLISIONS. When the whole set lands on one node with its committed host publishes, + * several servers claim the same host port: nextcloud, invoicing-app and route-proxy all want 80; + * minio and invoicing-api both want 9000; gitea and umami both want 3000. route-proxy is meant to + * FRONT the web apps on 80/443, so the web apps' own host publishes are only for direct access. + * To let the whole set converge, the colliding web/app host publishes are remapped to distinct + * host ports here (container ports unchanged); the provider ports the consumers actually connect to + * (postgres 5432, minio 9000, mongodb 27017, mssql 1433) are left as-is. See REMAP below. + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock + * scripts/build-module-runtime.sh builds one runtime per module that has code; the route-proxy image + * is built by scripts/build-route-proxy-image.sh; scenarios/whole-mesh-novox.yml stocks them all + * alongside every server image. + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll } from "./harness.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" + : false; + +const SCENARIO = "whole-mesh-novox"; +const NODE = "novox"; + +/** Where the committed module.json files live: the mesh-catalog beside mesh-control. */ +const catalogDir = process.env["MESH_LAB_CATALOG"] + ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") + ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + +/** + * The set, in dependency-reading order (the resolver accepts any order). Each row: the module, and + * the container names it should bring up on the node. Node-level modules (firewall, fail2ban) bring + * up no container — they install a package and run a service, checked separately. + */ +const MODULES: { name: string; containers: string[]; node?: boolean }[] = [ + { name: "postgres", containers: ["postgres", "mesh-postgres"] }, + { name: "redis", containers: ["redis", "mesh-redis"] }, + { name: "minio", containers: ["minio", "mesh-minio"] }, + { name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, + { name: "mssql", containers: ["mssql", "mesh-mssql"] }, + { name: "keycloak", containers: ["keycloak", "mesh-keycloak"] }, + { name: "gitea", containers: ["gitea", "mesh-gitea"] }, + { name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] }, + { name: "umami", containers: ["umami", "mesh-umami"] }, + { name: "photos", containers: ["photos", "mesh-photos"] }, + { name: "invoicing", containers: ["invoicing-app", "invoicing-api"] }, + { name: "portainer", containers: ["portainer", "mesh-portainer"] }, + { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] }, + { name: "registry", containers: ["mesh-registry"] }, + { name: "route-proxy", containers: ["route-proxy"] }, + { + name: "mailu", + containers: [ + "mailu-resolver", "mailu-redis", "mailu-admindb", "mailu-admin", "mailu-imap", + "mailu-smtp", "mailu-antispam", "mailu-webmail", "mailu-front", "mesh-mailu", + ], + }, + { name: "firewall", containers: [], node: true }, +]; + +/** + * Dropped from the converging set, with cause — recorded as a finding rather than silently omitted. + * + * fail2ban declares `capabilities: ["intrusion-prevention"]`, but mesh-host advertises no such + * capability: profile/detectors.go defines container-runtime, package-manager, service-manager, + * firewall, overlay, seat, privileged and graphical-session — nothing for intrusion-prevention. So + * NO node can ever host fail2ban. Worse, `mesh-control assign` records the assignment even while + * reporting it "cannot be applied", and the whole-node `push` then refuses to resolve the ENTIRE node + * ("nothing was sent") over that one un-hostable assignment — one bad module blocks every other. It + * is therefore left unassigned here so the rest of the set can be proven. (novox/hq — escalated.) + */ +const DROPPED: { name: string; why: string }[] = [ + { + name: "fail2ban", + why: 'declares capability "intrusion-prevention", which mesh-host has no detector for, so no node ' + + "can host it; and an unappliable assignment blocks whole-node resolution (nothing sent).", + }, +]; + +/** + * The provable CORE: modules that converge WHOLE on this node (every container up and stable) once + * the substrate resolves and applies the set. This bed gates green on the CORE — a regression in any + * of these turns it red. It is the substrate + all five providers + the four consumers that reach + * their providers and stay up + the four standalone apps. + */ +const CORE = new Set([ + "postgres", "redis", "minio", "mongodb", "mssql", + "keycloak", "gitea", "nextcloud", "invoicing", + "portainer", "verdaccio", "registry", "route-proxy", +]); + +/** + * KNOWN GAPS: modules that RESOLVE and are placed, but whose containers do not stay up because the + * committed catalog manifest is incomplete (or, for firewall, a node-service expectation is unmet). + * They are reported every run with the exact failure, and escalated (novox/hq) — but they do not gate + * green, because the gap is in the catalog/host, not in this bed or the mesh substrate. + * + * umami — the mesh-umami provisioner needs the umami server URL and admin password in its + * provisioner.env; the manifest wires neither, so it dies "UMAMI url or admin password + * is not set". The umami SERVER itself comes up. + * photos — the server image is a bare `alpine` placeholder (a real deployment runs immich at + * :2283, where the runtime's MESH_PHOTOS_URL points); alpine has no long-running command + * so it exits, and the runtime dies "no photos API key". Not genuinely converted. + * mailu — the manifest generates only secret/database/admin env; the Mailu images need their full + * configuration env (HOSTNAMES, DOMAIN, …), so every Mailu container dies rendering its + * template: "jinja2 UndefinedError: 'HOSTNAMES' is undefined" (and the resolver's + * unbound.conf is malformed). mailu-redis/admindb/admin/antispam do come up. + * firewall — resolves and applies its package and ruleset, but nftables.service does not stay + * running, so the node reports firewall.load failed. Diagnosed live in the report below. + */ +const KNOWN_GAPS = new Set(["umami", "photos", "mailu", "firewall"]); + +/** + * Host-port remaps applied at load time to break the co-located host-port collisions (see the file + * header). Keyed by module, then by the module.json port entry to replace. Container ports are + * preserved; only the host side changes. + */ +const REMAP: Record> = { + nextcloud: { "80": "8090:80" }, + umami: { "3000": "3090:3000" }, + invoicing: { "80": "8091:80", "9000": "9091:9000" }, +}; + +let instanceId = ""; +/** What the scenario's registry serves, by digest. */ +let stocked: string[] = []; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} + +/** The control plane, a container on the first node. */ +async function mesh(command: string, timeoutMs?: number): Promise { + return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); +} + +/** The repository path a reference serves under — registry.ts's repositoryFor, mirrored. */ +function repositoryFor(reference: string): string { + const withoutDigest = reference.split("@")[0] ?? reference; + const lastColon = withoutDigest.lastIndexOf(":"); + const lastSlash = withoutDigest.lastIndexOf("/"); + return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; +} + +/** The pinned reference this scenario's registry serves for a repository. */ +function pinned(repository: string): string { + const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); + assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`); + return found; +} + +/** The substrate bundle, its image references pointed at this scenario's own registry. */ +function bundleFor(images: string[]): string { + let text = readFileSync(bundle, "utf8"); + for (const ref of images) { + const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); + const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); + text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); + } + return text; +} + +/** + * Load a committed module.json, rewrite every container image to the scenario's pinned digest, and + * apply the host-port remaps. Returns the manifest as a string and whether it needs a broker account + * (a runtime that reads MESH_BROKER_FILE — providers and tooled apps do; plain/node modules do not). + */ +function loadManifest(name: string): { manifest: string; broker: boolean } { + const path = resolve(catalogDir, name, "module.json"); + const m = JSON.parse(readFileSync(path, "utf8")) as { + resources?: { type: string; image?: string; ports?: string[] }[]; + }; + const remap = REMAP[name] ?? {}; + for (const r of m.resources ?? []) { + if (r.type !== "container") continue; + if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image)); + if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); + } + const manifest = JSON.stringify(m); + return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +interface NodeState { + reached: boolean; + applied: boolean; + current: boolean; + waiting: boolean; + wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined; + raw: string; +} + +/** Ask the mesh, in its own terms, what a node has done with what it was sent. Never throws. */ +async function nodeState(node: string): Promise { + const asked = await on("anchor", `docker exec mesh-control /mesh-control status --json`); + if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + let state: { + wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + try { + state = JSON.parse(asked.out); + } catch { + return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + } + const word = state.reported.find((r) => r.node === node); + const bad = state.wrong.find((w) => w.node === node); + return { + reached: true, + applied: word?.outcome === "applied", + current: !!word?.current, + waiting: state.waiting.some((w) => w.node === node), + wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined, + raw: asked.out, + }; +} + +before(async () => { + if (skip) return; + assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + stocked = raised.images; + + // anchor raises the substrate from its bundle, digests rewritten to the scenario registry's. + await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); + await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); + const up = await must("anchor", `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { + assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); + } + + // Both machines join the one mesh and run a host so they apply what they are pushed. + for (const machine of ["anchor", NODE]) { + await mesh(`node add ${machine}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, new RegExp(`enrolled as ${machine}`), said); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + } +}, { timeout: 2_700_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("the whole novox service set resolves, installs and converges on one node in one push", { + skip, timeout: 3_300_000, +}, async () => { + for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`); + + // The overlay, so a consumer's binding `at` (the provider's private-network address) is non-empty. + // Provider and consumers are co-located on novox, but the address the mesh writes into a consumer's + // grant is the overlay address, so the overlay is placed on both nodes first (as two-node-db does). + await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); + await mesh(`overlay place ${NODE} --site lab`); + await mesh("assign anchor networking"); + await mesh(`assign ${NODE} networking`); + + // Add every module from its committed catalog manifest, issue the ones with a broker runtime, and + // assign all to novox. The resolver resolves the whole set at push time regardless of order. The + // loop is resilient: a module the node cannot host (a capability it does not advertise) is recorded + // and skipped rather than aborting the whole run, so ONE run yields the full per-module picture. + const issued: string[] = []; + const assigned = new Set(); + const refused: { name: string; why: string }[] = []; + for (const { name } of MODULES) { + try { + const { manifest, broker } = loadManifest(name); + await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); + await mesh(`module add /${name}.json`); + if (broker) { + await mesh(`module issue ${name} --node ${NODE}`); + issued.push(name); + } + await mesh(`assign ${NODE} ${name}`); + assigned.add(name); + } catch (err) { + const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | "); + refused.push({ name, why }); + console.log(`NOT ASSIGNED ${name}: ${why}`); + } + } + console.log(`issued broker accounts for: ${issued.join(", ")}`); + if (refused.length) console.log(`refused (node cannot host): ${refused.map((r) => r.name).join(", ")}`); + + // ONE push. Resolution happens here; a resolver rejection surfaces as a non-zero push. + let pushError = ""; + try { + await mesh(`push ${NODE}`, 120_000); + } catch (err) { + pushError = (err as Error).message; + console.log(`PUSH REJECTED:\n${pushError}`); + } + + // The node cannot reach applied+current while a KNOWN_GAP node-service (firewall.load) keeps + // failing, so convergence is measured directly: wait until every CORE container is up (the node + // still pulls ~14GiB first), bounded. `settle` is used only to read the node's own verdict for the + // report — the wait is on the containers. + const coreContainers = MODULES.filter((m) => CORE.has(m.name) && assigned.has(m.name)) + .flatMap((m) => m.containers); + const psNames = async (): Promise> => { + const out = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + const map = new Map(); + for (const line of out.split("\n")) { + const [n, ...rest] = line.split("\t"); + if (n) map.set(n.trim(), rest.join("\t").trim()); + } + return map; + }; + let psMap = new Map(); + if (!pushError) { + const until = Date.now() + 2_400_000; + while (Date.now() < until) { + psMap = await psNames(); + if (coreContainers.every((c) => (psMap.get(c) ?? "").startsWith("Up"))) break; + await new Promise((r) => setTimeout(r, 8000)); + } + // A moment for first-boot bounces to settle before the crash-loop check below. + await new Promise((r) => setTimeout(r, 15000)); + } + psMap = await psNames(); + const final = await nodeState(NODE); + const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up"); + + const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; + const nft = (await on(NODE, `systemctl is-active nftables 2>&1`)).out; + + // ================================================================================================ + // The per-module report — this run's deliverable. + // ================================================================================================ + const report: string[] = []; + report.push("================ WHOLE-MESH novox CONVERGENCE ================"); + report.push(`node reached=${final.reached} applied=${final.applied} current=${final.current} waiting=${final.waiting}`); + if (pushError) report.push(`PUSH REJECTED (resolver): ${pushError.split("\n").slice(0, 6).join("\n ")}`); + const failedResources = final.wrong?.failed ?? []; + if (final.wrong) { + report.push(`NODE WRONG: outcome=${final.wrong.outcome} refused=${final.wrong.refused ?? "-"}`); + for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`); + } + if (DROPPED.length) { + report.push("---- DROPPED (not assignable on any node) ----"); + for (const d of DROPPED) report.push(` ${d.name.padEnd(14)} ${d.why}`); + } + if (refused.length) { + report.push("---- REFUSED at assign ----"); + for (const r of refused) report.push(` ${r.name.padEnd(14)} ${r.why}`); + } + report.push("---- CORE (gates green) ----"); + const coreFailures: string[] = []; + const gapStatus: string[] = []; + for (const mod of MODULES) { + if (!assigned.has(mod.name)) continue; + const line = mod.node + ? `${mod.name.padEnd(14)} node-service nftables=${nft.trim()}` + : (() => { + const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`); + const allUp = mod.containers.every(running); + return `${mod.name.padEnd(14)} ${allUp ? "OK " : "GAP "} ${states.join(" ")}`; + })(); + if (CORE.has(mod.name)) { + const ok = !mod.node && mod.containers.every(running); + report.push(` ${line}`); + if (!ok) coreFailures.push(mod.name); + } else { + gapStatus.push(` ${line}`); + } + } + report.push("---- KNOWN GAPS (reported, escalated, do NOT gate green) ----"); + for (const l of gapStatus) report.push(l); + report.push("---- broker accounts (issued modules) ----"); + for (const name of issued) { + const present = new RegExp(`${NODE}-${name}\\b`).test(users); + report.push(` ${name.padEnd(14)} account ${present ? "present" : "MISSING"}`); + } + const summary = report.join("\n"); + console.log(summary); + + // Live diagnostics for the KNOWN_GAP failures, so the report carries the exact cause each run. + console.log(`\n---- firewall diagnostics ----\n${(await on(NODE, `systemctl status nftables --no-pager 2>&1 | head -12; echo '--- nftables.conf ---'; sed -n '1,20p' /etc/nftables.conf 2>&1; echo '--- journal ---'; journalctl -u nftables --no-pager -n 15 2>&1`)).out}`); + for (const mod of MODULES.filter((m) => KNOWN_GAPS.has(m.name) && !m.node && assigned.has(m.name))) { + for (const c of mod.containers) { + if (psMap.has(c) && !running(c)) { + console.log(`\n---- logs: ${c} (${psMap.get(c)}) ----\n${(await on(NODE, `docker logs ${c} 2>&1 | tail -20`)).out}`); + } + } + } + + // ================================================================================================ + // GREEN = the whole set RESOLVED (push accepted, resources applied), every CORE module converged + // whole, and NO core resource failed to apply. The KNOWN_GAPS (umami, photos, mailu, firewall) and + // DROPPED (fail2ban) are reported and escalated but do not gate — the gap is in the catalog/host. + // ================================================================================================ + assert.equal(pushError, "", `the whole set did not resolve — push was rejected:\n${pushError}`); + const coreResourceFailures = failedResources.filter((f) => { + const mod = f.id.split(".")[0] ?? ""; + return CORE.has(mod); + }); + assert.deepEqual(coreResourceFailures, [], + `a CORE resource failed to apply:\n${coreResourceFailures.map((f) => `${f.id}: ${f.error}`).join("\n")}\n${summary}`); + assert.deepEqual(coreFailures, [], + `these CORE modules did not converge whole: ${coreFailures.join(", ")}\n${summary}`); +}); -- 2.54.0 From 90651e1e73161733b5e6630e38fd7b4d7ba1989b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 8 Sep 2026 00:19:52 +0200 Subject: [PATCH 2/4] Add whole-mesh ace dry-run bed (stage 2 of whole-mesh rehearsal) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Install the real ace server's converted service set (24 modules) together on one node behind the substrate — sibling of the whole-mesh-novox bed, the media/home-automation half. Loads each committed module.json from mesh-catalog, rewrites image refs to the scenario registry's digests, remaps the co-located host-port collisions (qbittorrent/searxng/unifi :8080, nzbget/unifi :6789), and pre-creates the ADR-0051 operator-owned media library dirs under /services/media so the media stack's `accesses` resolve. Proven green: the whole 24-module set RESOLVES and applies (214 resources, node applied+current) — the ADR-0051 shared-dir `accesses` mechanism works cleanly across eight co-accessing media modules. The CORE 17 converge whole: postgres/redis/mssql, sonarr/radarr/lidarr/jackett/tautulli/bookshelf, mosquitto/influxdb/grafana/baserow/nodered/searxng/unifi/portainer. Reported as escalated gaps (do not gate green): six tool-runtime sidecars crash-loop because the committed manifest does not wire the app credential they need (plex MESH_PLEX_TOKEN, bazarr MESH_BAZARR_API_KEY, nzbget MESH_NZBGET_URL/PASSWORD, qbittorrent MESH_QBITTORRENT_URL/PASSWORD, ombi MESH_OMBI_API_KEY, home-assistant MESH_HOMEASSISTANT_TOKEN) — the umami/photos class from novox; each server is up, only the sidecar is down. sonarr/radarr/ lidarr/jackett/tautulli self-configure from the app's config file and their runtimes come up. letta's app has a first-boot postgres migration race (pgvector the deeper blocker, per two-node-db). Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- scenarios/whole-mesh-ace.yml | 99 ++++++ test/integration/whole-mesh-ace.test.ts | 436 ++++++++++++++++++++++++ 2 files changed, 535 insertions(+) create mode 100644 scenarios/whole-mesh-ace.yml create mode 100644 test/integration/whole-mesh-ace.test.ts diff --git a/scenarios/whole-mesh-ace.yml b/scenarios/whole-mesh-ace.yml new file mode 100644 index 0000000..b8465da --- /dev/null +++ b/scenarios/whole-mesh-ace.yml @@ -0,0 +1,99 @@ +# The whole `ace` server's converted service set, installed together on ONE node behind the mesh +# substrate — the media/home-automation half of the whole-mesh rehearsal (novox/hq). Sibling of +# scenarios/whole-mesh-novox.yml; same topology, a different (larger, media-heavy) module set. +# +# Substrate (store, broker, control) rides `anchor` and NOTHING else; ALL of ace's services ride the +# `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis +# providers co-located with them. The media stack (sonarr/radarr/lidarr/plex/bazarr/nzbget/ +# qbittorrent/bookshelf) shares the operator-owned library directories under /services/media (ADR +# 0051 `accesses`); the test pre-creates them on the node, as the operator would, before the push — +# the mesh confirms the paths exist and mounts them, but creates and chowns none of it. +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock +# The runtimes are built by scripts/build-module-runtime.sh (one per module); every server image must +# be in the local daemon to be stocked. The media/app images are pulled by their pinned digests and +# tagged :mesh so repositoryFor matches the module.json paths (postgres/redis/portainer/mssql reuse +# their existing local tags). The test loads each committed module.json from mesh-catalog and rewrites +# its image references to what this scenario's own registry serves by digest. +scenario: whole-mesh-ace + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + memory: 4GiB + cpus: 4 + disk: 20GiB + # The whole ace service set — 24 modules, ~50 containers, several heavy (Plex, Home Assistant, + # Letta ~1.8GiB, Baserow ~1.5GiB, the UniFi controller's JVM, mssql ~2GiB). Sized past novox. + ace: + at: { segment: hosting, address: [192.0.2.20] } + inbound: allow + memory: 18GiB + cpus: 8 + disk: 120GiB + +images: + # The first-node substrate. + - postgres:17-alpine + - cloudamqp/lavinmq:latest + - mesh-control:development + # The module server images. Reused local tags where the exact version does not matter for a boot + # (postgres/redis/portainer/mssql); pinned-digest :mesh tags for the media/app images. + - redis:7-alpine + - lscr.io/linuxserver/sonarr:mesh + - lscr.io/linuxserver/radarr:mesh + - lscr.io/linuxserver/lidarr:mesh + - lscr.io/linuxserver/bazarr:mesh + - lscr.io/linuxserver/nzbget:mesh + - lscr.io/linuxserver/qbittorrent:mesh + - lscr.io/linuxserver/jackett:mesh + - lscr.io/linuxserver/ombi:mesh + - lscr.io/linuxserver/tautulli:mesh + - lscr.io/linuxserver/unifi-controller:mesh + - plexinc/pms-docker:mesh + - ghcr.io/pennydreadful/bookshelf:mesh + - ghcr.io/home-assistant/home-assistant:mesh + - eclipse-mosquitto:mesh + - influxdb:mesh + - grafana/grafana:mesh + - baserow/baserow:mesh + - letta/letta:mesh + - nodered/node-red:mesh + - searxng/searxng:mesh + - valkey/valkey:mesh + - portainer/portainer-ce:latest + - mcr.microsoft.com/mssql/server:2022-latest + # The per-module runtimes (built by scripts/build-module-runtime.sh). + - mesh-runtime-postgres:development + - mesh-runtime-redis:development + - mesh-runtime-sonarr:development + - mesh-runtime-radarr:development + - mesh-runtime-lidarr:development + - mesh-runtime-plex:development + - mesh-runtime-bazarr:development + - mesh-runtime-nzbget:development + - mesh-runtime-qbittorrent:development + - mesh-runtime-jackett:development + - mesh-runtime-ombi:development + - mesh-runtime-tautulli:development + - mesh-runtime-bookshelf:development + - mesh-runtime-home-assistant:development + - mesh-runtime-mosquitto:development + - mesh-runtime-influxdb:development + - mesh-runtime-grafana:development + - mesh-runtime-baserow:development + - mesh-runtime-letta:development + - mesh-runtime-nodered:development + - mesh-runtime-searxng:development + - mesh-runtime-unifi:development + - mesh-runtime-portainer:development + - mesh-runtime-mssql:development + +place: + all: [host, runtime] diff --git a/test/integration/whole-mesh-ace.test.ts b/test/integration/whole-mesh-ace.test.ts new file mode 100644 index 0000000..77393d1 --- /dev/null +++ b/test/integration/whole-mesh-ace.test.ts @@ -0,0 +1,436 @@ +/** + * The whole `ace` server's converted service set, installed together on ONE node behind the + * substrate — the media / home-automation half of the whole-mesh rehearsal (novox/hq). Sibling of + * whole-mesh-novox.test.ts; same harness and topology, a larger, media-heavy module set. + * + * Substrate (store, broker, control) rides `anchor` and nothing else; ALL of ace's services ride the + * `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis + * providers co-located with them. The media stack shares the operator-owned library directories + * under /services/media (ADR 0051 `accesses`): the mesh writes an `access` resource that CONFIRMS + * each path exists and mounts it, but creates and chowns none of it — so before() pre-creates those + * directories on the node, exactly as the operator would. + * + * The SET (24 modules, all converted in mesh-catalog/modules/): + * providers postgres redis mssql consumers baserow letta + * media sonarr radarr lidarr plex bazarr nzbget qbittorrent jackett ombi tautulli bookshelf + * home/data home-assistant mosquitto influxdb grafana nodered searxng + * apps unifi portainer + * + * Each committed module.json is LOADED from mesh-catalog (not hand-written); its container image + * references are rewritten to what this scenario's own registry serves by digest, and the co-located + * host-port collisions are remapped at load time (see REMAP). + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll } from "./harness.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" + : false; + +const SCENARIO = "whole-mesh-ace"; +const NODE = "ace"; + +const catalogDir = process.env["MESH_LAB_CATALOG"] + ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") + ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + +/** The operator-owned media library the ADR-0051 `accesses` point at — pre-created before the push. */ +const MEDIA_DIRS = [ + "/services/media/series", "/services/media/anime", "/services/media/movies", + "/services/media/music", "/services/media/audiobooks", "/services/media/downloads", + "/services/media/books", +]; + +/** + * The set. Each row names the module and the containers it should bring up. `runOnce` names + * containers that seed state and exit (mosquitto's dynsec bootstrap) — they must have run, not stay + * up. + */ +const MODULES: { name: string; containers: string[]; runOnce?: string[] }[] = [ + { name: "postgres", containers: ["postgres", "mesh-postgres"] }, + { name: "redis", containers: ["redis", "mesh-redis"] }, + { name: "mssql", containers: ["mssql", "mesh-mssql"] }, + { name: "sonarr", containers: ["sonarr", "mesh-sonarr"] }, + { name: "radarr", containers: ["radarr", "mesh-radarr"] }, + { name: "lidarr", containers: ["lidarr", "mesh-lidarr"] }, + { name: "plex", containers: ["plex", "mesh-plex"] }, + { name: "bazarr", containers: ["bazarr", "mesh-bazarr"] }, + { name: "nzbget", containers: ["nzbget", "mesh-nzbget"] }, + { name: "qbittorrent", containers: ["qbittorrent", "mesh-qbittorrent"] }, + { name: "jackett", containers: ["jackett", "mesh-jackett"] }, + { name: "ombi", containers: ["ombi", "mesh-ombi"] }, + { name: "tautulli", containers: ["tautulli", "mesh-tautulli"] }, + { name: "bookshelf", containers: ["bookshelf", "mesh-bookshelf"] }, + { name: "home-assistant", containers: ["home-assistant", "mesh-home-assistant"] }, + { name: "mosquitto", containers: ["mosquitto", "mesh-mosquitto"], runOnce: ["mosquitto-bootstrap"] }, + { name: "influxdb", containers: ["influxdb", "mesh-influxdb"] }, + { name: "grafana", containers: ["grafana", "mesh-grafana"] }, + { name: "baserow", containers: ["baserow", "mesh-baserow"] }, + { name: "letta", containers: ["letta", "mesh-letta"] }, + { name: "nodered", containers: ["nodered", "mesh-nodered"] }, + { name: "searxng", containers: ["valkey", "searxng", "mesh-searxng"] }, + { name: "unifi", containers: ["unifi-controller", "mesh-unifi"] }, + { name: "portainer", containers: ["portainer", "mesh-portainer"] }, +]; + +/** Filled in after the first observation run — see the header note on iterate-to-green. */ +const DROPPED: { name: string; why: string }[] = []; + +/** + * The provable CORE that gates green. Refined from the observation run: the whole set of 24 + * RESOLVES and applies (214 resources, node applied+current), including the ADR-0051 media + * `accesses` shared-dir mechanism — and these 17 converge WHOLE (every non-runOnce container up). + * KNOWN_GAPS below are reported and escalated but do not gate. + */ +const CORE = new Set([ + "postgres", "redis", "mssql", + "sonarr", "radarr", "lidarr", "jackett", "tautulli", "bookshelf", + "mosquitto", "influxdb", "grafana", "baserow", "nodered", "searxng", "unifi", "portainer", +]); + +/** + * KNOWN GAPS: resolve and place, but a container does not stay up. Two classes. + * + * A) TOOL-RUNTIME NEEDS AN OPERATOR CREDENTIAL THE MANIFEST DOES NOT WIRE. The mesh- sidecar + * cannot construct its client and crash-loops (verbatim below); the SERVER of each is UP — only + * the tool sidecar is down. This is the umami/photos class from whole-mesh-novox, systemic across + * the media/home tools whose key a human sets in the app UI rather than one derivable from a + * config file (sonarr/radarr/lidarr/jackett/tautulli DO self-configure from the app's config file, + * so their runtimes come up): + * plex — "no Plex token — set MESH_PLEX_TOKEN or make the data dir readable" + * bazarr — "no Bazarr API key — set MESH_BAZARR_API_KEY" + * nzbget — "NZBGet not configured — set MESH_NZBGET_URL and MESH_NZBGET_PASSWORD" + * qbittorrent — "qBittorrent not configured — set MESH_QBITTORRENT_URL and MESH_QBITTORRENT_PASSWORD" + * ombi — "no Ombi API key — set MESH_OMBI_API_KEY" + * home-assistant — "no Home Assistant token — set MESH_HOMEASSISTANT_TOKEN" + * + * B) APP MIGRATION AGAINST POSTGRES. + * letta — the letta APP's DB migration fails on first boot ("connection to server at + * ace.internal … port 5432 failed: Connection refused"); baserow, the other postgres + * consumer, comes up over the same overlay path, so this is letta's own startup + * ordering / lack of retry. The deeper blocker once it connects is the postgres `vector` + * (pgvector) extension a non-superuser consumer cannot CREATE — documented the same way + * in assigned-two-node-db.test.ts. Its runtime mesh-letta and its credential are fine. + */ +const KNOWN_GAPS = new Set([ + "plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta", +]); + +/** + * Host-port remaps applied at load time to break the co-located host-port collisions. In this set + * three servers claim :8080 (qbittorrent, searxng, the UniFi controller) and two claim :6789 (nzbget, + * the UniFi controller). UniFi keeps its published ports; qbittorrent/searxng/nzbget are remapped. + * Container ports are preserved; only the host side changes. + */ +const REMAP: Record> = { + qbittorrent: { "8080": "8090:8080" }, + searxng: { "8080": "8092:8080" }, + nzbget: { "6789": "6790:6789" }, +}; + +let instanceId = ""; +let stocked: string[] = []; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} + +async function mesh(command: string, timeoutMs?: number): Promise { + return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); +} + +function repositoryFor(reference: string): string { + const withoutDigest = reference.split("@")[0] ?? reference; + const lastColon = withoutDigest.lastIndexOf(":"); + const lastSlash = withoutDigest.lastIndexOf("/"); + return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; +} + +function pinned(repository: string): string { + const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); + assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`); + return found; +} + +function bundleFor(images: string[]): string { + let text = readFileSync(bundle, "utf8"); + for (const ref of images) { + const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); + const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); + text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); + } + return text; +} + +function loadManifest(name: string): { manifest: string; broker: boolean } { + const path = resolve(catalogDir, name, "module.json"); + const m = JSON.parse(readFileSync(path, "utf8")) as { + resources?: { type: string; image?: string; ports?: string[] }[]; + }; + const remap = REMAP[name] ?? {}; + for (const r of m.resources ?? []) { + if (r.type !== "container") continue; + if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image)); + if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); + } + const manifest = JSON.stringify(m); + return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +interface NodeState { + reached: boolean; + applied: boolean; + current: boolean; + waiting: boolean; + wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined; + raw: string; +} + +async function nodeState(node: string): Promise { + const asked = await on("anchor", `docker exec mesh-control /mesh-control status --json`); + if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + let state: { + wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + try { + state = JSON.parse(asked.out); + } catch { + return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + } + const word = state.reported.find((r) => r.node === node); + const bad = state.wrong.find((w) => w.node === node); + return { + reached: true, + applied: word?.outcome === "applied", + current: !!word?.current, + waiting: state.waiting.some((w) => w.node === node), + wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined, + raw: asked.out, + }; +} + +before(async () => { + if (skip) return; + assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + stocked = raised.images; + + await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); + await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); + const up = await must("anchor", `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { + assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); + } + + for (const machine of ["anchor", NODE]) { + await mesh(`node add ${machine}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, new RegExp(`enrolled as ${machine}`), said); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + } + + // The operator provides the media library: the ADR-0051 `access` resources CONFIRM these paths and + // the media containers mount them, but the mesh creates none of it. Without this the media stack's + // apply is refused ("the path is not present"). + await must(NODE, `mkdir -p ${MEDIA_DIRS.join(" ")}`); +}, { timeout: 2_700_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("the whole ace service set resolves, installs and converges on one node in one push", { + skip, timeout: 3_300_000, +}, async () => { + for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`); + + // The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres/redis). + await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); + await mesh(`overlay place ${NODE} --site lab`); + await mesh("assign anchor networking"); + await mesh(`assign ${NODE} networking`); + + // Add / issue / assign, resiliently: a module the node cannot host is recorded and skipped so one + // bad assignment cannot poison the whole-node push. + const issued: string[] = []; + const assigned = new Set(); + const refused: { name: string; why: string }[] = []; + for (const { name } of MODULES) { + if (DROPPED.some((d) => d.name === name)) continue; + try { + const { manifest, broker } = loadManifest(name); + await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); + await mesh(`module add /${name}.json`); + if (broker) { + await mesh(`module issue ${name} --node ${NODE}`); + issued.push(name); + } + await mesh(`assign ${NODE} ${name}`); + assigned.add(name); + } catch (err) { + const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | "); + refused.push({ name, why }); + console.log(`NOT ASSIGNED ${name}: ${why}`); + } + } + console.log(`issued broker accounts for: ${issued.length} modules`); + if (refused.length) console.log(`refused (node cannot host): ${refused.map((r) => r.name).join(", ")}`); + + // ONE push. + let pushError = ""; + try { + await mesh(`push ${NODE}`, 180_000); + } catch (err) { + pushError = (err as Error).message; + console.log(`PUSH REJECTED:\n${pushError}`); + } + + // Wait for every CORE container to be up (the node pulls ~20GiB of images first), bounded. + const coreContainers = MODULES.filter((m) => CORE.has(m.name) && assigned.has(m.name)) + .flatMap((m) => m.containers); + const psNames = async (): Promise> => { + const out = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + const map = new Map(); + for (const line of out.split("\n")) { + const [n, ...rest] = line.split("\t"); + if (n) map.set(n.trim(), rest.join("\t").trim()); + } + return map; + }; + let psMap = new Map(); + if (!pushError) { + const until = Date.now() + 2_700_000; + while (Date.now() < until) { + psMap = await psNames(); + if (coreContainers.every((c) => (psMap.get(c) ?? "").startsWith("Up"))) break; + await new Promise((r) => setTimeout(r, 8000)); + } + await new Promise((r) => setTimeout(r, 20000)); // let first-boot bounces settle + } + psMap = await psNames(); + const final = await nodeState(NODE); + const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up"); + // A run-once (mosquitto's dynsec bootstrap) seeds state and exits; the mesh removes it on success, + // so absent-from-`docker ps -a` means it completed and was reaped (the node is applied+current, so + // its resource did apply). Present means it must be up or have exited cleanly. + const ranOnce = (name: string): boolean => !psMap.has(name) || /^(Up|Exited \(0\))/.test(psMap.get(name) ?? ""); + + const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; + + // ================================================================================================ + // The per-module report — the deliverable. + // ================================================================================================ + const report: string[] = []; + report.push("================ WHOLE-MESH ace CONVERGENCE ================"); + report.push(`node reached=${final.reached} applied=${final.applied} current=${final.current} waiting=${final.waiting}`); + if (pushError) report.push(`PUSH REJECTED (resolver): ${pushError.split("\n").slice(0, 8).join("\n ")}`); + const failedResources = final.wrong?.failed ?? []; + if (final.wrong) { + report.push(`NODE WRONG: outcome=${final.wrong.outcome} refused=${final.wrong.refused ?? "-"}`); + for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`); + } + if (DROPPED.length) { + report.push("---- DROPPED ----"); + for (const d of DROPPED) report.push(` ${d.name.padEnd(16)} ${d.why}`); + } + if (refused.length) { + report.push("---- REFUSED at assign ----"); + for (const r of refused) report.push(` ${r.name.padEnd(16)} ${r.why}`); + } + const line = (mod: typeof MODULES[number]): { text: string; ok: boolean } => { + const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`); + const extra = (mod.runOnce ?? []).map((c) => `${c}:${ranOnce(c) ? "ran" : (psMap.get(c) ?? "MISSING")}`); + const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce); + return { text: `${mod.name.padEnd(16)} ${ok ? "OK " : "GAP "} ${[...states, ...extra].join(" ")}`, ok }; + }; + report.push("---- CORE (gates green) ----"); + const coreFailures: string[] = []; + const gaps: string[] = []; + for (const mod of MODULES) { + if (!assigned.has(mod.name)) continue; + const { text, ok } = line(mod); + if (CORE.has(mod.name)) { + report.push(` ${text}`); + if (!ok) coreFailures.push(mod.name); + } else { + gaps.push(` ${text}`); + } + } + report.push("---- KNOWN GAPS (reported, escalated, do NOT gate green) ----"); + for (const g of gaps) report.push(g); + report.push(`broker accounts issued: ${issued.filter((n) => new RegExp(`${NODE}-${n}\\b`).test(users)).length}/${issued.length} present`); + const summary = report.join("\n"); + console.log(summary); + + // Diagnostics for anything not up: exact crash cause per container. + const toDump = MODULES.filter((m) => assigned.has(m.name) && (coreFailures.includes(m.name) || KNOWN_GAPS.has(m.name))); + if (toDump.length) { + console.log(`\n---- ${NODE} mesh-host.log tail ----\n${(await on(NODE, `tail -60 /var/log/mesh-host.log`)).out}`); + for (const mod of toDump) { + for (const c of mod.containers) { + if (psMap.has(c) && !running(c)) { + console.log(`\n---- logs: ${c} (${psMap.get(c)}) ----\n${(await on(NODE, `docker logs ${c} 2>&1 | tail -20`)).out}`); + } + } + } + } + + // ================================================================================================ + // GREEN = the whole set RESOLVED (push accepted), every CORE module converged whole, and no CORE + // resource failed to apply. KNOWN_GAPS and DROPPED are reported and escalated but do not gate. + // ================================================================================================ + assert.equal(pushError, "", `the whole set did not resolve — push was rejected:\n${pushError}`); + const coreResourceFailures = failedResources.filter((f) => CORE.has(f.id.split(".")[0] ?? "")); + assert.deepEqual(coreResourceFailures, [], + `a CORE resource failed to apply:\n${coreResourceFailures.map((f) => `${f.id}: ${f.error}`).join("\n")}\n${summary}`); + assert.deepEqual(coreFailures, [], + `these CORE modules did not converge whole: ${coreFailures.join(", ")}\n${summary}`); +}); -- 2.54.0 From fcdcd338c027b0688edad9894f01ab0c9f0a076c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 8 Sep 2026 18:22:56 +0200 Subject: [PATCH 3/4] Add whole-mesh full three-node bed (stage 3: both server sets, one substrate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Combine the novox (17-module) and ace (24-module) sets on ONE substrate and prove both node-plans converge together. anchor runs the substrate only; novox and ace each run their own self-contained set (own postgres/redis), so nothing crosses a node boundary except enrolment and the shared broker/store. The four modules both nodes run (postgres, redis, mssql, portainer) are added once and assigned to each node, each getting its own per-node broker account. An overlay is placed across all three nodes. Proven green: both nodes converge together on the one substrate. ace reaches applied+current with all 17 of its CORE up (and letta too this run); novox reaches all 13 CORE up with its only failed resource the known firewall.load oneshot gap. The two node-plans share one broker without collision — distinct novox- and ace- accounts for the modules both run. No new cross-node bug (overlay/DNS/identity/port) surfaced; ports are per-VM and the sets are node-self-contained. Tolerates the same nine credential-sidecar gaps and firewall's nftables.service oneshot documented in the per-server beds. Resource envelope: 3 VMs (anchor 4GiB, novox 16GiB, ace 18GiB) + registry scenery, ~79 union images (~35GB) stocked to one registry VM and pulled concurrently by both nodes; fit within 125GiB host RAM and the 180GiB lab pool. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- scenarios/whole-mesh-full.yml | 130 +++++++ test/integration/whole-mesh-full.test.ts | 431 +++++++++++++++++++++++ 2 files changed, 561 insertions(+) create mode 100644 scenarios/whole-mesh-full.yml create mode 100644 test/integration/whole-mesh-full.test.ts diff --git a/scenarios/whole-mesh-full.yml b/scenarios/whole-mesh-full.yml new file mode 100644 index 0000000..2845078 --- /dev/null +++ b/scenarios/whole-mesh-full.yml @@ -0,0 +1,130 @@ +# The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the +# whole-mesh rehearsal (novox/hq). Combines scenarios/whole-mesh-novox.yml and whole-mesh-ace.yml. +# +# anchor — substrate ONLY (store, broker, control). +# novox — the 17-module novox set (providers + web apps + route-proxy + mailu + firewall). +# ace — the 24-module ace set (media/home stack), its /services/media library pre-created. +# +# An overlay is placed across all three so cross-node `at` resolves. Each service node is +# self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and +# the shared broker/store on anchor — which is exactly what this stage proves converges for two +# independent node-plans at once on one substrate. +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock +# The images are the UNION of the two per-server scenarios; every one is already built/pulled by the +# per-server bed prerequisites (scripts/build-module-runtime.sh, build-route-proxy-image.sh, the +# mailu/keycloak and media :mesh digest pulls). +scenario: whole-mesh-full + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + memory: 4GiB + cpus: 4 + disk: 20GiB + novox: + at: { segment: hosting, address: [192.0.2.20] } + inbound: allow + memory: 16GiB + cpus: 6 + disk: 100GiB + ace: + at: { segment: hosting, address: [192.0.2.30] } + inbound: allow + memory: 18GiB + cpus: 6 + disk: 120GiB + +images: + # --- substrate + shared --- + - postgres:17-alpine + - cloudamqp/lavinmq:latest + - mesh-control:development + - redis:7-alpine + - portainer/portainer-ce:latest + - mcr.microsoft.com/mssql/server:2022-latest + # --- novox server images --- + - minio/minio:latest + - mongo:7 + - quay.io/keycloak/keycloak:mesh + - gitea/gitea:1.22 + - nextcloud:stable + - ghcr.io/umami-software/umami:postgresql-latest + - alpine:latest + - verdaccio/verdaccio:6 + - registry:2 + - registry-api.novox.be/novox/invoicing-app:latest + - registry-api.novox.be/novox/invoicing-api:latest + - ghcr.io/mailu/unbound:mesh + - ghcr.io/mailu/admin:mesh + - ghcr.io/mailu/dovecot:mesh + - ghcr.io/mailu/postfix:mesh + - ghcr.io/mailu/rspamd:mesh + - ghcr.io/mailu/webmail:mesh + - ghcr.io/mailu/nginx:mesh + # --- ace server images --- + - lscr.io/linuxserver/sonarr:mesh + - lscr.io/linuxserver/radarr:mesh + - lscr.io/linuxserver/lidarr:mesh + - lscr.io/linuxserver/bazarr:mesh + - lscr.io/linuxserver/nzbget:mesh + - lscr.io/linuxserver/qbittorrent:mesh + - lscr.io/linuxserver/jackett:mesh + - lscr.io/linuxserver/ombi:mesh + - lscr.io/linuxserver/tautulli:mesh + - lscr.io/linuxserver/unifi-controller:mesh + - plexinc/pms-docker:mesh + - ghcr.io/pennydreadful/bookshelf:mesh + - ghcr.io/home-assistant/home-assistant:mesh + - eclipse-mosquitto:mesh + - influxdb:mesh + - grafana/grafana:mesh + - baserow/baserow:mesh + - letta/letta:mesh + - nodered/node-red:mesh + - searxng/searxng:mesh + - valkey/valkey:mesh + # --- per-module runtimes (union) --- + - mesh-runtime-postgres:development + - mesh-runtime-redis:development + - mesh-runtime-mssql:development + - mesh-runtime-portainer:development + - mesh-runtime-minio:development + - mesh-runtime-mongodb:development + - mesh-runtime-keycloak:development + - mesh-runtime-gitea:development + - mesh-runtime-nextcloud:development + - mesh-runtime-umami:development + - mesh-runtime-photos:development + - mesh-runtime-verdaccio:development + - mesh-runtime-mailu:development + - mesh-route-proxy:development + - mesh-runtime-sonarr:development + - mesh-runtime-radarr:development + - mesh-runtime-lidarr:development + - mesh-runtime-plex:development + - mesh-runtime-bazarr:development + - mesh-runtime-nzbget:development + - mesh-runtime-qbittorrent:development + - mesh-runtime-jackett:development + - mesh-runtime-ombi:development + - mesh-runtime-tautulli:development + - mesh-runtime-bookshelf:development + - mesh-runtime-home-assistant:development + - mesh-runtime-mosquitto:development + - mesh-runtime-influxdb:development + - mesh-runtime-grafana:development + - mesh-runtime-baserow:development + - mesh-runtime-letta:development + - mesh-runtime-nodered:development + - mesh-runtime-searxng:development + - mesh-runtime-unifi:development + +place: + all: [host, runtime] diff --git a/test/integration/whole-mesh-full.test.ts b/test/integration/whole-mesh-full.test.ts new file mode 100644 index 0000000..51b2a32 --- /dev/null +++ b/test/integration/whole-mesh-full.test.ts @@ -0,0 +1,431 @@ +/** + * The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the + * whole-mesh rehearsal (novox/hq). Combines whole-mesh-novox.test.ts and whole-mesh-ace.test.ts. + * + * anchor — substrate ONLY (store, broker, control). + * novox — the 17-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu, + * firewall. fail2ban is dropped (no `intrusion-prevention` detector — see that bed). + * ace — the 24-module ace set (whole-mesh-ace): the media/home stack; its /services/media + * library is pre-created so the ADR-0051 `accesses` resolve. + * + * An overlay is placed across all three so cross-node `at` resolves. Each service node is + * self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and + * the shared broker/store on anchor. The four modules both nodes run (postgres, redis, mssql, + * portainer) are ADDED once and assigned to each node; each gets its own per-node broker account. + * + * This bed tolerates the SAME known gaps the per-server beds proved and escalated (nine + * credential-sidecar crash-loops and firewall's oneshot nftables.service); it gates green on each + * node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two node-plans + * converge together on one substrate. + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll } from "./harness.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" + : false; + +const SCENARIO = "whole-mesh-full"; + +const catalogDir = process.env["MESH_LAB_CATALOG"] + ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") + ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + +const MEDIA_DIRS = [ + "/services/media/series", "/services/media/anime", "/services/media/movies", + "/services/media/music", "/services/media/audiobooks", "/services/media/downloads", + "/services/media/books", +]; + +type Mod = { name: string; containers: string[]; node?: boolean; runOnce?: string[] }; + +/** The novox node's 17-module set (fail2ban dropped). CORE gates; the rest are documented gaps. */ +const NOVOX: Mod[] = [ + { name: "postgres", containers: ["postgres", "mesh-postgres"] }, + { name: "redis", containers: ["redis", "mesh-redis"] }, + { name: "minio", containers: ["minio", "mesh-minio"] }, + { name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, + { name: "mssql", containers: ["mssql", "mesh-mssql"] }, + { name: "keycloak", containers: ["keycloak", "mesh-keycloak"] }, + { name: "gitea", containers: ["gitea", "mesh-gitea"] }, + { name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] }, + { name: "umami", containers: ["umami", "mesh-umami"] }, + { name: "photos", containers: ["photos", "mesh-photos"] }, + { name: "invoicing", containers: ["invoicing-app", "invoicing-api"] }, + { name: "portainer", containers: ["portainer", "mesh-portainer"] }, + { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] }, + { name: "registry", containers: ["mesh-registry"] }, + { name: "route-proxy", containers: ["route-proxy"] }, + { + name: "mailu", + containers: [ + "mailu-resolver", "mailu-redis", "mailu-admindb", "mailu-admin", "mailu-imap", + "mailu-smtp", "mailu-antispam", "mailu-webmail", "mailu-front", "mesh-mailu", + ], + }, + { name: "firewall", containers: [], node: true }, +]; +const CORE_NOVOX = new Set([ + "postgres", "redis", "minio", "mongodb", "mssql", + "keycloak", "gitea", "nextcloud", "invoicing", + "portainer", "verdaccio", "registry", "route-proxy", +]); +const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall"]); + +/** The ace node's 24-module set. */ +const ACE: Mod[] = [ + { name: "postgres", containers: ["postgres", "mesh-postgres"] }, + { name: "redis", containers: ["redis", "mesh-redis"] }, + { name: "mssql", containers: ["mssql", "mesh-mssql"] }, + { name: "sonarr", containers: ["sonarr", "mesh-sonarr"] }, + { name: "radarr", containers: ["radarr", "mesh-radarr"] }, + { name: "lidarr", containers: ["lidarr", "mesh-lidarr"] }, + { name: "plex", containers: ["plex", "mesh-plex"] }, + { name: "bazarr", containers: ["bazarr", "mesh-bazarr"] }, + { name: "nzbget", containers: ["nzbget", "mesh-nzbget"] }, + { name: "qbittorrent", containers: ["qbittorrent", "mesh-qbittorrent"] }, + { name: "jackett", containers: ["jackett", "mesh-jackett"] }, + { name: "ombi", containers: ["ombi", "mesh-ombi"] }, + { name: "tautulli", containers: ["tautulli", "mesh-tautulli"] }, + { name: "bookshelf", containers: ["bookshelf", "mesh-bookshelf"] }, + { name: "home-assistant", containers: ["home-assistant", "mesh-home-assistant"] }, + { name: "mosquitto", containers: ["mosquitto", "mesh-mosquitto"], runOnce: ["mosquitto-bootstrap"] }, + { name: "influxdb", containers: ["influxdb", "mesh-influxdb"] }, + { name: "grafana", containers: ["grafana", "mesh-grafana"] }, + { name: "baserow", containers: ["baserow", "mesh-baserow"] }, + { name: "letta", containers: ["letta", "mesh-letta"] }, + { name: "nodered", containers: ["nodered", "mesh-nodered"] }, + { name: "searxng", containers: ["valkey", "searxng", "mesh-searxng"] }, + { name: "unifi", containers: ["unifi-controller", "mesh-unifi"] }, + { name: "portainer", containers: ["portainer", "mesh-portainer"] }, +]; +const CORE_ACE = new Set([ + "postgres", "redis", "mssql", + "sonarr", "radarr", "lidarr", "jackett", "tautulli", "bookshelf", + "mosquitto", "influxdb", "grafana", "baserow", "nodered", "searxng", "unifi", "portainer", +]); +const GAPS_ACE = new Set(["plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta"]); + +const PLAN: { node: string; mods: Mod[]; core: Set; gaps: Set }[] = [ + { node: "novox", mods: NOVOX, core: CORE_NOVOX, gaps: GAPS_NOVOX }, + { node: "ace", mods: ACE, core: CORE_ACE, gaps: GAPS_ACE }, +]; + +/** + * Host-port remaps (per module — host ports are per-VM, so novox's and ace's never clash). Union of + * both per-server beds' remaps. + */ +const REMAP: Record> = { + nextcloud: { "80": "8090:80" }, + umami: { "3000": "3090:3000" }, + invoicing: { "80": "8091:80", "9000": "9091:9000" }, + qbittorrent: { "8080": "8090:8080" }, + searxng: { "8080": "8092:8080" }, + nzbget: { "6789": "6790:6789" }, +}; + +let instanceId = ""; +let stocked: string[] = []; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} + +async function mesh(command: string, timeoutMs?: number): Promise { + return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); +} + +function repositoryFor(reference: string): string { + const withoutDigest = reference.split("@")[0] ?? reference; + const lastColon = withoutDigest.lastIndexOf(":"); + const lastSlash = withoutDigest.lastIndexOf("/"); + return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; +} + +function pinned(repository: string): string { + const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); + assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`); + return found; +} + +function bundleFor(images: string[]): string { + let text = readFileSync(bundle, "utf8"); + for (const ref of images) { + const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); + const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); + text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); + } + return text; +} + +function loadManifest(name: string): { manifest: string; broker: boolean } { + const path = resolve(catalogDir, name, "module.json"); + const m = JSON.parse(readFileSync(path, "utf8")) as { + resources?: { type: string; image?: string; ports?: string[] }[]; + }; + const remap = REMAP[name] ?? {}; + for (const r of m.resources ?? []) { + if (r.type !== "container") continue; + if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image)); + if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); + } + const manifest = JSON.stringify(m); + return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +interface NodeState { + reached: boolean; + applied: boolean; + current: boolean; + waiting: boolean; + wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined; + raw: string; +} + +async function nodeState(node: string): Promise { + const asked = await on("anchor", `docker exec mesh-control /mesh-control status --json`); + if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + let state: { + wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + try { + state = JSON.parse(asked.out); + } catch { + return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; + } + const word = state.reported.find((r) => r.node === node); + const bad = state.wrong.find((w) => w.node === node); + return { + reached: true, + applied: word?.outcome === "applied", + current: !!word?.current, + waiting: state.waiting.some((w) => w.node === node), + wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined, + raw: asked.out, + }; +} + +async function psMapOf(node: string): Promise> { + const out = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + const map = new Map(); + for (const line of out.split("\n")) { + const [n, ...rest] = line.split("\t"); + if (n) map.set(n.trim(), rest.join("\t").trim()); + } + return map; +} + +before(async () => { + if (skip) return; + assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + stocked = raised.images; + + await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); + await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); + const up = await must("anchor", `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { + assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); + } + + for (const machine of ["anchor", "novox", "ace"]) { + await mesh(`node add ${machine}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); + assert.match(said, new RegExp(`enrolled as ${machine}`), said); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + } + + // The operator provides ace's media library (ADR 0051 accesses confirm the paths, create nothing). + await must("ace", `mkdir -p ${MEDIA_DIRS.join(" ")}`); +}, { timeout: 3_000_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("both server sets converge together on one substrate", { skip, timeout: 3_600_000 }, async () => { + // Overlay across all three, so every node's private address exists and cross-node `at` resolves. + await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); + await mesh("overlay place novox --site lab"); + await mesh("overlay place ace --site lab"); + await mesh("assign anchor networking"); + await mesh("assign novox networking"); + await mesh("assign ace networking"); + + // Add every unique module ONCE (the four shared modules are added once, assigned to each node), then + // issue a per-node broker account and assign, resiliently. + const added = new Map(); // name -> needs broker + async function ensureAdded(name: string): Promise { + const known = added.get(name); + if (known !== undefined) return known; + const { manifest, broker } = loadManifest(name); + await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); + await mesh(`module add /${name}.json`); + added.set(name, broker); + return broker; + } + + const assigned: Record> = { novox: new Set(), ace: new Set() }; + const refused: Record = { novox: [], ace: [] }; + for (const { node, mods } of PLAN) { + for (const { name } of mods) { + try { + const broker = await ensureAdded(name); + if (broker) await mesh(`module issue ${name} --node ${node}`); + await mesh(`assign ${node} ${name}`); + assigned[node]!.add(name); + } catch (err) { + const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | "); + refused[node]!.push({ name, why }); + console.log(`NOT ASSIGNED ${node}/${name}: ${why}`); + } + } + } + + // ONE push per node. + const pushError: Record = { novox: "", ace: "" }; + for (const node of ["novox", "ace"]) { + try { + await mesh(`push ${node}`, 240_000); + } catch (err) { + pushError[node] = (err as Error).message; + console.log(`PUSH REJECTED (${node}):\n${pushError[node]}`); + } + } + + // Wait for both nodes' CORE containers to come up (they pull concurrently from the one registry). + const psMaps: Record> = { novox: new Map(), ace: new Map() }; + for (const { node, mods, core } of PLAN) { + if (pushError[node]) continue; + const coreContainers = mods.filter((m) => core.has(m.name) && assigned[node]!.has(m.name)).flatMap((m) => m.containers); + const until = Date.now() + 2_700_000; + while (Date.now() < until) { + psMaps[node] = await psMapOf(node); + if (coreContainers.every((c) => (psMaps[node]!.get(c) ?? "").startsWith("Up"))) break; + await new Promise((r) => setTimeout(r, 10000)); + } + } + await new Promise((r) => setTimeout(r, 20000)); // let first-boot bounces settle + + // ================================================================================================ + // Per-node report + gating. GREEN = each node's push accepted, every CORE module converged whole, + // and no NON-GAP resource failed to apply. The nine credential-sidecar gaps and firewall's oneshot + // are tolerated (documented + escalated in the per-server beds). + // ================================================================================================ + const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; + const allProblems: string[] = []; + const report: string[] = ["================ FULL MESH CONVERGENCE ================"]; + + for (const { node, mods, core, gaps } of PLAN) { + const psMap = psMaps[node] = await psMapOf(node); + const st = await nodeState(node); + const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up"); + const ranOnce = (name: string): boolean => !psMap.has(name) || /^(Up|Exited \(0\))/.test(psMap.get(name) ?? ""); + const failedResources = st.wrong?.failed ?? []; + + report.push(`\n---- node ${node}: reached=${st.reached} applied=${st.applied} current=${st.current} waiting=${st.waiting} ----`); + if (pushError[node]) report.push(` PUSH REJECTED: ${pushError[node].split("\n").slice(0, 6).join("\n ")}`); + if (st.wrong) { + report.push(` NODE WRONG: outcome=${st.wrong.outcome}`); + for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`); + } + for (const r of refused[node]!) report.push(` REFUSED ${r.name}: ${r.why}`); + + const coreFailures: string[] = []; + for (const mod of mods) { + if (!assigned[node]!.has(mod.name)) continue; + const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`); + const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce); + const tag = core.has(mod.name) ? (ok ? "OK " : "FAIL") : (ok ? "ok " : "GAP "); + report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(15)} ${tag} ${states.join(" ")}`); + if (core.has(mod.name) && !ok) coreFailures.push(mod.name); + } + const issuedHere = mods.filter((m) => new RegExp(`${node}-${m.name}\\b`).test(users)).length; + report.push(` broker accounts: ${issuedHere} present for ${node}`); + + // Gate: push accepted, all CORE up, no NON-GAP resource failed. A failed resource names its + // owning module inside the error (`applying "firewall.load": …`), not in `id` (which is the outer + // "apply" key), so the owner is extracted from either — and a failure owned by a KNOWN_GAP module + // (firewall's oneshot nftables.service) is tolerated. + const gapOwnerOf = (f: { id: string; error: string }): string => { + const m = f.error.match(/applying "([^".]+)\./); + return m?.[1] ?? (f.id.split(".")[0] ?? ""); + }; + if (pushError[node]) allProblems.push(`${node}: push rejected`); + if (coreFailures.length) allProblems.push(`${node}: CORE not converged: ${coreFailures.join(", ")}`); + const nonGapFailed = failedResources.filter((f) => !gaps.has(gapOwnerOf(f))); + if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`); + } + + const summary = report.join("\n"); + console.log(summary); + + // Cross-node identity proof: each node's own scoped broker accounts exist and are distinct — the + // two node-plans share one broker without colliding (both run a `postgres`, `redis`, `mssql`). + for (const acct of ["novox-postgres", "ace-postgres", "novox-redis", "ace-redis"]) { + if (!new RegExp(acct).test(users)) allProblems.push(`missing broker account ${acct}`); + } + + // Diagnostics for any CORE failure (the gaps are expected; a CORE failure is what we must see). + for (const { node, mods, core } of PLAN) { + const psMap = psMaps[node]!; + for (const mod of mods) { + if (!core.has(mod.name) || !assigned[node]!.has(mod.name)) continue; + for (const c of mod.containers) { + if (psMap.has(c) && !(psMap.get(c) ?? "").startsWith("Up")) { + console.log(`\n---- ${node} logs: ${c} (${psMap.get(c)}) ----\n${(await on(node, `docker logs ${c} 2>&1 | tail -25`)).out}`); + } + } + } + } + + assert.deepEqual(allProblems, [], `the full mesh did not converge together:\n ${allProblems.join("\n ")}\n\n${summary}`); +}); -- 2.54.0 From 591f2a641c76aef3126fcc09f9360843fd3565b1 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 8 Sep 2026 20:05:04 +0200 Subject: [PATCH 4/4] whole-mesh-full: prove the dry-run fixes (fail2ban hostable, credential own-secrets) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-runs the capstone from main after the dry-run fixes merged. fail2ban: added to the novox set. The capability fix (intrusion-prevention -> firewall) makes it HOSTABLE — it is now assigned, not refused — which is the gate. Its service reaching active is a host concern the offline lab cannot meet (the VM ships nftables but not fail2ban, and the isolated segment has no route to the package mirror, so pacman cannot fetch it), so fail2ban joins GAPS_NOVOX: its failed package resource is tolerated like firewall's oneshot nftables.service. 7 credential sidecars: before the push, a FAKE app credential is delivered for each (plex/bazarr/ombi/home-assistant/nzbget/qbittorrent on ace, umami on novox) through the real operator path — `secret accept --from`. The bed asserts each sidecar advances PAST its old "no credential" crash (it reads the delivered value); app-auth failure against the real app with a bogus value is expected and not gated. Result: SUITE_EXIT=0. Both node-plans converge on one substrate (novox 13/13 core, ace 17/17 core), fail2ban hostable, all 7 sidecars past their crash. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- test/integration/whole-mesh-full.test.ts | 135 +++++++++++++++++++++-- 1 file changed, 126 insertions(+), 9 deletions(-) diff --git a/test/integration/whole-mesh-full.test.ts b/test/integration/whole-mesh-full.test.ts index 51b2a32..447c363 100644 --- a/test/integration/whole-mesh-full.test.ts +++ b/test/integration/whole-mesh-full.test.ts @@ -3,8 +3,10 @@ * whole-mesh rehearsal (novox/hq). Combines whole-mesh-novox.test.ts and whole-mesh-ace.test.ts. * * anchor — substrate ONLY (store, broker, control). - * novox — the 17-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu, - * firewall. fail2ban is dropped (no `intrusion-prevention` detector — see that bed). + * novox — the 18-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu, + * firewall, fail2ban. fail2ban is now HOSTABLE: the dry-run fixes (mesh-control/catalog + * main) changed its declared capability from the never-detected "intrusion-prevention" to + * "firewall", the detector every node with nft already advertises. * ace — the 24-module ace set (whole-mesh-ace): the media/home stack; its /services/media * library is pre-created so the ADR-0051 `accesses` resolve. * @@ -13,10 +15,24 @@ * the shared broker/store on anchor. The four modules both nodes run (postgres, redis, mssql, * portainer) are ADDED once and assigned to each node; each gets its own per-node broker account. * - * This bed tolerates the SAME known gaps the per-server beds proved and escalated (nine - * credential-sidecar crash-loops and firewall's oneshot nftables.service); it gates green on each - * node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two node-plans - * converge together on one substrate. + * THE DRY-RUN FIXES THIS RUN PROVES (mesh-control + mesh-catalog main): + * - fail2ban is HOSTABLE (capability "firewall"): it is assigned, not refused. Before, it declared + * the never-detected "intrusion-prevention" capability, so no node could host it and its + * un-hostable assignment refused the whole node's push. Hostability is the gate. Its service + * reaching active is a host concern this offline lab cannot meet — the VM ships nftables (so the + * firewall detector is advertised) but not fail2ban, and the isolated segment has no route to the + * package mirror, so pacman cannot fetch it. That is a documented lab gap, reported not gated. + * - the 7 tool-runtime credential modules (ace: plex, bazarr, ombi, home-assistant, nzbget, + * qbittorrent; novox: umami) now read their app credential from an operator-provided own-secret. + * This bed delivers a FAKE value for each through the real operator path (`secret accept`) + * BEFORE the push, and gates on the sidecar getting PAST its old "no credential" crash (it reads + * the delivered value). A fake value will not authenticate against the real app — the sidecar may + * still fail at app-auth, which is expected and does NOT gate; only the crash being GONE gates. + * + * It otherwise tolerates the SAME known gaps the per-server beds proved and escalated (the credential + * sidecars' app-auth failures, photos/mailu, and firewall's oneshot nftables.service); it gates green + * on each node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two + * node-plans converge together on one substrate. * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock */ @@ -83,13 +99,14 @@ const NOVOX: Mod[] = [ ], }, { name: "firewall", containers: [], node: true }, + { name: "fail2ban", containers: [], node: true }, ]; const CORE_NOVOX = new Set([ "postgres", "redis", "minio", "mongodb", "mssql", "keycloak", "gitea", "nextcloud", "invoicing", "portainer", "verdaccio", "registry", "route-proxy", ]); -const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall"]); +const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall", "fail2ban"]); /** The ace node's 24-module set. */ const ACE: Mod[] = [ @@ -143,6 +160,25 @@ const REMAP: Record> = { nzbget: { "6789": "6790:6789" }, }; +/** + * The 7 tool-runtime credential modules (novox/hq dry-run fix). Each now reads its app credential + * from an operator-provided own-secret (`name`, an own-secret path in its module.json), mounted into + * the sidecar at MESH_*_FILE. This bed delivers a FAKE value for each via the real operator path + * (`secret accept --from `) BEFORE the push, and asserts the sidecar + * gets PAST `crash` — the exact message its client threw when nothing was mounted. A fake value does + * not authenticate against the real app, so the sidecar may still fail later at app-auth (expected, + * not gated); only the "no credential" crash being GONE proves the wiring and gates. + */ +const CREDENTIALS: { node: string; module: string; name: string; crash: string }[] = [ + { node: "ace", module: "plex", name: "token", crash: "no Plex token" }, + { node: "ace", module: "bazarr", name: "api-key", crash: "no Bazarr API key" }, + { node: "ace", module: "ombi", name: "api-key", crash: "no Ombi API key" }, + { node: "ace", module: "home-assistant", name: "token", crash: "no Home Assistant token" }, + { node: "ace", module: "nzbget", name: "password", crash: "NZBGet not configured" }, + { node: "ace", module: "qbittorrent", name: "password", crash: "qBittorrent not configured" }, + { node: "novox", module: "umami", name: "admin", crash: "admin password is not set" }, +]; + let instanceId = ""; let stocked: string[] = []; @@ -330,6 +366,31 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 } } + // Operator-provided app credentials (novox/hq dry-run fix). BEFORE the push, hand the mesh a FAKE + // value for each of the 7 credential modules through the real operator path — `secret accept`, + // which seals the value to the node and records it as `accepted` (the mesh will not invent one). + // The push then delivers it to the sidecar's own-secret path. The `--from` file is staged into the + // mesh-control container (one file per distinct secret name). A module the node could not host is + // skipped (its secret has nowhere to go). + const credentialDelivered = new Map(); + for (const name of new Set(CREDENTIALS.map((c) => c.name))) { + await must("anchor", `printf %s ${quote(`fake-${name}-value`)} > /tmp/fake-${name} && docker cp /tmp/fake-${name} mesh-control:/fake-${name}`); + } + for (const c of CREDENTIALS) { + if (!assigned[c.node]!.has(c.module)) { + credentialDelivered.set(`${c.node}/${c.module}`, false); + console.log(`CREDENTIAL SKIPPED ${c.node}/${c.module}: not assigned, nowhere to deliver`); + continue; + } + try { + await mesh(`secret accept ${c.node} ${c.module} ${c.name} --from /fake-${c.name}`); + credentialDelivered.set(`${c.node}/${c.module}`, true); + } catch (err) { + credentialDelivered.set(`${c.node}/${c.module}`, false); + console.log(`CREDENTIAL ACCEPT FAILED ${c.node}/${c.module}: ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`); + } + } + // ONE push per node. const pushError: Record = { novox: "", ace: "" }; for (const node of ["novox", "ace"]) { @@ -357,8 +418,10 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 // ================================================================================================ // Per-node report + gating. GREEN = each node's push accepted, every CORE module converged whole, - // and no NON-GAP resource failed to apply. The nine credential-sidecar gaps and firewall's oneshot - // are tolerated (documented + escalated in the per-server beds). + // no NON-GAP resource failed to apply, fail2ban is hostable (assigned, not refused), and every + // credential sidecar advanced past its "no credential" crash. Tolerated: the credential sidecars' + // app-auth failures (bogus fake value), photos/mailu, firewall's oneshot nftables.service, and + // fail2ban's package (the offline lab cannot fetch it — a documented host gap). // ================================================================================================ const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; const allProblems: string[] = []; @@ -405,6 +468,60 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`); } + // ================================================================================================ + // The dry-run fixes, proved by name. + // ================================================================================================ + + // fail2ban is now HOSTABLE (capability "firewall"): what the dry-run fix buys is that a node can + // host it at all. Before, it declared the never-detected "intrusion-prevention" capability, so NO + // node could host it AND its un-hostable assignment refused the whole node's push. So the GATE is + // hostability: it must be ASSIGNED and NOT refused. + // + // Its systemd service reaching active is a SEPARATE, host-level concern this offline lab cannot + // satisfy: the VM base image ships `nftables` (so firewall's package resolves and the `firewall` + // detector is advertised — which is exactly why fail2ban is now hostable) but NOT `fail2ban`, and + // the lab segment (RFC 5737 192.0.2.0/24) has no route to the package mirror, so pacman times out + // fetching fail2ban and its deps. That is a documented LAB gap (fail2ban ∈ GAPS_NOVOX, so its + // failed `fail2ban.package` resource is tolerated like firewall's oneshot nftables.service) — it is + // reported, not gated. On an online node the package installs and the service runs. + { + const refusedF2B = refused["novox"]!.find((r) => r.name === "fail2ban"); + const assignedF2B = assigned["novox"]!.has("fail2ban"); + const active = (await on("novox", `systemctl is-active fail2ban 2>&1`)).out.trim(); + const pkg = (await on("novox", `pacman -Q fail2ban 2>&1`)).out.trim(); + report.push(`\n---- fail2ban (novox): HOSTABLE assigned=${assignedF2B} refused=${refusedF2B ? "YES" : "no"} | service=${active} package="${pkg}" ----`); + if (refusedF2B) { + allProblems.push(`fail2ban still not hostable on novox: ${refusedF2B.why}`); + } else if (!assignedF2B) { + allProblems.push(`fail2ban was not assigned to novox`); + } + if (active !== "active") { + report.push(` service not active — offline lab could not install the package (documented gap, not gated); detail:`); + report.push(` ${(await on("novox", `systemctl status fail2ban --no-pager 2>&1 | head -8`)).out}`); + } + } + + // The 7 credential sidecars: each got its fake own-secret, so each must have advanced PAST the old + // "no credential" crash (it read the delivered value). It may still fail at app-auth against the + // real app with a bogus value — that is expected and does NOT gate; only the crash being gone does. + report.push(`\n---- credential sidecars: past the "no credential" crash? (fake secret delivered) ----`); + for (const c of CREDENTIALS) { + const container = `mesh-${c.module}`; + const psMap = psMaps[c.node]!; + const status = (psMap.get(container) ?? "MISSING").split(" ")[0] ?? "MISSING"; + const delivered = credentialDelivered.get(`${c.node}/${c.module}`) ?? false; + const logs = (await on(c.node, `docker logs ${container} 2>&1 | tail -60`)).out; + const stillCrashes = logs.includes(c.crash); + const appAuth = logs.split("\n").reverse().find((l) => /fail|reject|error|401|403|refused/i.test(l) && !l.includes(c.crash))?.trim().slice(0, 90) ?? ""; + report.push(` ${c.node}/${c.module.padEnd(15)} secret=${delivered ? "delivered" : "SKIPPED"} sidecar=${status.padEnd(10)} crash("${c.crash}")=${stillCrashes ? "STILL PRESENT" : "gone"}${appAuth ? ` last:"${appAuth}"` : ""}`); + if (delivered && stillCrashes) { + allProblems.push(`${c.node}/${c.module}: credential wiring did not take — sidecar still crashes "${c.crash}"`); + } + if (!delivered && assigned[c.node]!.has(c.module)) { + allProblems.push(`${c.node}/${c.module}: fake credential was not delivered (secret accept failed)`); + } + } + const summary = report.join("\n"); console.log(summary); -- 2.54.0