diff --git a/test/integration/whole-mesh-full.test.ts b/test/integration/whole-mesh-full.test.ts index 51b2a32..447c363 100644 --- a/test/integration/whole-mesh-full.test.ts +++ b/test/integration/whole-mesh-full.test.ts @@ -3,8 +3,10 @@ * whole-mesh rehearsal (novox/hq). Combines whole-mesh-novox.test.ts and whole-mesh-ace.test.ts. * * anchor — substrate ONLY (store, broker, control). - * novox — the 17-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu, - * firewall. fail2ban is dropped (no `intrusion-prevention` detector — see that bed). + * novox — the 18-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu, + * firewall, fail2ban. fail2ban is now HOSTABLE: the dry-run fixes (mesh-control/catalog + * main) changed its declared capability from the never-detected "intrusion-prevention" to + * "firewall", the detector every node with nft already advertises. * ace — the 24-module ace set (whole-mesh-ace): the media/home stack; its /services/media * library is pre-created so the ADR-0051 `accesses` resolve. * @@ -13,10 +15,24 @@ * the shared broker/store on anchor. The four modules both nodes run (postgres, redis, mssql, * portainer) are ADDED once and assigned to each node; each gets its own per-node broker account. * - * This bed tolerates the SAME known gaps the per-server beds proved and escalated (nine - * credential-sidecar crash-loops and firewall's oneshot nftables.service); it gates green on each - * node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two node-plans - * converge together on one substrate. + * THE DRY-RUN FIXES THIS RUN PROVES (mesh-control + mesh-catalog main): + * - fail2ban is HOSTABLE (capability "firewall"): it is assigned, not refused. Before, it declared + * the never-detected "intrusion-prevention" capability, so no node could host it and its + * un-hostable assignment refused the whole node's push. Hostability is the gate. Its service + * reaching active is a host concern this offline lab cannot meet — the VM ships nftables (so the + * firewall detector is advertised) but not fail2ban, and the isolated segment has no route to the + * package mirror, so pacman cannot fetch it. That is a documented lab gap, reported not gated. + * - the 7 tool-runtime credential modules (ace: plex, bazarr, ombi, home-assistant, nzbget, + * qbittorrent; novox: umami) now read their app credential from an operator-provided own-secret. + * This bed delivers a FAKE value for each through the real operator path (`secret accept`) + * BEFORE the push, and gates on the sidecar getting PAST its old "no credential" crash (it reads + * the delivered value). A fake value will not authenticate against the real app — the sidecar may + * still fail at app-auth, which is expected and does NOT gate; only the crash being GONE gates. + * + * It otherwise tolerates the SAME known gaps the per-server beds proved and escalated (the credential + * sidecars' app-auth failures, photos/mailu, and firewall's oneshot nftables.service); it gates green + * on each node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two + * node-plans converge together on one substrate. * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock */ @@ -83,13 +99,14 @@ const NOVOX: Mod[] = [ ], }, { name: "firewall", containers: [], node: true }, + { name: "fail2ban", containers: [], node: true }, ]; const CORE_NOVOX = new Set([ "postgres", "redis", "minio", "mongodb", "mssql", "keycloak", "gitea", "nextcloud", "invoicing", "portainer", "verdaccio", "registry", "route-proxy", ]); -const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall"]); +const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall", "fail2ban"]); /** The ace node's 24-module set. */ const ACE: Mod[] = [ @@ -143,6 +160,25 @@ const REMAP: Record> = { nzbget: { "6789": "6790:6789" }, }; +/** + * The 7 tool-runtime credential modules (novox/hq dry-run fix). Each now reads its app credential + * from an operator-provided own-secret (`name`, an own-secret path in its module.json), mounted into + * the sidecar at MESH_*_FILE. This bed delivers a FAKE value for each via the real operator path + * (`secret accept --from `) BEFORE the push, and asserts the sidecar + * gets PAST `crash` — the exact message its client threw when nothing was mounted. A fake value does + * not authenticate against the real app, so the sidecar may still fail later at app-auth (expected, + * not gated); only the "no credential" crash being GONE proves the wiring and gates. + */ +const CREDENTIALS: { node: string; module: string; name: string; crash: string }[] = [ + { node: "ace", module: "plex", name: "token", crash: "no Plex token" }, + { node: "ace", module: "bazarr", name: "api-key", crash: "no Bazarr API key" }, + { node: "ace", module: "ombi", name: "api-key", crash: "no Ombi API key" }, + { node: "ace", module: "home-assistant", name: "token", crash: "no Home Assistant token" }, + { node: "ace", module: "nzbget", name: "password", crash: "NZBGet not configured" }, + { node: "ace", module: "qbittorrent", name: "password", crash: "qBittorrent not configured" }, + { node: "novox", module: "umami", name: "admin", crash: "admin password is not set" }, +]; + let instanceId = ""; let stocked: string[] = []; @@ -330,6 +366,31 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 } } + // Operator-provided app credentials (novox/hq dry-run fix). BEFORE the push, hand the mesh a FAKE + // value for each of the 7 credential modules through the real operator path — `secret accept`, + // which seals the value to the node and records it as `accepted` (the mesh will not invent one). + // The push then delivers it to the sidecar's own-secret path. The `--from` file is staged into the + // mesh-control container (one file per distinct secret name). A module the node could not host is + // skipped (its secret has nowhere to go). + const credentialDelivered = new Map(); + for (const name of new Set(CREDENTIALS.map((c) => c.name))) { + await must("anchor", `printf %s ${quote(`fake-${name}-value`)} > /tmp/fake-${name} && docker cp /tmp/fake-${name} mesh-control:/fake-${name}`); + } + for (const c of CREDENTIALS) { + if (!assigned[c.node]!.has(c.module)) { + credentialDelivered.set(`${c.node}/${c.module}`, false); + console.log(`CREDENTIAL SKIPPED ${c.node}/${c.module}: not assigned, nowhere to deliver`); + continue; + } + try { + await mesh(`secret accept ${c.node} ${c.module} ${c.name} --from /fake-${c.name}`); + credentialDelivered.set(`${c.node}/${c.module}`, true); + } catch (err) { + credentialDelivered.set(`${c.node}/${c.module}`, false); + console.log(`CREDENTIAL ACCEPT FAILED ${c.node}/${c.module}: ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`); + } + } + // ONE push per node. const pushError: Record = { novox: "", ace: "" }; for (const node of ["novox", "ace"]) { @@ -357,8 +418,10 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 // ================================================================================================ // Per-node report + gating. GREEN = each node's push accepted, every CORE module converged whole, - // and no NON-GAP resource failed to apply. The nine credential-sidecar gaps and firewall's oneshot - // are tolerated (documented + escalated in the per-server beds). + // no NON-GAP resource failed to apply, fail2ban is hostable (assigned, not refused), and every + // credential sidecar advanced past its "no credential" crash. Tolerated: the credential sidecars' + // app-auth failures (bogus fake value), photos/mailu, firewall's oneshot nftables.service, and + // fail2ban's package (the offline lab cannot fetch it — a documented host gap). // ================================================================================================ const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; const allProblems: string[] = []; @@ -405,6 +468,60 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`); } + // ================================================================================================ + // The dry-run fixes, proved by name. + // ================================================================================================ + + // fail2ban is now HOSTABLE (capability "firewall"): what the dry-run fix buys is that a node can + // host it at all. Before, it declared the never-detected "intrusion-prevention" capability, so NO + // node could host it AND its un-hostable assignment refused the whole node's push. So the GATE is + // hostability: it must be ASSIGNED and NOT refused. + // + // Its systemd service reaching active is a SEPARATE, host-level concern this offline lab cannot + // satisfy: the VM base image ships `nftables` (so firewall's package resolves and the `firewall` + // detector is advertised — which is exactly why fail2ban is now hostable) but NOT `fail2ban`, and + // the lab segment (RFC 5737 192.0.2.0/24) has no route to the package mirror, so pacman times out + // fetching fail2ban and its deps. That is a documented LAB gap (fail2ban ∈ GAPS_NOVOX, so its + // failed `fail2ban.package` resource is tolerated like firewall's oneshot nftables.service) — it is + // reported, not gated. On an online node the package installs and the service runs. + { + const refusedF2B = refused["novox"]!.find((r) => r.name === "fail2ban"); + const assignedF2B = assigned["novox"]!.has("fail2ban"); + const active = (await on("novox", `systemctl is-active fail2ban 2>&1`)).out.trim(); + const pkg = (await on("novox", `pacman -Q fail2ban 2>&1`)).out.trim(); + report.push(`\n---- fail2ban (novox): HOSTABLE assigned=${assignedF2B} refused=${refusedF2B ? "YES" : "no"} | service=${active} package="${pkg}" ----`); + if (refusedF2B) { + allProblems.push(`fail2ban still not hostable on novox: ${refusedF2B.why}`); + } else if (!assignedF2B) { + allProblems.push(`fail2ban was not assigned to novox`); + } + if (active !== "active") { + report.push(` service not active — offline lab could not install the package (documented gap, not gated); detail:`); + report.push(` ${(await on("novox", `systemctl status fail2ban --no-pager 2>&1 | head -8`)).out}`); + } + } + + // The 7 credential sidecars: each got its fake own-secret, so each must have advanced PAST the old + // "no credential" crash (it read the delivered value). It may still fail at app-auth against the + // real app with a bogus value — that is expected and does NOT gate; only the crash being gone does. + report.push(`\n---- credential sidecars: past the "no credential" crash? (fake secret delivered) ----`); + for (const c of CREDENTIALS) { + const container = `mesh-${c.module}`; + const psMap = psMaps[c.node]!; + const status = (psMap.get(container) ?? "MISSING").split(" ")[0] ?? "MISSING"; + const delivered = credentialDelivered.get(`${c.node}/${c.module}`) ?? false; + const logs = (await on(c.node, `docker logs ${container} 2>&1 | tail -60`)).out; + const stillCrashes = logs.includes(c.crash); + const appAuth = logs.split("\n").reverse().find((l) => /fail|reject|error|401|403|refused/i.test(l) && !l.includes(c.crash))?.trim().slice(0, 90) ?? ""; + report.push(` ${c.node}/${c.module.padEnd(15)} secret=${delivered ? "delivered" : "SKIPPED"} sidecar=${status.padEnd(10)} crash("${c.crash}")=${stillCrashes ? "STILL PRESENT" : "gone"}${appAuth ? ` last:"${appAuth}"` : ""}`); + if (delivered && stillCrashes) { + allProblems.push(`${c.node}/${c.module}: credential wiring did not take — sidecar still crashes "${c.crash}"`); + } + if (!delivered && assigned[c.node]!.has(c.module)) { + allProblems.push(`${c.node}/${c.module}: fake credential was not delivered (secret accept failed)`); + } + } + const summary = report.join("\n"); console.log(summary);