test/integration/whole-mesh-full.test.ts

@@ -3,8 +3,10 @@
  * whole-mesh rehearsal (novox/hq). Combines whole-mesh-novox.test.ts and whole-mesh-ace.test.ts.
  *
  *   anchor — substrate ONLY (store, broker, control).
- *   novox  — the 17-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu,
- *            firewall. fail2ban is dropped (no `intrusion-prevention` detector — see that bed).
+ *   novox  — the 18-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu,
+ *            firewall, fail2ban. fail2ban is now HOSTABLE: the dry-run fixes (mesh-control/catalog
+ *            main) changed its declared capability from the never-detected "intrusion-prevention" to
+ *            "firewall", the detector every node with nft already advertises.
  *   ace    — the 24-module ace set (whole-mesh-ace): the media/home stack; its /services/media
  *            library is pre-created so the ADR-0051 `accesses` resolve.
  *
@@ -13,10 +15,24 @@
  * the shared broker/store on anchor. The four modules both nodes run (postgres, redis, mssql,
  * portainer) are ADDED once and assigned to each node; each gets its own per-node broker account.
  *
- * This bed tolerates the SAME known gaps the per-server beds proved and escalated (nine
- * credential-sidecar crash-loops and firewall's oneshot nftables.service); it gates green on each
- * node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two node-plans
- * converge together on one substrate.
+ * THE DRY-RUN FIXES THIS RUN PROVES (mesh-control + mesh-catalog main):
+ *   - fail2ban is HOSTABLE (capability "firewall"): it is assigned, not refused. Before, it declared
+ *     the never-detected "intrusion-prevention" capability, so no node could host it and its
+ *     un-hostable assignment refused the whole node's push. Hostability is the gate. Its service
+ *     reaching active is a host concern this offline lab cannot meet — the VM ships nftables (so the
+ *     firewall detector is advertised) but not fail2ban, and the isolated segment has no route to the
+ *     package mirror, so pacman cannot fetch it. That is a documented lab gap, reported not gated.
+ *   - the 7 tool-runtime credential modules (ace: plex, bazarr, ombi, home-assistant, nzbget,
+ *     qbittorrent; novox: umami) now read their app credential from an operator-provided own-secret.
+ *     This bed delivers a FAKE value for each through the real operator path (`secret accept`)
+ *     BEFORE the push, and gates on the sidecar getting PAST its old "no credential" crash (it reads
+ *     the delivered value). A fake value will not authenticate against the real app — the sidecar may
+ *     still fail at app-auth, which is expected and does NOT gate; only the crash being GONE gates.
+ *
+ * It otherwise tolerates the SAME known gaps the per-server beds proved and escalated (the credential
+ * sidecars' app-auth failures, photos/mailu, and firewall's oneshot nftables.service); it gates green
+ * on each node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two
+ * node-plans converge together on one substrate.
  *
  *   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
  */
@@ -83,13 +99,14 @@ const NOVOX: Mod[] = [
     ],
   },
   { name: "firewall", containers: [], node: true },
+  { name: "fail2ban", containers: [], node: true },
 ];
 const CORE_NOVOX = new Set([
   "postgres", "redis", "minio", "mongodb", "mssql",
   "keycloak", "gitea", "nextcloud", "invoicing",
   "portainer", "verdaccio", "registry", "route-proxy",
 ]);
-const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall"]);
+const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall", "fail2ban"]);
 
 /** The ace node's 24-module set. */
 const ACE: Mod[] = [
@@ -143,6 +160,25 @@ const REMAP: Record<string, Record<string, string>> = {
   nzbget: { "6789": "6790:6789" },
 };
 
+/**
+ * The 7 tool-runtime credential modules (novox/hq dry-run fix). Each now reads its app credential
+ * from an operator-provided own-secret (`name`, an own-secret path in its module.json), mounted into
+ * the sidecar at MESH_*_FILE. This bed delivers a FAKE value for each via the real operator path
+ * (`secret accept <node> <module> <name> --from <file>`) BEFORE the push, and asserts the sidecar
+ * gets PAST `crash` — the exact message its client threw when nothing was mounted. A fake value does
+ * not authenticate against the real app, so the sidecar may still fail later at app-auth (expected,
+ * not gated); only the "no credential" crash being GONE proves the wiring and gates.
+ */
+const CREDENTIALS: { node: string; module: string; name: string; crash: string }[] = [
+  { node: "ace", module: "plex", name: "token", crash: "no Plex token" },
+  { node: "ace", module: "bazarr", name: "api-key", crash: "no Bazarr API key" },
+  { node: "ace", module: "ombi", name: "api-key", crash: "no Ombi API key" },
+  { node: "ace", module: "home-assistant", name: "token", crash: "no Home Assistant token" },
+  { node: "ace", module: "nzbget", name: "password", crash: "NZBGet not configured" },
+  { node: "ace", module: "qbittorrent", name: "password", crash: "qBittorrent not configured" },
+  { node: "novox", module: "umami", name: "admin", crash: "admin password is not set" },
+];
+
 let instanceId = "";
 let stocked: string[] = [];
 
@@ -330,6 +366,31 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6
     }
   }
 
+  // Operator-provided app credentials (novox/hq dry-run fix). BEFORE the push, hand the mesh a FAKE
+  // value for each of the 7 credential modules through the real operator path — `secret accept`,
+  // which seals the value to the node and records it as `accepted` (the mesh will not invent one).
+  // The push then delivers it to the sidecar's own-secret path. The `--from` file is staged into the
+  // mesh-control container (one file per distinct secret name). A module the node could not host is
+  // skipped (its secret has nowhere to go).
+  const credentialDelivered = new Map<string, boolean>();
+  for (const name of new Set(CREDENTIALS.map((c) => c.name))) {
+    await must("anchor", `printf %s ${quote(`fake-${name}-value`)} > /tmp/fake-${name} && docker cp /tmp/fake-${name} mesh-control:/fake-${name}`);
+  }
+  for (const c of CREDENTIALS) {
+    if (!assigned[c.node]!.has(c.module)) {
+      credentialDelivered.set(`${c.node}/${c.module}`, false);
+      console.log(`CREDENTIAL SKIPPED ${c.node}/${c.module}: not assigned, nowhere to deliver`);
+      continue;
+    }
+    try {
+      await mesh(`secret accept ${c.node} ${c.module} ${c.name} --from /fake-${c.name}`);
+      credentialDelivered.set(`${c.node}/${c.module}`, true);
+    } catch (err) {
+      credentialDelivered.set(`${c.node}/${c.module}`, false);
+      console.log(`CREDENTIAL ACCEPT FAILED ${c.node}/${c.module}: ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`);
+    }
+  }
+
   // ONE push per node.
   const pushError: Record<string, string> = { novox: "", ace: "" };
   for (const node of ["novox", "ace"]) {
@@ -357,8 +418,10 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6
 
   // ================================================================================================
   // Per-node report + gating. GREEN = each node's push accepted, every CORE module converged whole,
-  // and no NON-GAP resource failed to apply. The nine credential-sidecar gaps and firewall's oneshot
-  // are tolerated (documented + escalated in the per-server beds).
+  // no NON-GAP resource failed to apply, fail2ban is hostable (assigned, not refused), and every
+  // credential sidecar advanced past its "no credential" crash. Tolerated: the credential sidecars'
+  // app-auth failures (bogus fake value), photos/mailu, firewall's oneshot nftables.service, and
+  // fail2ban's package (the offline lab cannot fetch it — a documented host gap).
   // ================================================================================================
   const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out;
   const allProblems: string[] = [];
@@ -405,6 +468,60 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6
     if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`);
   }
 
+  // ================================================================================================
+  // The dry-run fixes, proved by name.
+  // ================================================================================================
+
+  // fail2ban is now HOSTABLE (capability "firewall"): what the dry-run fix buys is that a node can
+  // host it at all. Before, it declared the never-detected "intrusion-prevention" capability, so NO
+  // node could host it AND its un-hostable assignment refused the whole node's push. So the GATE is
+  // hostability: it must be ASSIGNED and NOT refused.
+  //
+  // Its systemd service reaching active is a SEPARATE, host-level concern this offline lab cannot
+  // satisfy: the VM base image ships `nftables` (so firewall's package resolves and the `firewall`
+  // detector is advertised — which is exactly why fail2ban is now hostable) but NOT `fail2ban`, and
+  // the lab segment (RFC 5737 192.0.2.0/24) has no route to the package mirror, so pacman times out
+  // fetching fail2ban and its deps. That is a documented LAB gap (fail2ban ∈ GAPS_NOVOX, so its
+  // failed `fail2ban.package` resource is tolerated like firewall's oneshot nftables.service) — it is
+  // reported, not gated. On an online node the package installs and the service runs.
+  {
+    const refusedF2B = refused["novox"]!.find((r) => r.name === "fail2ban");
+    const assignedF2B = assigned["novox"]!.has("fail2ban");
+    const active = (await on("novox", `systemctl is-active fail2ban 2>&1`)).out.trim();
+    const pkg = (await on("novox", `pacman -Q fail2ban 2>&1`)).out.trim();
+    report.push(`\n---- fail2ban (novox): HOSTABLE assigned=${assignedF2B} refused=${refusedF2B ? "YES" : "no"} | service=${active} package="${pkg}" ----`);
+    if (refusedF2B) {
+      allProblems.push(`fail2ban still not hostable on novox: ${refusedF2B.why}`);
+    } else if (!assignedF2B) {
+      allProblems.push(`fail2ban was not assigned to novox`);
+    }
+    if (active !== "active") {
+      report.push(`  service not active — offline lab could not install the package (documented gap, not gated); detail:`);
+      report.push(`  ${(await on("novox", `systemctl status fail2ban --no-pager 2>&1 | head -8`)).out}`);
+    }
+  }
+
+  // The 7 credential sidecars: each got its fake own-secret, so each must have advanced PAST the old
+  // "no credential" crash (it read the delivered value). It may still fail at app-auth against the
+  // real app with a bogus value — that is expected and does NOT gate; only the crash being gone does.
+  report.push(`\n---- credential sidecars: past the "no credential" crash? (fake secret delivered) ----`);
+  for (const c of CREDENTIALS) {
+    const container = `mesh-${c.module}`;
+    const psMap = psMaps[c.node]!;
+    const status = (psMap.get(container) ?? "MISSING").split(" ")[0] ?? "MISSING";
+    const delivered = credentialDelivered.get(`${c.node}/${c.module}`) ?? false;
+    const logs = (await on(c.node, `docker logs ${container} 2>&1 | tail -60`)).out;
+    const stillCrashes = logs.includes(c.crash);
+    const appAuth = logs.split("\n").reverse().find((l) => /fail|reject|error|401|403|refused/i.test(l) && !l.includes(c.crash))?.trim().slice(0, 90) ?? "";
+    report.push(`  ${c.node}/${c.module.padEnd(15)} secret=${delivered ? "delivered" : "SKIPPED"} sidecar=${status.padEnd(10)} crash("${c.crash}")=${stillCrashes ? "STILL PRESENT" : "gone"}${appAuth ? `  last:"${appAuth}"` : ""}`);
+    if (delivered && stillCrashes) {
+      allProblems.push(`${c.node}/${c.module}: credential wiring did not take — sidecar still crashes "${c.crash}"`);
+    }
+    if (!delivered && assigned[c.node]!.has(c.module)) {
+      allProblems.push(`${c.node}/${c.module}: fake credential was not delivered (secret accept failed)`);
+    }
+  }
+
   const summary = report.join("\n");
   console.log(summary);