mesh-lab/test/integration/whole-mesh-full.test.ts

/**
 * The FULL mesh in its REAL production shape: two segments, one access point, one overlay — and the
 * first multi-segment whole-mesh bed. It rewrites the flat three-node whole-mesh-full (separate
 * anchor, everything on one public segment) into what production actually is:
 *
 *   hosting (public)                  home (private, behind a NAT access point)
 *     novox 192.0.2.20 — the ANCHOR:    ace    192.168.1.10  the home server, media/IoT set
 *       substrate (store/broker/        shanks 192.168.1.20  workstation (light: portainer only)
 *       control) + the whole novox      g14    192.168.1.30  workstation (light: portainer only)
 *       set + overlay hub + ingress
 *
 * There is NO separate anchor: novox IS the anchor. The substrate runs on novox, and novox also
 * enrols as a node and receives its own service set — the substrate host and a service node at once.
 *
 * THE THING THIS BED EXISTS TO PROVE (the flat beds never could): does the WireGuard overlay tunnel
 * FORM across the access point? A home node (ace/shanks/g14) dials novox's PUBLIC hub endpoint
 * 192.0.2.20:51820/udp OUT through the household gateway's masquerade; the handshake has to complete
 * through that NAT and the keepalive has to hold the hole open. Phase A drives exactly this and
 * verifies it — WireGuard handshake state AND a ping over the overlay from a home node to novox —
 * BEFORE any heavy module lands, so the cross-segment-overlay verdict survives whatever the module
 * convergence then does. Phase B converges the full node sets and reports per node.
 *
 * SUBSTRATE-ON-NOVOX PORT COLLISIONS (a real consequence of collapsing the anchor onto novox that the
 * separate-anchor beds never hit): the substrate store binds 127.0.0.1:5432 and novox's postgres
 * provider publishes 5432; the substrate broker binds 5671 + 127.0.0.1:5672 and novox's lavinmq
 * provider publishes 5672. The two provider host publishes are REMAPPED off the substrate's ports
 * (REMAP below); consumers reach the providers over the mesh network on the container port, so the
 * host side is free to move. Reported as a topology finding.
 *
 * PERSISTENT RAISE. With MESH_LAB_KEEP set the instance is raised under a fixed id
 * (whole-mesh-full-live) and NOT torn down — it is left standing and browsable. Without it the bed
 * behaves like every other: raise in before(), destroy in after().
 *
 *   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
 */

import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";

const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";

const skip = !capability.usable
  ? `lab not usable: ${capability.why}`
  : !binary || !existsSync(binary)
    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
    : !bundle || !existsSync(bundle)
      ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)"
      : false;

const SCENARIO = "whole-mesh-full";
/** novox hosts the substrate and the control plane; it is where `mesh` commands run. */
const CONTROL = "novox";
/** Every node that enrols. novox is on hosting; the rest are behind the home gateway. */
const NODES = ["novox", "ace", "shanks", "g14"];
const HOME_NODES = ["ace", "shanks", "g14"];

/** Keep the instance standing and browsable rather than tearing it down. */
const KEEP = !!process.env["MESH_LAB_KEEP"];
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined);

const catalogDir = process.env["MESH_LAB_CATALOG"]
  ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
  ?? resolve(process.cwd(), "..", "mesh-catalog", "modules");

const MEDIA_DIRS = [
  "/services/media/series", "/services/media/anime", "/services/media/movies",
  "/services/media/music", "/services/media/audiobooks", "/services/media/downloads",
  "/services/media/books",
];

type Mod = { name: string; containers: string[]; node?: boolean; runOnce?: string[] };

/**
 * The novox set (feat/novox-conversions @ 431310f). The slug fix means only-office/de-spiegel/
 * amqp-email-forwarder now resolve (their minted login was over the 20-char cap before), so they
 * are INCLUDED. CORE gates; the rest are reported gaps (documented in the whole-mesh-novox bed):
 * umami (provisioner url/admin unset), mailu (nox-schema gaps), only-office/de-spiegel (new plain
 * apps, boot secondary), amqp-email-forwarder (hard-coded AMQP vhost authz), firewall/fail2ban
 * (offline lab cannot fetch the package).
 */
const NOVOX: Mod[] = [
  { name: "postgres", containers: ["postgres", "mesh-postgres"] },
  { name: "redis", containers: ["redis", "mesh-redis"] },
  { name: "minio", containers: ["minio", "mesh-minio"] },
  { name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
  { name: "mssql", containers: ["mssql", "mesh-mssql"] },
  { name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] },
  { name: "route-proxy", containers: ["route-proxy"] },
  { name: "keycloak", containers: ["keycloak", "mesh-keycloak"] },
  { name: "gitea", containers: ["gitea", "mesh-gitea"] },
  { name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] },
  { name: "umami", containers: ["umami", "mesh-umami"] },
  { name: "photos", containers: ["photos-server", "photos-admin-client", "photos-client-eef", "photos-client-filip"] },
  { name: "invoicing", containers: ["invoicing-app", "invoicing-api"] },
  { name: "novox.be", containers: ["novox-be"] },
  { name: "only-office", containers: ["office-novox-be"] },
  { name: "de-spiegel", containers: ["de-spiegel-novox-be"] },
  { name: "amqp-email-forwarder", containers: ["amqp-email-forwarder"] },
  { name: "portainer", containers: ["portainer", "mesh-portainer"] },
  { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] },
  { name: "registry", containers: ["mesh-registry"] },
  {
    name: "mailu",
    containers: [
      "mailu-resolver", "mailu-redis", "mailu-admin", "mailu-imap", "mailu-smtp",
      "mailu-antispam", "mailu-antivirus", "mailu-webmail", "mailu-webdav", "mailu-fetchmail",
      "mailu-front", "mesh-mailu",
    ],
  },
  { name: "firewall", containers: [], node: true },
  { name: "fail2ban", containers: [], node: true },
];
const CORE_NOVOX = new Set([
  "postgres", "redis", "minio", "mongodb", "mssql", "lavinmq",
  "route-proxy", "keycloak", "gitea", "nextcloud", "invoicing", "photos", "novox.be",
  "portainer", "verdaccio", "registry",
]);
const GAPS_NOVOX = new Set([
  "umami", "mailu", "firewall", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder",
]);

/** The ace media/home set. */
const ACE: Mod[] = [
  { name: "postgres", containers: ["postgres", "mesh-postgres"] },
  { name: "redis", containers: ["redis", "mesh-redis"] },
  { name: "mssql", containers: ["mssql", "mesh-mssql"] },
  { name: "sonarr", containers: ["sonarr", "mesh-sonarr"] },
  { name: "radarr", containers: ["radarr", "mesh-radarr"] },
  { name: "lidarr", containers: ["lidarr", "mesh-lidarr"] },
  { name: "plex", containers: ["plex", "mesh-plex"] },
  { name: "bazarr", containers: ["bazarr", "mesh-bazarr"] },
  { name: "nzbget", containers: ["nzbget", "mesh-nzbget"] },
  { name: "qbittorrent", containers: ["qbittorrent", "mesh-qbittorrent"] },
  { name: "jackett", containers: ["jackett", "mesh-jackett"] },
  { name: "ombi", containers: ["ombi", "mesh-ombi"] },
  { name: "tautulli", containers: ["tautulli", "mesh-tautulli"] },
  { name: "bookshelf", containers: ["bookshelf", "mesh-bookshelf"] },
  { name: "home-assistant", containers: ["home-assistant", "mesh-home-assistant"] },
  { name: "mosquitto", containers: ["mosquitto", "mesh-mosquitto"], runOnce: ["mosquitto-bootstrap"] },
  { name: "influxdb", containers: ["influxdb", "mesh-influxdb"] },
  { name: "grafana", containers: ["grafana", "mesh-grafana"] },
  { name: "baserow", containers: ["baserow", "mesh-baserow"] },
  { name: "letta", containers: ["letta", "mesh-letta"] },
  { name: "nodered", containers: ["nodered", "mesh-nodered"] },
  { name: "searxng", containers: ["valkey", "searxng", "mesh-searxng"] },
  { name: "unifi", containers: ["unifi-controller", "mesh-unifi"] },
  { name: "portainer", containers: ["portainer", "mesh-portainer"] },
];
const CORE_ACE = new Set([
  "postgres", "redis", "mssql",
  "sonarr", "radarr", "lidarr", "jackett", "tautulli", "bookshelf",
  "mosquitto", "influxdb", "grafana", "baserow", "nodered", "searxng", "unifi", "portainer",
]);
const GAPS_ACE = new Set(["plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta"]);

/** The two workstations run one light module each, to prove a real module converges and joins the overlay. */
const LIGHT: Mod[] = [{ name: "portainer", containers: ["portainer", "mesh-portainer"] }];
const CORE_LIGHT = new Set(["portainer"]);
const GAPS_LIGHT = new Set<string>();

const PLAN: { node: string; mods: Mod[]; core: Set<string>; gaps: Set<string> }[] = [
  { node: "novox", mods: NOVOX, core: CORE_NOVOX, gaps: GAPS_NOVOX },
  { node: "ace", mods: ACE, core: CORE_ACE, gaps: GAPS_ACE },
  { node: "shanks", mods: LIGHT, core: CORE_LIGHT, gaps: GAPS_LIGHT },
  { node: "g14", mods: LIGHT, core: CORE_LIGHT, gaps: GAPS_LIGHT },
];

/**
 * Host-port remaps (per module; host ports are per-VM so novox's and ace's never clash across nodes).
 * The two SUBSTRATE collisions are the new ones: postgres 5432 and lavinmq 5672 are moved off the
 * substrate store/broker's host ports, which only exist on novox because that is where the substrate
 * runs. The rest break the novox web/app host-port collisions (route-proxy fronts 80/443).
 */
const REMAP: Record<string, Record<string, string>> = {
  postgres: { "5432": "127.0.0.1:15432:5432" },
  lavinmq: { "5672": "127.0.0.1:15673:5672" },
  nextcloud: { "80": "8090:80" },
  umami: { "3000": "3090:3000" },
  invoicing: { "80": "8091:80", "9000": "9091:9000" },
  qbittorrent: { "8080": "8090:8080" },
  searxng: { "8080": "8092:8080" },
  nzbget: { "6789": "6790:6789" },
};

/** Operator-provided app credentials, delivered as fake values through the real `secret accept` path. */
const CREDENTIALS: { node: string; module: string; name: string; crash: string }[] = [
  { node: "ace", module: "plex", name: "token", crash: "no Plex token" },
  { node: "ace", module: "bazarr", name: "api-key", crash: "no Bazarr API key" },
  { node: "ace", module: "ombi", name: "api-key", crash: "no Ombi API key" },
  { node: "ace", module: "home-assistant", name: "token", crash: "no Home Assistant token" },
  { node: "ace", module: "nzbget", name: "password", crash: "NZBGet not configured" },
  { node: "ace", module: "qbittorrent", name: "password", crash: "qBittorrent not configured" },
  { node: "novox", module: "umami", name: "admin", crash: "admin password is not set" },
];

/** Operator secrets for the credential modules that own-secret their whole app (mailu, de-spiegel). */
const OPERATOR_SECRETS: { node: string; module: string; name: string; value: string }[] = [
  { node: "novox", module: "mailu", name: "secret-key", value: "0123456789abcdef0123456789abcdef" },
  { node: "novox", module: "mailu", name: "admin", value: "MailuAdminFakePass123" },
  { node: "novox", module: "mailu", name: "api-token", value: "mailuapitokenfake0123456789abcd" },
  { node: "novox", module: "de-spiegel", name: "smtp-user", value: "despiegel-smtp-fake" },
  { node: "novox", module: "de-spiegel", name: "smtp-pass", value: "despiegel-pass-fake" },
  { node: "novox", module: "amqp-email-forwarder", name: "smtp-user", value: "eef-smtp-fake" },
  { node: "novox", module: "amqp-email-forwarder", name: "smtp-pass", value: "eef-pass-fake" },
];

let instanceId = "";
let stocked: string[] = [];

function quote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
  const { stdout } = await exec(instanceId, machine, [
    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
  ], timeoutMs);
  const marker = stdout.lastIndexOf("__exit=");
  if (marker < 0) return { out: stdout, ok: false };
  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}

async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
  const { out, ok } = await on(machine, command, timeoutMs);
  if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
  return out;
}

/** The control plane, a container on novox (the anchor). */
async function mesh(command: string, timeoutMs?: number): Promise<string> {
  return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}

function repositoryFor(reference: string): string {
  const withoutDigest = reference.split("@")[0] ?? reference;
  const lastColon = withoutDigest.lastIndexOf(":");
  const lastSlash = withoutDigest.lastIndexOf("/");
  return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest;
}

function pinned(repository: string): string {
  const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
  assert.ok(found, `the scenario stocks no ${repository}; it serves\n  ${stocked.join("\n  ")}`);
  return found;
}

function bundleFor(images: string[]): string {
  let text = readFileSync(bundle, "utf8");
  for (const ref of images) {
    const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
    const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
    text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
  }
  return text;
}

function loadManifest(name: string): { manifest: string; broker: boolean } {
  const path = resolve(catalogDir, name, "module.json");
  const m = JSON.parse(readFileSync(path, "utf8")) as {
    resources?: { type: string; image?: string; ports?: string[] }[];
  };
  const remap = REMAP[name] ?? {};
  for (const r of m.resources ?? []) {
    if (r.type !== "container") continue;
    if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image));
    if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
  }
  const manifest = JSON.stringify(m);
  return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
}

function tokenFrom(said: string): string {
  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
  assert.ok(found, `no token in:\n${said}`);
  return found;
}

interface NodeState {
  reached: boolean;
  applied: boolean;
  current: boolean;
  waiting: boolean;
  wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined;
  raw: string;
}

async function nodeState(node: string): Promise<NodeState> {
  const asked = await on(CONTROL, `docker exec mesh-control /mesh-control status --json`);
  if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out };
  let state: {
    wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[];
    waiting: { node: string }[];
    reported: { node: string; outcome: string; current: boolean }[];
  };
  try {
    state = JSON.parse(asked.out);
  } catch {
    return { reached: false, applied: false, current: false, waiting: false, raw: asked.out };
  }
  const word = state.reported.find((r) => r.node === node);
  const bad = state.wrong.find((w) => w.node === node);
  return {
    reached: true,
    applied: word?.outcome === "applied",
    current: !!word?.current,
    waiting: state.waiting.some((w) => w.node === node),
    wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined,
    raw: asked.out,
  };
}

async function psMapOf(node: string): Promise<Map<string, string>> {
  const out = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
  const map = new Map<string, string>();
  for (const line of out.split("\n")) {
    const [n, ...rest] = line.split("\t");
    if (n) map.set(n.trim(), rest.join("\t").trim());
  }
  return map;
}

/** A node's overlay (mesh0) address, or "" if it has none yet. */
async function overlayAddr(node: string): Promise<string> {
  const out = (await on(node, `ip -4 -o addr show mesh0 2>/dev/null | awk '{print $4}' | cut -d/ -f1`)).out;
  return out.split("\n").map((l) => l.trim()).find(Boolean) ?? "";
}

before(async () => {
  if (skip) return;
  assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);

  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
    onProgress: (m) => console.log(`raise: ${m}`),
    ...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
  });
  instanceId = raised.instanceId;
  stocked = raised.images;
  console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);

  // novox raises the substrate from its bundle, digests rewritten to the scenario registry's. This
  // is the collapse: the substrate rides novox, not a separate anchor. The bundle hardcodes the
  // broker's advertised address as 192.0.2.10:5671 (the OLD separate-anchor address) — and a token
  // carries MESH_BROKER_ADDRESS verbatim as the endpoint an enrolling node dials. With the substrate
  // on novox that endpoint must be novox's own public address, or every node (novox included) would
  // enrol against a dead address. The broker serves its cert on all interfaces and the token pins by
  // fingerprint, not hostname, so only the address needs correcting.
  const bundleText = bundleFor(raised.images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671");
  await must(CONTROL, `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleText}\nMESHBUNDLE`);
  await must(CONTROL, `${HOST_PATH} apply /tmp/substrate.lock`, 900_000);
  const up = await must(CONTROL, `docker ps --format '{{.Names}}'`);
  for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) {
    assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`);
  }

  // Every node joins the one mesh and runs a host. The home nodes reach novox's public 192.0.2.20:5671
  // by dialling OUT through the household gateway — the enrol itself is the first proof that outbound
  // home→public works. novox enrols too: substrate host and service node at once.
  for (const machine of NODES) {
    await mesh(`node add ${machine}`);
    const token = tokenFrom(await mesh(`token issue --node ${machine}`));
    const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
    assert.match(said, new RegExp(`enrolled as ${machine}`), said);
    await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
  }

  // The operator provides ace's media library (ADR 0051 accesses confirm the paths, create nothing).
  await must("ace", `mkdir -p ${MEDIA_DIRS.join(" ")}`);
}, { timeout: 3_600_000 });

after(async () => {
  if (KEEP) {
    console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
    return;
  }
  if (instanceId) await destroy(instanceId);
  await destroyAll(`${SCENARIO}-`);
}, { timeout: 900_000 });

test("the full mesh forms across the access point and both server sets converge", {
  skip, timeout: 5_400_000,
}, async () => {
  // ================================================================================================
  // PHASE A — THE HEADLINE. Place the overlay (hub on novox at its public endpoint; the home nodes
  // dial out, no endpoint of their own), assign networking to every node, push, and VERIFY the tunnel
  // forms ACROSS the gateway. This runs BEFORE any heavy module, so the cross-segment-overlay verdict
  // is captured whatever the module convergence then does.
  // ================================================================================================
  await mesh("overlay place novox --hub --endpoint 192.0.2.20:51820 --site hosting");
  for (const node of HOME_NODES) await mesh(`overlay place ${node} --site home`);
  for (const node of NODES) await mesh(`assign ${node} networking`);
  for (const node of NODES) {
    try {
      await mesh(`push ${node}`, 180_000);
    } catch (err) {
      console.log(`networking push rejected (${node}): ${(err as Error).message.split("\n").slice(0, 4).join(" | ")}`);
    }
  }

  // Give the home nodes time to dial the hub and complete a handshake through the NAT.
  const overlay: Record<string, string> = {};
  const deadline = Date.now() + 300_000;
  while (Date.now() < deadline) {
    for (const node of NODES) if (!overlay[node]) overlay[node] = await overlayAddr(node);
    if (NODES.every((n) => overlay[n])) break;
    await new Promise((r) => setTimeout(r, 8000));
  }
  // A little longer for handshakes to settle (keepalive interval).
  await new Promise((r) => setTimeout(r, 30000));

  const overlayReport: string[] = ["================ CROSS-SEGMENT OVERLAY (the headline) ================"];
  for (const node of NODES) overlayReport.push(`  ${node.padEnd(8)} mesh0 = ${overlay[node] || "NONE"}`);

  // The hub's WireGuard peers and their handshakes, from novox.
  const hubWg = (await on("novox", `wg show 2>&1 || echo 'wg tool absent'`)).out;
  overlayReport.push(`\n---- novox (hub) wg show ----\n${hubWg}`);

  // From each home node: its wg peer state (endpoint should be 192.0.2.20:51820, with a recent
  // handshake) AND a ping to novox's overlay address — the functional proof the tunnel carries
  // traffic across the gateway.
  const overlayFormed: Record<string, boolean> = {};
  const novoxOverlay = overlay["novox"] ?? "";
  for (const node of HOME_NODES) {
    const wg = (await on(node, `wg show 2>&1 || echo 'wg tool absent'`)).out;
    const handshake = (await on(node, `wg show all latest-handshakes 2>/dev/null | awk '{print $2}' | sort -rn | head -1`)).out.trim();
    const ping = novoxOverlay
      ? await on(node, `ping -c 3 -W 2 ${novoxOverlay} 2>&1 | tail -3`)
      : { out: "novox has no overlay address to ping", ok: false };
    const handshakeSecs = Number(handshake) || 0;
    // Formed = we can reach novox over the overlay from this home node (traffic across the NAT).
    overlayFormed[node] = ping.ok;
    overlayReport.push(`\n---- ${node} (home) ----`);
    overlayReport.push(wg.split("\n").map((l) => `  ${l}`).join("\n"));
    overlayReport.push(`  latest-handshake epoch: ${handshake || "none"}${handshakeSecs ? "" : " (no handshake recorded)"}`);
    overlayReport.push(`  ping novox(${novoxOverlay}) over overlay: ${ping.ok ? "REPLIES" : "NO REPLY"}`);
    overlayReport.push(ping.out.split("\n").map((l) => `    ${l}`).join("\n"));
  }
  const anyHomeFormed = HOME_NODES.some((n) => overlayFormed[n]);
  const allHomeFormed = HOME_NODES.every((n) => overlayFormed[n]);
  overlayReport.push(`\nVERDICT: overlay across the access point ${allHomeFormed ? "FORMED for all home nodes" : anyHomeFormed ? "FORMED for some home nodes" : "DID NOT FORM"}.`);
  const overlaySummary = overlayReport.join("\n");
  console.log(overlaySummary);

  // ================================================================================================
  // PHASE B — converge the full node sets on top of the overlay.
  // ================================================================================================
  const added = new Map<string, boolean>();
  async function ensureAdded(name: string): Promise<boolean> {
    const known = added.get(name);
    if (known !== undefined) return known;
    const { manifest, broker } = loadManifest(name);
    await must(CONTROL, `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`);
    await mesh(`module add /${name}.json`);
    added.set(name, broker);
    return broker;
  }

  const assigned: Record<string, Set<string>> = { novox: new Set(), ace: new Set(), shanks: new Set(), g14: new Set() };
  const refused: Record<string, { name: string; why: string }[]> = { novox: [], ace: [], shanks: [], g14: [] };
  for (const { node, mods } of PLAN) {
    for (const { name } of mods) {
      try {
        const broker = await ensureAdded(name);
        if (broker) await mesh(`module issue ${name} --node ${node}`);
        await mesh(`assign ${node} ${name}`);
        assigned[node]!.add(name);
      } catch (err) {
        const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | ");
        refused[node]!.push({ name, why });
        console.log(`NOT ASSIGNED ${node}/${name}: ${why}`);
      }
    }
  }

  // Operator-provided app credentials (own-secrets), delivered as fake values through `secret accept`.
  const credentialDelivered = new Map<string, boolean>();
  for (const name of new Set(CREDENTIALS.map((c) => c.name))) {
    await must(CONTROL, `printf %s ${quote(`fake-${name}-value`)} > /tmp/fake-${name} && docker cp /tmp/fake-${name} mesh-control:/fake-${name}`);
  }
  for (const c of CREDENTIALS) {
    if (!assigned[c.node]!.has(c.module)) {
      credentialDelivered.set(`${c.node}/${c.module}`, false);
      continue;
    }
    try {
      await mesh(`secret accept ${c.node} ${c.module} ${c.name} --from /fake-${c.name}`);
      credentialDelivered.set(`${c.node}/${c.module}`, true);
    } catch (err) {
      credentialDelivered.set(`${c.node}/${c.module}`, false);
      console.log(`CREDENTIAL ACCEPT FAILED ${c.node}/${c.module}: ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`);
    }
  }
  // Whole-app own-secrets (mailu/de-spiegel/amqp-email-forwarder).
  for (const s of OPERATOR_SECRETS) {
    if (!assigned[s.node]!.has(s.module)) continue;
    try {
      const inControl = `/secret-${s.module}-${s.name}`;
      await must(CONTROL, `printf %s ${quote(s.value)} > /tmp${inControl} && docker cp /tmp${inControl} mesh-control:${inControl}`);
      await mesh(`secret accept ${s.node} ${s.module} ${s.name} --from ${inControl}`);
    } catch (err) {
      console.log(`OPERATOR SECRET FAILED ${s.node}/${s.module}/${s.name}: ${(err as Error).message.split("\n").slice(0, 2).join(" | ")}`);
    }
  }

  // ONE push per node (workstations first — cheap — then the heavy service nodes).
  const pushError: Record<string, string> = {};
  for (const node of ["shanks", "g14", "novox", "ace"]) {
    try {
      await mesh(`push ${node}`, 300_000);
    } catch (err) {
      pushError[node] = (err as Error).message;
      console.log(`PUSH REJECTED (${node}):\n${pushError[node]!.split("\n").slice(0, 6).join("\n")}`);
    }
  }

  // Wait for each node's CORE containers to come up (all nodes pull concurrently from the one registry).
  const psMaps: Record<string, Map<string, string>> = { novox: new Map(), ace: new Map(), shanks: new Map(), g14: new Map() };
  for (const { node, mods, core } of PLAN) {
    if (pushError[node]) continue;
    const coreContainers = mods.filter((m) => core.has(m.name) && assigned[node]!.has(m.name)).flatMap((m) => m.containers);
    const until = Date.now() + 3_000_000;
    while (Date.now() < until) {
      psMaps[node] = await psMapOf(node);
      if (coreContainers.every((c) => (psMaps[node]!.get(c) ?? "").startsWith("Up"))) break;
      await new Promise((r) => setTimeout(r, 10000));
    }
  }
  await new Promise((r) => setTimeout(r, 20000));

  // ================================================================================================
  // Per-node convergence report.
  // ================================================================================================
  const users = (await on("novox", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out;
  const allProblems: string[] = [];
  const report: string[] = ["================ FULL MESH CONVERGENCE ================"];

  for (const { node, mods, core, gaps } of PLAN) {
    const psMap = psMaps[node] = await psMapOf(node);
    const st = await nodeState(node);
    const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up");
    const ranOnce = (name: string): boolean => !psMap.has(name) || /^(Up|Exited \(0\))/.test(psMap.get(name) ?? "");
    const failedResources = st.wrong?.failed ?? [];

    report.push(`\n---- node ${node}: reached=${st.reached} applied=${st.applied} current=${st.current} waiting=${st.waiting} ----`);
    if (pushError[node]) report.push(`  PUSH REJECTED: ${pushError[node]!.split("\n").slice(0, 6).join("\n    ")}`);
    if (st.wrong) {
      report.push(`  NODE WRONG: outcome=${st.wrong.outcome}`);
      for (const f of failedResources) report.push(`    failed ${f.id}: ${f.error}`);
    }
    for (const r of refused[node]!) report.push(`  REFUSED ${r.name}: ${r.why}`);

    const coreFailures: string[] = [];
    for (const mod of mods) {
      if (!assigned[node]!.has(mod.name)) continue;
      if (mod.node) {
        report.push(`  ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(20)} ${core.has(mod.name) ? "CORE" : "gap "} node-service`);
        continue;
      }
      const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`);
      const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce);
      const tag = core.has(mod.name) ? (ok ? "OK  " : "FAIL") : (ok ? "ok  " : "GAP ");
      report.push(`  ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(20)} ${tag} ${states.join("  ")}`);
      if (core.has(mod.name) && !ok) coreFailures.push(mod.name);
    }
    const issuedHere = mods.filter((m) => new RegExp(`${node}-${m.name}\\b`).test(users)).length;
    report.push(`  broker accounts: ${issuedHere} present for ${node}`);

    const gapOwnerOf = (f: { id: string; error: string }): string => {
      const m = f.error.match(/applying "([^".]+)\./);
      return m?.[1] ?? (f.id.split(".")[0] ?? "");
    };
    if (pushError[node]) allProblems.push(`${node}: push rejected`);
    if (coreFailures.length) allProblems.push(`${node}: CORE not converged: ${coreFailures.join(", ")}`);
    const nonGapFailed = failedResources.filter((f) => !gaps.has(gapOwnerOf(f)));
    if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`);
  }

  const summary = report.join("\n");
  console.log(summary);

  // Diagnostics for any CORE container that did not come up.
  for (const { node, mods, core } of PLAN) {
    const psMap = psMaps[node]!;
    for (const mod of mods) {
      if (!core.has(mod.name) || !assigned[node]!.has(mod.name)) continue;
      for (const c of mod.containers) {
        if (psMap.has(c) && !(psMap.get(c) ?? "").startsWith("Up")) {
          console.log(`\n---- ${node} logs: ${c} (${psMap.get(c)}) ----\n${(await on(node, `docker logs ${c} 2>&1 | tail -25`)).out}`);
        }
      }
    }
  }

  // ================================================================================================
  // GATING. The headline gates: the cross-segment overlay must FORM for at least one home node
  // (that is the thing this bed exists to prove). Convergence gates on each node's CORE and no
  // non-gap resource failing. The KEEP run is about leaving a browsable instance, so its convergence
  // is reported but not hard-gated; a normal run gates fully.
  // ================================================================================================
  assert.ok(anyHomeFormed,
    `the overlay did NOT form across the access point — no home node could reach novox over the overlay:\n${overlaySummary}`);

  if (!KEEP) {
    assert.deepEqual(allProblems, [], `the full mesh did not converge:\n  ${allProblems.join("\n  ")}\n\n${summary}`);
  } else if (allProblems.length) {
    console.log(`\nCONVERGENCE PROBLEMS (reported, not gated on a KEEP run):\n  ${allProblems.join("\n  ")}`);
  }
});