/** * The whole `ace` server's converted service set, installed together on ONE node behind the * foundation — the media / home-automation half of the whole-mesh rehearsal (novox/hq). Sibling of * whole-mesh-novox.test.ts; same harness and topology, a larger, media-heavy module set. * * Foundation (store, broker, control) rides `anchor` and nothing else; ALL of ace's services ride the * `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres * providers co-located with them. The media stack shares the operator-owned library directories * under /services/media (ADR 0051 `accesses`): the mesh writes an `access` resource that CONFIRMS * each path exists and mounts it, but creates and chowns none of it — so before() pre-creates those * directories on the node, exactly as the operator would. * * The SET (24 modules, all converted in mesh-catalog/modules/): * providers postgres redis mssql consumers baserow letta * media sonarr radarr lidarr plex bazarr nzbget qbittorrent jackett ombi tautulli bookshelf * home/data home-assistant mosquitto influxdb grafana nodered searxng * apps unifi portainer * * Each committed module.json is LOADED from mesh-catalog (not hand-written); its container image * references of OURS are rewritten to the IDs the machine holds, and the co-located * host-port collisions are remapped at load time (see REMAP). * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { existsSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { labIsUsable, destroyAll, foundationBundle, deriveTheFilterOn, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" : catalogueIsPresent(); const SCENARIO = "whole-mesh-ace"; const NODE = "ace"; /** The operator-owned media library the ADR-0051 `accesses` point at — pre-created before the push. */ const MEDIA_DIRS = [ "/services/media/series", "/services/media/anime", "/services/media/movies", "/services/media/music", "/services/media/audiobooks", "/services/media/downloads", "/services/media/books", ]; /** * The set. Each row names the module and the containers it should bring up. `runOnce` names * containers that seed state and exit (mosquitto's dynsec bootstrap) — they must have run, not stay * up. */ const MODULES: { name: string; containers: string[]; runOnce?: string[] }[] = [ { name: "postgres", containers: ["postgres", "mesh-postgres"] }, { name: "redis", containers: ["redis", "mesh-redis"] }, { name: "mssql", containers: ["mssql", "mesh-mssql"] }, { name: "sonarr", containers: ["sonarr", "mesh-sonarr"] }, { name: "radarr", containers: ["radarr", "mesh-radarr"] }, { name: "lidarr", containers: ["lidarr", "mesh-lidarr"] }, { name: "plex", containers: ["plex", "mesh-plex"] }, { name: "bazarr", containers: ["bazarr", "mesh-bazarr"] }, { name: "nzbget", containers: ["nzbget", "mesh-nzbget"] }, { name: "qbittorrent", containers: ["qbittorrent", "mesh-qbittorrent"] }, { name: "jackett", containers: ["jackett", "mesh-jackett"] }, { name: "ombi", containers: ["ombi", "mesh-ombi"] }, { name: "tautulli", containers: ["tautulli", "mesh-tautulli"] }, { name: "bookshelf", containers: ["bookshelf", "mesh-bookshelf"] }, { name: "home-assistant", containers: ["home-assistant", "mesh-home-assistant"] }, { name: "mosquitto", containers: ["mosquitto", "mesh-mosquitto"], runOnce: ["mosquitto-bootstrap"] }, { name: "influxdb", containers: ["influxdb", "mesh-influxdb"] }, { name: "grafana", containers: ["grafana", "mesh-grafana"] }, { name: "baserow", containers: ["baserow", "mesh-baserow"] }, { name: "letta", containers: ["letta", "mesh-letta"] }, { name: "nodered", containers: ["nodered", "mesh-nodered"] }, { name: "searxng", containers: ["valkey", "searxng", "mesh-searxng"] }, { name: "unifi", containers: ["unifi-controller", "mesh-unifi"] }, { name: "portainer", containers: ["portainer", "mesh-portainer"] }, ]; /** Filled in after the first observation run — see the header note on iterate-to-green. */ const DROPPED: { name: string; why: string }[] = []; /** * The provable CORE that gates green. Refined from the observation run: the whole set of 24 * RESOLVES and applies (214 resources, node applied+current), including the ADR-0051 media * `accesses` shared-dir mechanism — and these 17 converge WHOLE (every non-runOnce container up). * KNOWN_GAPS below are reported and escalated but do not gate. */ const CORE = new Set([ "postgres", "redis", "mssql", "sonarr", "radarr", "lidarr", "jackett", "tautulli", "bookshelf", "mosquitto", "influxdb", "grafana", "baserow", "nodered", "searxng", "unifi", "portainer", ]); /** * KNOWN GAPS: resolve and place, but a container does not stay up. Two classes. * * A) TOOL-RUNTIME NEEDS AN OPERATOR CREDENTIAL THE MANIFEST DOES NOT WIRE. The mesh- sidecar * cannot construct its client and crash-loops (verbatim below); the SERVER of each is UP — only * the tool sidecar is down. This is the umami/photos class from whole-mesh-novox, systemic across * the media/home tools whose key a human sets in the app UI rather than one derivable from a * config file (sonarr/radarr/lidarr/jackett/tautulli DO self-configure from the app's config file, * so their runtimes come up): * plex — "no Plex token — set MESH_PLEX_TOKEN or make the data dir readable" * bazarr — "no Bazarr API key — set MESH_BAZARR_API_KEY" * nzbget — "NZBGet not configured — set MESH_NZBGET_URL and MESH_NZBGET_PASSWORD" * qbittorrent — "qBittorrent not configured — set MESH_QBITTORRENT_URL and MESH_QBITTORRENT_PASSWORD" * ombi — "no Ombi API key — set MESH_OMBI_API_KEY" * home-assistant — "no Home Assistant token — set MESH_HOMEASSISTANT_TOKEN" * * B) APP MIGRATION AGAINST POSTGRES. * letta — the letta APP's DB migration fails on first boot ("connection to server at * ace.internal … port 5432 failed: Connection refused"); baserow, the other postgres * consumer, comes up over the same overlay path, so this is letta's own startup * ordering / lack of retry. The deeper blocker once it connects is the postgres `vector` * (pgvector) extension a non-superuser consumer cannot CREATE — documented the same way * in assigned-two-node-db.test.ts. Its runtime mesh-letta and its credential are fine. */ const KNOWN_GAPS = new Set([ "plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta", ]); /** * Host-port remaps applied at load time to break the co-located host-port collisions. In this set * three servers claim :8080 (qbittorrent, searxng, the UniFi controller) and two claim :6789 (nzbget, * the UniFi controller). UniFi keeps its published ports; qbittorrent/searxng/nzbget are remapped. * Container ports are preserved; only the host side changes. */ const REMAP: Record> = { qbittorrent: { "8080": "8090:8080" }, searxng: { "8080": "8092:8080" }, nzbget: { "6789": "6790:6789" }, }; let instanceId = ""; let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { const { stdout } = await exec(instanceId, machine, [ "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, ], timeoutMs); const marker = stdout.lastIndexOf("__exit="); if (marker < 0) return { out: stdout, ok: false }; return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; } async function must(machine: string, command: string, timeoutMs?: number): Promise { const { out, ok } = await on(machine, command, timeoutMs); if (!ok) throw new Error(`${machine}: ${command}\n${out}`); return out; } async function mesh(command: string, timeoutMs?: number): Promise { return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); } /** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); } /** The catalogue's manifest as the lab runs it (harness). This bed's own loader never resolved a * runtime ARTIFACT to a stocked image, so every module whose runtime the mesh builds travelled to * the machine unresolved — the shared loader does (novox/hq 04-ISSUES/073). */ function loadManifest(name: string): { manifest: string; broker: boolean } { const manifest = catalogueModule(name, held, { ports: REMAP[name] }); return { manifest, broker: needsBrokerAccount(manifest) }; } function tokenFrom(said: string): string { const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); assert.ok(found, `no token in:\n${said}`); return found; } interface NodeState { reached: boolean; applied: boolean; current: boolean; waiting: boolean; wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined; raw: string; } async function nodeState(node: string): Promise { const asked = await on("anchor", `docker exec mesh-controller /mesh-controller status --json`); if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; let state: { wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[]; waiting: { node: string }[]; reported: { node: string; outcome: string; current: boolean }[]; }; try { state = JSON.parse(asked.out); } catch { return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; } const word = state.reported.find((r) => r.node === node); const bad = state.wrong.find((w) => w.node === node); return { reached: true, applied: word?.outcome === "applied", current: !!word?.current, waiting: state.waiting.some((w) => w.node === node), wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined, raw: asked.out, }; } before(async () => { if (skip) return; const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; held = raised.images; await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000); const up = await must("anchor", `docker ps --format '{{.Names}}'`); for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`); } for (const machine of ["anchor", NODE]) { await mesh(`node add ${machine}`); const token = tokenFrom(await mesh(`token issue --node ${machine}`)); const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); assert.match(said, new RegExp(`enrolled as ${machine}`), said); await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); } // The operator provides the media library: the ADR-0051 `access` resources CONFIRM these paths and // the media containers mount them, but the mesh creates none of it. Without this the media stack's // apply is refused ("the path is not present"). await must(NODE, `mkdir -p ${MEDIA_DIRS.join(" ")}`); }, { timeout: 2_700_000 }); after(async () => { if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 900_000 }); test("the whole ace service set resolves, installs and converges on one node in one push", { skip, timeout: 3_300_000, }, async () => { for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`); // The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres). await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); await mesh(`overlay place ${NODE} --site lab`); await mesh("assign anchor networking"); await mesh(`assign ${NODE} networking`); // The anchor's derived filter, admitting the hub's port — what genesis does on the control-node, // and what a bed raised from the bundle must do itself (ADR 0088; see the harness). await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on }); // Add / issue / assign, resiliently: a module the node cannot host is recorded and skipped so one // bad assignment cannot poison the whole-node push. const issued: string[] = []; const assigned = new Set(); const refused: { name: string; why: string }[] = []; for (const { name } of MODULES) { if (DROPPED.some((d) => d.name === name)) continue; try { const { manifest, broker } = loadManifest(name); await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`); await mesh(`module add /${name}.json`); if (broker) { await mesh(`module issue ${name} --node ${NODE}`); issued.push(name); } await mesh(`assign ${NODE} ${name}`); assigned.add(name); } catch (err) { const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | "); refused.push({ name, why }); console.log(`NOT ASSIGNED ${name}: ${why}`); } } console.log(`issued broker accounts for: ${issued.length} modules`); if (refused.length) console.log(`refused (node cannot host): ${refused.map((r) => r.name).join(", ")}`); // ONE push. let pushError = ""; try { await mesh(`push ${NODE}`, 180_000); } catch (err) { pushError = (err as Error).message; console.log(`PUSH REJECTED:\n${pushError}`); } // Wait for every CORE container to be up (the node pulls ~20GiB of images first), bounded. const coreContainers = MODULES.filter((m) => CORE.has(m.name) && assigned.has(m.name)) .flatMap((m) => m.containers); const psNames = async (): Promise> => { const out = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; const map = new Map(); for (const line of out.split("\n")) { const [n, ...rest] = line.split("\t"); if (n) map.set(n.trim(), rest.join("\t").trim()); } return map; }; let psMap = new Map(); if (!pushError) { const until = Date.now() + 2_700_000; while (Date.now() < until) { psMap = await psNames(); if (coreContainers.every((c) => (psMap.get(c) ?? "").startsWith("Up"))) break; await new Promise((r) => setTimeout(r, 8000)); } await new Promise((r) => setTimeout(r, 20000)); // let first-boot bounces settle } psMap = await psNames(); const final = await nodeState(NODE); const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up"); // A run-once (mosquitto's dynsec bootstrap) seeds state and exits; the mesh removes it on success, // so absent-from-`docker ps -a` means it completed and was reaped (the node is applied+current, so // its resource did apply). Present means it must be up or have exited cleanly. const ranOnce = (name: string): boolean => !psMap.has(name) || /^(Up|Exited \(0\))/.test(psMap.get(name) ?? ""); const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; // ================================================================================================ // The per-module report — the deliverable. // ================================================================================================ const report: string[] = []; report.push("================ WHOLE-MESH ace CONVERGENCE ================"); report.push(`node reached=${final.reached} applied=${final.applied} current=${final.current} waiting=${final.waiting}`); if (pushError) report.push(`PUSH REJECTED (resolver): ${pushError.split("\n").slice(0, 8).join("\n ")}`); const failedResources = final.wrong?.failed ?? []; if (final.wrong) { report.push(`NODE WRONG: outcome=${final.wrong.outcome} refused=${final.wrong.refused ?? "-"}`); for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`); } if (DROPPED.length) { report.push("---- DROPPED ----"); for (const d of DROPPED) report.push(` ${d.name.padEnd(16)} ${d.why}`); } if (refused.length) { report.push("---- REFUSED at assign ----"); for (const r of refused) report.push(` ${r.name.padEnd(16)} ${r.why}`); } const line = (mod: typeof MODULES[number]): { text: string; ok: boolean } => { const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`); const extra = (mod.runOnce ?? []).map((c) => `${c}:${ranOnce(c) ? "ran" : (psMap.get(c) ?? "MISSING")}`); const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce); return { text: `${mod.name.padEnd(16)} ${ok ? "OK " : "GAP "} ${[...states, ...extra].join(" ")}`, ok }; }; report.push("---- CORE (gates green) ----"); const coreFailures: string[] = []; const gaps: string[] = []; for (const mod of MODULES) { if (!assigned.has(mod.name)) continue; const { text, ok } = line(mod); if (CORE.has(mod.name)) { report.push(` ${text}`); if (!ok) coreFailures.push(mod.name); } else { gaps.push(` ${text}`); } } report.push("---- KNOWN GAPS (reported, escalated, do NOT gate green) ----"); for (const g of gaps) report.push(g); report.push(`broker accounts issued: ${issued.filter((n) => new RegExp(`${NODE}-${n}\\b`).test(users)).length}/${issued.length} present`); const summary = report.join("\n"); console.log(summary); // Diagnostics for anything not up: exact crash cause per container. const toDump = MODULES.filter((m) => assigned.has(m.name) && (coreFailures.includes(m.name) || KNOWN_GAPS.has(m.name))); if (toDump.length) { console.log(`\n---- ${NODE} mesh-host.log tail ----\n${(await on(NODE, `tail -60 /var/log/mesh-host.log`)).out}`); for (const mod of toDump) { for (const c of mod.containers) { if (psMap.has(c) && !running(c)) { console.log(`\n---- logs: ${c} (${psMap.get(c)}) ----\n${(await on(NODE, `docker logs ${c} 2>&1 | tail -20`)).out}`); } } } } // ================================================================================================ // GREEN = the whole set RESOLVED (push accepted), every CORE module converged whole, and no CORE // resource failed to apply. KNOWN_GAPS and DROPPED are reported and escalated but do not gate. // ================================================================================================ assert.equal(pushError, "", `the whole set did not resolve — push was rejected:\n${pushError}`); const coreResourceFailures = failedResources.filter((f) => CORE.has(f.id.split(".")[0] ?? "")); assert.deepEqual(coreResourceFailures, [], `a CORE resource failed to apply:\n${coreResourceFailures.map((f) => `${f.id}: ${f.error}`).join("\n")}\n${summary}`); assert.deepEqual(coreFailures, [], `these CORE modules did not converge whole: ${coreFailures.join(", ")}\n${summary}`); });