Add whole-mesh full three-node bed (stage 3: both server sets, one substrate)
Combine the novox (17-module) and ace (24-module) sets on ONE substrate and prove both node-plans converge together. anchor runs the substrate only; novox and ace each run their own self-contained set (own postgres/redis), so nothing crosses a node boundary except enrolment and the shared broker/store. The four modules both nodes run (postgres, redis, mssql, portainer) are added once and assigned to each node, each getting its own per-node broker account. An overlay is placed across all three nodes. Proven green: both nodes converge together on the one substrate. ace reaches applied+current with all 17 of its CORE up (and letta too this run); novox reaches all 13 CORE up with its only failed resource the known firewall.load oneshot gap. The two node-plans share one broker without collision — distinct novox-<mod> and ace-<mod> accounts for the modules both run. No new cross-node bug (overlay/DNS/identity/port) surfaced; ports are per-VM and the sets are node-self-contained. Tolerates the same nine credential-sidecar gaps and firewall's nftables.service oneshot documented in the per-server beds. Resource envelope: 3 VMs (anchor 4GiB, novox 16GiB, ace 18GiB) + registry scenery, ~79 union images (~35GB) stocked to one registry VM and pulled concurrently by both nodes; fit within 125GiB host RAM and the 180GiB lab pool. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF
This commit is contained in:
@@ -0,0 +1,431 @@
|
||||
/**
|
||||
* The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the
|
||||
* whole-mesh rehearsal (novox/hq). Combines whole-mesh-novox.test.ts and whole-mesh-ace.test.ts.
|
||||
*
|
||||
* anchor — substrate ONLY (store, broker, control).
|
||||
* novox — the 17-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu,
|
||||
* firewall. fail2ban is dropped (no `intrusion-prevention` detector — see that bed).
|
||||
* ace — the 24-module ace set (whole-mesh-ace): the media/home stack; its /services/media
|
||||
* library is pre-created so the ADR-0051 `accesses` resolve.
|
||||
*
|
||||
* An overlay is placed across all three so cross-node `at` resolves. Each service node is
|
||||
* self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and
|
||||
* the shared broker/store on anchor. The four modules both nodes run (postgres, redis, mssql,
|
||||
* portainer) are ADDED once and assigned to each node; each gets its own per-node broker account.
|
||||
*
|
||||
* This bed tolerates the SAME known gaps the per-server beds proved and escalated (nine
|
||||
* credential-sidecar crash-loops and firewall's oneshot nftables.service); it gates green on each
|
||||
* node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two node-plans
|
||||
* converge together on one substrate.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
|
||||
*/
|
||||
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll } from "./harness.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
|
||||
|
||||
const skip = !capability.usable
|
||||
? `lab not usable: ${capability.why}`
|
||||
: !binary || !existsSync(binary)
|
||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||
: !bundle || !existsSync(bundle)
|
||||
? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)"
|
||||
: false;
|
||||
|
||||
const SCENARIO = "whole-mesh-full";
|
||||
|
||||
const catalogDir = process.env["MESH_LAB_CATALOG"]
|
||||
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
|
||||
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
|
||||
|
||||
const MEDIA_DIRS = [
|
||||
"/services/media/series", "/services/media/anime", "/services/media/movies",
|
||||
"/services/media/music", "/services/media/audiobooks", "/services/media/downloads",
|
||||
"/services/media/books",
|
||||
];
|
||||
|
||||
type Mod = { name: string; containers: string[]; node?: boolean; runOnce?: string[] };
|
||||
|
||||
/** The novox node's 17-module set (fail2ban dropped). CORE gates; the rest are documented gaps. */
|
||||
const NOVOX: Mod[] = [
|
||||
{ name: "postgres", containers: ["postgres", "mesh-postgres"] },
|
||||
{ name: "redis", containers: ["redis", "mesh-redis"] },
|
||||
{ name: "minio", containers: ["minio", "mesh-minio"] },
|
||||
{ name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
|
||||
{ name: "mssql", containers: ["mssql", "mesh-mssql"] },
|
||||
{ name: "keycloak", containers: ["keycloak", "mesh-keycloak"] },
|
||||
{ name: "gitea", containers: ["gitea", "mesh-gitea"] },
|
||||
{ name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] },
|
||||
{ name: "umami", containers: ["umami", "mesh-umami"] },
|
||||
{ name: "photos", containers: ["photos", "mesh-photos"] },
|
||||
{ name: "invoicing", containers: ["invoicing-app", "invoicing-api"] },
|
||||
{ name: "portainer", containers: ["portainer", "mesh-portainer"] },
|
||||
{ name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] },
|
||||
{ name: "registry", containers: ["mesh-registry"] },
|
||||
{ name: "route-proxy", containers: ["route-proxy"] },
|
||||
{
|
||||
name: "mailu",
|
||||
containers: [
|
||||
"mailu-resolver", "mailu-redis", "mailu-admindb", "mailu-admin", "mailu-imap",
|
||||
"mailu-smtp", "mailu-antispam", "mailu-webmail", "mailu-front", "mesh-mailu",
|
||||
],
|
||||
},
|
||||
{ name: "firewall", containers: [], node: true },
|
||||
];
|
||||
const CORE_NOVOX = new Set([
|
||||
"postgres", "redis", "minio", "mongodb", "mssql",
|
||||
"keycloak", "gitea", "nextcloud", "invoicing",
|
||||
"portainer", "verdaccio", "registry", "route-proxy",
|
||||
]);
|
||||
const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall"]);
|
||||
|
||||
/** The ace node's 24-module set. */
|
||||
const ACE: Mod[] = [
|
||||
{ name: "postgres", containers: ["postgres", "mesh-postgres"] },
|
||||
{ name: "redis", containers: ["redis", "mesh-redis"] },
|
||||
{ name: "mssql", containers: ["mssql", "mesh-mssql"] },
|
||||
{ name: "sonarr", containers: ["sonarr", "mesh-sonarr"] },
|
||||
{ name: "radarr", containers: ["radarr", "mesh-radarr"] },
|
||||
{ name: "lidarr", containers: ["lidarr", "mesh-lidarr"] },
|
||||
{ name: "plex", containers: ["plex", "mesh-plex"] },
|
||||
{ name: "bazarr", containers: ["bazarr", "mesh-bazarr"] },
|
||||
{ name: "nzbget", containers: ["nzbget", "mesh-nzbget"] },
|
||||
{ name: "qbittorrent", containers: ["qbittorrent", "mesh-qbittorrent"] },
|
||||
{ name: "jackett", containers: ["jackett", "mesh-jackett"] },
|
||||
{ name: "ombi", containers: ["ombi", "mesh-ombi"] },
|
||||
{ name: "tautulli", containers: ["tautulli", "mesh-tautulli"] },
|
||||
{ name: "bookshelf", containers: ["bookshelf", "mesh-bookshelf"] },
|
||||
{ name: "home-assistant", containers: ["home-assistant", "mesh-home-assistant"] },
|
||||
{ name: "mosquitto", containers: ["mosquitto", "mesh-mosquitto"], runOnce: ["mosquitto-bootstrap"] },
|
||||
{ name: "influxdb", containers: ["influxdb", "mesh-influxdb"] },
|
||||
{ name: "grafana", containers: ["grafana", "mesh-grafana"] },
|
||||
{ name: "baserow", containers: ["baserow", "mesh-baserow"] },
|
||||
{ name: "letta", containers: ["letta", "mesh-letta"] },
|
||||
{ name: "nodered", containers: ["nodered", "mesh-nodered"] },
|
||||
{ name: "searxng", containers: ["valkey", "searxng", "mesh-searxng"] },
|
||||
{ name: "unifi", containers: ["unifi-controller", "mesh-unifi"] },
|
||||
{ name: "portainer", containers: ["portainer", "mesh-portainer"] },
|
||||
];
|
||||
const CORE_ACE = new Set([
|
||||
"postgres", "redis", "mssql",
|
||||
"sonarr", "radarr", "lidarr", "jackett", "tautulli", "bookshelf",
|
||||
"mosquitto", "influxdb", "grafana", "baserow", "nodered", "searxng", "unifi", "portainer",
|
||||
]);
|
||||
const GAPS_ACE = new Set(["plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta"]);
|
||||
|
||||
const PLAN: { node: string; mods: Mod[]; core: Set<string>; gaps: Set<string> }[] = [
|
||||
{ node: "novox", mods: NOVOX, core: CORE_NOVOX, gaps: GAPS_NOVOX },
|
||||
{ node: "ace", mods: ACE, core: CORE_ACE, gaps: GAPS_ACE },
|
||||
];
|
||||
|
||||
/**
|
||||
* Host-port remaps (per module — host ports are per-VM, so novox's and ace's never clash). Union of
|
||||
* both per-server beds' remaps.
|
||||
*/
|
||||
const REMAP: Record<string, Record<string, string>> = {
|
||||
nextcloud: { "80": "8090:80" },
|
||||
umami: { "3000": "3090:3000" },
|
||||
invoicing: { "80": "8091:80", "9000": "9091:9000" },
|
||||
qbittorrent: { "8080": "8090:8080" },
|
||||
searxng: { "8080": "8092:8080" },
|
||||
nzbget: { "6789": "6790:6789" },
|
||||
};
|
||||
|
||||
let instanceId = "";
|
||||
let stocked: string[] = [];
|
||||
|
||||
function quote(s: string): string {
|
||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, [
|
||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||
], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||
return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
|
||||
}
|
||||
|
||||
function repositoryFor(reference: string): string {
|
||||
const withoutDigest = reference.split("@")[0] ?? reference;
|
||||
const lastColon = withoutDigest.lastIndexOf(":");
|
||||
const lastSlash = withoutDigest.lastIndexOf("/");
|
||||
return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest;
|
||||
}
|
||||
|
||||
function pinned(repository: string): string {
|
||||
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
|
||||
assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
function bundleFor(images: string[]): string {
|
||||
let text = readFileSync(bundle, "utf8");
|
||||
for (const ref of images) {
|
||||
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
|
||||
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
|
||||
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
||||
const path = resolve(catalogDir, name, "module.json");
|
||||
const m = JSON.parse(readFileSync(path, "utf8")) as {
|
||||
resources?: { type: string; image?: string; ports?: string[] }[];
|
||||
};
|
||||
const remap = REMAP[name] ?? {};
|
||||
for (const r of m.resources ?? []) {
|
||||
if (r.type !== "container") continue;
|
||||
if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image));
|
||||
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
|
||||
}
|
||||
const manifest = JSON.stringify(m);
|
||||
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
|
||||
}
|
||||
|
||||
function tokenFrom(said: string): string {
|
||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||
assert.ok(found, `no token in:\n${said}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
interface NodeState {
|
||||
reached: boolean;
|
||||
applied: boolean;
|
||||
current: boolean;
|
||||
waiting: boolean;
|
||||
wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined;
|
||||
raw: string;
|
||||
}
|
||||
|
||||
async function nodeState(node: string): Promise<NodeState> {
|
||||
const asked = await on("anchor", `docker exec mesh-control /mesh-control status --json`);
|
||||
if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out };
|
||||
let state: {
|
||||
wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[];
|
||||
waiting: { node: string }[];
|
||||
reported: { node: string; outcome: string; current: boolean }[];
|
||||
};
|
||||
try {
|
||||
state = JSON.parse(asked.out);
|
||||
} catch {
|
||||
return { reached: false, applied: false, current: false, waiting: false, raw: asked.out };
|
||||
}
|
||||
const word = state.reported.find((r) => r.node === node);
|
||||
const bad = state.wrong.find((w) => w.node === node);
|
||||
return {
|
||||
reached: true,
|
||||
applied: word?.outcome === "applied",
|
||||
current: !!word?.current,
|
||||
waiting: state.waiting.some((w) => w.node === node),
|
||||
wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined,
|
||||
raw: asked.out,
|
||||
};
|
||||
}
|
||||
|
||||
async function psMapOf(node: string): Promise<Map<string, string>> {
|
||||
const out = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
||||
const map = new Map<string, string>();
|
||||
for (const line of out.split("\n")) {
|
||||
const [n, ...rest] = line.split("\t");
|
||||
if (n) map.set(n.trim(), rest.join("\t").trim());
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
|
||||
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
stocked = raised.images;
|
||||
|
||||
await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
||||
await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000);
|
||||
const up = await must("anchor", `docker ps --format '{{.Names}}'`);
|
||||
for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) {
|
||||
assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`);
|
||||
}
|
||||
|
||||
for (const machine of ["anchor", "novox", "ace"]) {
|
||||
await mesh(`node add ${machine}`);
|
||||
const token = tokenFrom(await mesh(`token issue --node ${machine}`));
|
||||
const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
|
||||
assert.match(said, new RegExp(`enrolled as ${machine}`), said);
|
||||
await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||
}
|
||||
|
||||
// The operator provides ace's media library (ADR 0051 accesses confirm the paths, create nothing).
|
||||
await must("ace", `mkdir -p ${MEDIA_DIRS.join(" ")}`);
|
||||
}, { timeout: 3_000_000 });
|
||||
|
||||
after(async () => {
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 900_000 });
|
||||
|
||||
test("both server sets converge together on one substrate", { skip, timeout: 3_600_000 }, async () => {
|
||||
// Overlay across all three, so every node's private address exists and cross-node `at` resolves.
|
||||
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
||||
await mesh("overlay place novox --site lab");
|
||||
await mesh("overlay place ace --site lab");
|
||||
await mesh("assign anchor networking");
|
||||
await mesh("assign novox networking");
|
||||
await mesh("assign ace networking");
|
||||
|
||||
// Add every unique module ONCE (the four shared modules are added once, assigned to each node), then
|
||||
// issue a per-node broker account and assign, resiliently.
|
||||
const added = new Map<string, boolean>(); // name -> needs broker
|
||||
async function ensureAdded(name: string): Promise<boolean> {
|
||||
const known = added.get(name);
|
||||
if (known !== undefined) return known;
|
||||
const { manifest, broker } = loadManifest(name);
|
||||
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`);
|
||||
await mesh(`module add /${name}.json`);
|
||||
added.set(name, broker);
|
||||
return broker;
|
||||
}
|
||||
|
||||
const assigned: Record<string, Set<string>> = { novox: new Set(), ace: new Set() };
|
||||
const refused: Record<string, { name: string; why: string }[]> = { novox: [], ace: [] };
|
||||
for (const { node, mods } of PLAN) {
|
||||
for (const { name } of mods) {
|
||||
try {
|
||||
const broker = await ensureAdded(name);
|
||||
if (broker) await mesh(`module issue ${name} --node ${node}`);
|
||||
await mesh(`assign ${node} ${name}`);
|
||||
assigned[node]!.add(name);
|
||||
} catch (err) {
|
||||
const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | ");
|
||||
refused[node]!.push({ name, why });
|
||||
console.log(`NOT ASSIGNED ${node}/${name}: ${why}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ONE push per node.
|
||||
const pushError: Record<string, string> = { novox: "", ace: "" };
|
||||
for (const node of ["novox", "ace"]) {
|
||||
try {
|
||||
await mesh(`push ${node}`, 240_000);
|
||||
} catch (err) {
|
||||
pushError[node] = (err as Error).message;
|
||||
console.log(`PUSH REJECTED (${node}):\n${pushError[node]}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for both nodes' CORE containers to come up (they pull concurrently from the one registry).
|
||||
const psMaps: Record<string, Map<string, string>> = { novox: new Map(), ace: new Map() };
|
||||
for (const { node, mods, core } of PLAN) {
|
||||
if (pushError[node]) continue;
|
||||
const coreContainers = mods.filter((m) => core.has(m.name) && assigned[node]!.has(m.name)).flatMap((m) => m.containers);
|
||||
const until = Date.now() + 2_700_000;
|
||||
while (Date.now() < until) {
|
||||
psMaps[node] = await psMapOf(node);
|
||||
if (coreContainers.every((c) => (psMaps[node]!.get(c) ?? "").startsWith("Up"))) break;
|
||||
await new Promise((r) => setTimeout(r, 10000));
|
||||
}
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 20000)); // let first-boot bounces settle
|
||||
|
||||
// ================================================================================================
|
||||
// Per-node report + gating. GREEN = each node's push accepted, every CORE module converged whole,
|
||||
// and no NON-GAP resource failed to apply. The nine credential-sidecar gaps and firewall's oneshot
|
||||
// are tolerated (documented + escalated in the per-server beds).
|
||||
// ================================================================================================
|
||||
const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out;
|
||||
const allProblems: string[] = [];
|
||||
const report: string[] = ["================ FULL MESH CONVERGENCE ================"];
|
||||
|
||||
for (const { node, mods, core, gaps } of PLAN) {
|
||||
const psMap = psMaps[node] = await psMapOf(node);
|
||||
const st = await nodeState(node);
|
||||
const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up");
|
||||
const ranOnce = (name: string): boolean => !psMap.has(name) || /^(Up|Exited \(0\))/.test(psMap.get(name) ?? "");
|
||||
const failedResources = st.wrong?.failed ?? [];
|
||||
|
||||
report.push(`\n---- node ${node}: reached=${st.reached} applied=${st.applied} current=${st.current} waiting=${st.waiting} ----`);
|
||||
if (pushError[node]) report.push(` PUSH REJECTED: ${pushError[node].split("\n").slice(0, 6).join("\n ")}`);
|
||||
if (st.wrong) {
|
||||
report.push(` NODE WRONG: outcome=${st.wrong.outcome}`);
|
||||
for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`);
|
||||
}
|
||||
for (const r of refused[node]!) report.push(` REFUSED ${r.name}: ${r.why}`);
|
||||
|
||||
const coreFailures: string[] = [];
|
||||
for (const mod of mods) {
|
||||
if (!assigned[node]!.has(mod.name)) continue;
|
||||
const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`);
|
||||
const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce);
|
||||
const tag = core.has(mod.name) ? (ok ? "OK " : "FAIL") : (ok ? "ok " : "GAP ");
|
||||
report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(15)} ${tag} ${states.join(" ")}`);
|
||||
if (core.has(mod.name) && !ok) coreFailures.push(mod.name);
|
||||
}
|
||||
const issuedHere = mods.filter((m) => new RegExp(`${node}-${m.name}\\b`).test(users)).length;
|
||||
report.push(` broker accounts: ${issuedHere} present for ${node}`);
|
||||
|
||||
// Gate: push accepted, all CORE up, no NON-GAP resource failed. A failed resource names its
|
||||
// owning module inside the error (`applying "firewall.load": …`), not in `id` (which is the outer
|
||||
// "apply" key), so the owner is extracted from either — and a failure owned by a KNOWN_GAP module
|
||||
// (firewall's oneshot nftables.service) is tolerated.
|
||||
const gapOwnerOf = (f: { id: string; error: string }): string => {
|
||||
const m = f.error.match(/applying "([^".]+)\./);
|
||||
return m?.[1] ?? (f.id.split(".")[0] ?? "");
|
||||
};
|
||||
if (pushError[node]) allProblems.push(`${node}: push rejected`);
|
||||
if (coreFailures.length) allProblems.push(`${node}: CORE not converged: ${coreFailures.join(", ")}`);
|
||||
const nonGapFailed = failedResources.filter((f) => !gaps.has(gapOwnerOf(f)));
|
||||
if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`);
|
||||
}
|
||||
|
||||
const summary = report.join("\n");
|
||||
console.log(summary);
|
||||
|
||||
// Cross-node identity proof: each node's own scoped broker accounts exist and are distinct — the
|
||||
// two node-plans share one broker without colliding (both run a `postgres`, `redis`, `mssql`).
|
||||
for (const acct of ["novox-postgres", "ace-postgres", "novox-redis", "ace-redis"]) {
|
||||
if (!new RegExp(acct).test(users)) allProblems.push(`missing broker account ${acct}`);
|
||||
}
|
||||
|
||||
// Diagnostics for any CORE failure (the gaps are expected; a CORE failure is what we must see).
|
||||
for (const { node, mods, core } of PLAN) {
|
||||
const psMap = psMaps[node]!;
|
||||
for (const mod of mods) {
|
||||
if (!core.has(mod.name) || !assigned[node]!.has(mod.name)) continue;
|
||||
for (const c of mod.containers) {
|
||||
if (psMap.has(c) && !(psMap.get(c) ?? "").startsWith("Up")) {
|
||||
console.log(`\n---- ${node} logs: ${c} (${psMap.get(c)}) ----\n${(await on(node, `docker logs ${c} 2>&1 | tail -25`)).out}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.deepEqual(allProblems, [], `the full mesh did not converge together:\n ${allProblems.join("\n ")}\n\n${summary}`);
|
||||
});
|
||||
Reference in New Issue
Block a user