Add whole-mesh novox dry-run bed (stage 1 of whole-mesh rehearsal)
Install the real novox server's converted service set together on one node behind the substrate — the whole-catalogue install this rebuild never ran. The bed loads each committed module.json from mesh-catalog (no hand-written manifests), rewrites image refs to the scenario registry's digests, and remaps the co-located host-port collisions (nextcloud/invoicing/route-proxy :80, minio/invoicing :9000, gitea/umami :3000). Proven green: the whole set of 17 modules RESOLVES and applies (191 resources); the CORE 13 converge whole — all five providers (postgres, redis, minio, mongodb, mssql) plus keycloak, gitea, nextcloud and invoicing reaching their providers and staying up, plus portainer, verdaccio, registry and route-proxy. Reported as escalated gaps (do not gate green): fail2ban (declares capability intrusion-prevention that no host detector provides, and an unappliable assignment blocks whole-node resolution), umami/photos/mailu (catalog manifests do not wire the runtime/app env the images need; photos' server image is an alpine placeholder), and firewall (nftables.service is a oneshot that exits, but the module declares state running so mesh-host marks it failed). Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF
This commit is contained in:
@@ -0,0 +1,466 @@
|
||||
/**
|
||||
* The whole `novox` server's converted service set, installed together on ONE node behind the
|
||||
* substrate — the whole-catalogue install this rebuild has never actually run. First stage of a
|
||||
* whole-mesh rehearsal (novox/hq).
|
||||
*
|
||||
* Topology (proven by assigned-two-node-db.test.ts): the substrate (store, broker, control) rides
|
||||
* `anchor` and nothing else; ALL of novox's services ride the `novox` node. novox's own postgres
|
||||
* provider owns 5432 there, so it cannot co-locate with the substrate store. An overlay is placed so
|
||||
* each consumer's binding `at` resolves to novox's private address and reaches the providers
|
||||
* co-located with it.
|
||||
*
|
||||
* The SET (18 modules, all converted in mesh-catalog/modules/):
|
||||
* providers postgres redis minio mongodb mssql
|
||||
* consumers keycloak gitea nextcloud umami photos invoicing
|
||||
* apps portainer verdaccio registry route-proxy mailu
|
||||
* node-level firewall fail2ban
|
||||
*
|
||||
* Each committed module.json is LOADED from mesh-catalog — not hand-written — and its container
|
||||
* image references are rewritten to what this scenario's own registry serves by digest (the same
|
||||
* pinned(repositoryFor(image)) rule the two-node-db bed applies by hand). Two things this bed
|
||||
* discovered about the co-located set are handled at load time and RECORDED as findings:
|
||||
*
|
||||
* HOST-PORT COLLISIONS. When the whole set lands on one node with its committed host publishes,
|
||||
* several servers claim the same host port: nextcloud, invoicing-app and route-proxy all want 80;
|
||||
* minio and invoicing-api both want 9000; gitea and umami both want 3000. route-proxy is meant to
|
||||
* FRONT the web apps on 80/443, so the web apps' own host publishes are only for direct access.
|
||||
* To let the whole set converge, the colliding web/app host publishes are remapped to distinct
|
||||
* host ports here (container ports unchanged); the provider ports the consumers actually connect to
|
||||
* (postgres 5432, minio 9000, mongodb 27017, mssql 1433) are left as-is. See REMAP below.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
|
||||
* scripts/build-module-runtime.sh builds one runtime per module that has code; the route-proxy image
|
||||
* is built by scripts/build-route-proxy-image.sh; scenarios/whole-mesh-novox.yml stocks them all
|
||||
* alongside every server image.
|
||||
*/
|
||||
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll } from "./harness.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
|
||||
|
||||
const skip = !capability.usable
|
||||
? `lab not usable: ${capability.why}`
|
||||
: !binary || !existsSync(binary)
|
||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||
: !bundle || !existsSync(bundle)
|
||||
? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)"
|
||||
: false;
|
||||
|
||||
const SCENARIO = "whole-mesh-novox";
|
||||
const NODE = "novox";
|
||||
|
||||
/** Where the committed module.json files live: the mesh-catalog beside mesh-control. */
|
||||
const catalogDir = process.env["MESH_LAB_CATALOG"]
|
||||
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
|
||||
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
|
||||
|
||||
/**
|
||||
* The set, in dependency-reading order (the resolver accepts any order). Each row: the module, and
|
||||
* the container names it should bring up on the node. Node-level modules (firewall, fail2ban) bring
|
||||
* up no container — they install a package and run a service, checked separately.
|
||||
*/
|
||||
const MODULES: { name: string; containers: string[]; node?: boolean }[] = [
|
||||
{ name: "postgres", containers: ["postgres", "mesh-postgres"] },
|
||||
{ name: "redis", containers: ["redis", "mesh-redis"] },
|
||||
{ name: "minio", containers: ["minio", "mesh-minio"] },
|
||||
{ name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
|
||||
{ name: "mssql", containers: ["mssql", "mesh-mssql"] },
|
||||
{ name: "keycloak", containers: ["keycloak", "mesh-keycloak"] },
|
||||
{ name: "gitea", containers: ["gitea", "mesh-gitea"] },
|
||||
{ name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] },
|
||||
{ name: "umami", containers: ["umami", "mesh-umami"] },
|
||||
{ name: "photos", containers: ["photos", "mesh-photos"] },
|
||||
{ name: "invoicing", containers: ["invoicing-app", "invoicing-api"] },
|
||||
{ name: "portainer", containers: ["portainer", "mesh-portainer"] },
|
||||
{ name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] },
|
||||
{ name: "registry", containers: ["mesh-registry"] },
|
||||
{ name: "route-proxy", containers: ["route-proxy"] },
|
||||
{
|
||||
name: "mailu",
|
||||
containers: [
|
||||
"mailu-resolver", "mailu-redis", "mailu-admindb", "mailu-admin", "mailu-imap",
|
||||
"mailu-smtp", "mailu-antispam", "mailu-webmail", "mailu-front", "mesh-mailu",
|
||||
],
|
||||
},
|
||||
{ name: "firewall", containers: [], node: true },
|
||||
];
|
||||
|
||||
/**
|
||||
* Dropped from the converging set, with cause — recorded as a finding rather than silently omitted.
|
||||
*
|
||||
* fail2ban declares `capabilities: ["intrusion-prevention"]`, but mesh-host advertises no such
|
||||
* capability: profile/detectors.go defines container-runtime, package-manager, service-manager,
|
||||
* firewall, overlay, seat, privileged and graphical-session — nothing for intrusion-prevention. So
|
||||
* NO node can ever host fail2ban. Worse, `mesh-control assign` records the assignment even while
|
||||
* reporting it "cannot be applied", and the whole-node `push` then refuses to resolve the ENTIRE node
|
||||
* ("nothing was sent") over that one un-hostable assignment — one bad module blocks every other. It
|
||||
* is therefore left unassigned here so the rest of the set can be proven. (novox/hq — escalated.)
|
||||
*/
|
||||
const DROPPED: { name: string; why: string }[] = [
|
||||
{
|
||||
name: "fail2ban",
|
||||
why: 'declares capability "intrusion-prevention", which mesh-host has no detector for, so no node '
|
||||
+ "can host it; and an unappliable assignment blocks whole-node resolution (nothing sent).",
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* The provable CORE: modules that converge WHOLE on this node (every container up and stable) once
|
||||
* the substrate resolves and applies the set. This bed gates green on the CORE — a regression in any
|
||||
* of these turns it red. It is the substrate + all five providers + the four consumers that reach
|
||||
* their providers and stay up + the four standalone apps.
|
||||
*/
|
||||
const CORE = new Set([
|
||||
"postgres", "redis", "minio", "mongodb", "mssql",
|
||||
"keycloak", "gitea", "nextcloud", "invoicing",
|
||||
"portainer", "verdaccio", "registry", "route-proxy",
|
||||
]);
|
||||
|
||||
/**
|
||||
* KNOWN GAPS: modules that RESOLVE and are placed, but whose containers do not stay up because the
|
||||
* committed catalog manifest is incomplete (or, for firewall, a node-service expectation is unmet).
|
||||
* They are reported every run with the exact failure, and escalated (novox/hq) — but they do not gate
|
||||
* green, because the gap is in the catalog/host, not in this bed or the mesh substrate.
|
||||
*
|
||||
* umami — the mesh-umami provisioner needs the umami server URL and admin password in its
|
||||
* provisioner.env; the manifest wires neither, so it dies "UMAMI url or admin password
|
||||
* is not set". The umami SERVER itself comes up.
|
||||
* photos — the server image is a bare `alpine` placeholder (a real deployment runs immich at
|
||||
* :2283, where the runtime's MESH_PHOTOS_URL points); alpine has no long-running command
|
||||
* so it exits, and the runtime dies "no photos API key". Not genuinely converted.
|
||||
* mailu — the manifest generates only secret/database/admin env; the Mailu images need their full
|
||||
* configuration env (HOSTNAMES, DOMAIN, …), so every Mailu container dies rendering its
|
||||
* template: "jinja2 UndefinedError: 'HOSTNAMES' is undefined" (and the resolver's
|
||||
* unbound.conf is malformed). mailu-redis/admindb/admin/antispam do come up.
|
||||
* firewall — resolves and applies its package and ruleset, but nftables.service does not stay
|
||||
* running, so the node reports firewall.load failed. Diagnosed live in the report below.
|
||||
*/
|
||||
const KNOWN_GAPS = new Set(["umami", "photos", "mailu", "firewall"]);
|
||||
|
||||
/**
|
||||
* Host-port remaps applied at load time to break the co-located host-port collisions (see the file
|
||||
* header). Keyed by module, then by the module.json port entry to replace. Container ports are
|
||||
* preserved; only the host side changes.
|
||||
*/
|
||||
const REMAP: Record<string, Record<string, string>> = {
|
||||
nextcloud: { "80": "8090:80" },
|
||||
umami: { "3000": "3090:3000" },
|
||||
invoicing: { "80": "8091:80", "9000": "9091:9000" },
|
||||
};
|
||||
|
||||
let instanceId = "";
|
||||
/** What the scenario's registry serves, by digest. */
|
||||
let stocked: string[] = [];
|
||||
|
||||
function quote(s: string): string {
|
||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, [
|
||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||
], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The control plane, a container on the first node. */
|
||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||
return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
|
||||
}
|
||||
|
||||
/** The repository path a reference serves under — registry.ts's repositoryFor, mirrored. */
|
||||
function repositoryFor(reference: string): string {
|
||||
const withoutDigest = reference.split("@")[0] ?? reference;
|
||||
const lastColon = withoutDigest.lastIndexOf(":");
|
||||
const lastSlash = withoutDigest.lastIndexOf("/");
|
||||
return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest;
|
||||
}
|
||||
|
||||
/** The pinned reference this scenario's registry serves for a repository. */
|
||||
function pinned(repository: string): string {
|
||||
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
|
||||
assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
/** The substrate bundle, its image references pointed at this scenario's own registry. */
|
||||
function bundleFor(images: string[]): string {
|
||||
let text = readFileSync(bundle, "utf8");
|
||||
for (const ref of images) {
|
||||
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
|
||||
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
|
||||
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a committed module.json, rewrite every container image to the scenario's pinned digest, and
|
||||
* apply the host-port remaps. Returns the manifest as a string and whether it needs a broker account
|
||||
* (a runtime that reads MESH_BROKER_FILE — providers and tooled apps do; plain/node modules do not).
|
||||
*/
|
||||
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
||||
const path = resolve(catalogDir, name, "module.json");
|
||||
const m = JSON.parse(readFileSync(path, "utf8")) as {
|
||||
resources?: { type: string; image?: string; ports?: string[] }[];
|
||||
};
|
||||
const remap = REMAP[name] ?? {};
|
||||
for (const r of m.resources ?? []) {
|
||||
if (r.type !== "container") continue;
|
||||
if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image));
|
||||
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
|
||||
}
|
||||
const manifest = JSON.stringify(m);
|
||||
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
|
||||
}
|
||||
|
||||
function tokenFrom(said: string): string {
|
||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||
assert.ok(found, `no token in:\n${said}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
interface NodeState {
|
||||
reached: boolean;
|
||||
applied: boolean;
|
||||
current: boolean;
|
||||
waiting: boolean;
|
||||
wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined;
|
||||
raw: string;
|
||||
}
|
||||
|
||||
/** Ask the mesh, in its own terms, what a node has done with what it was sent. Never throws. */
|
||||
async function nodeState(node: string): Promise<NodeState> {
|
||||
const asked = await on("anchor", `docker exec mesh-control /mesh-control status --json`);
|
||||
if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out };
|
||||
let state: {
|
||||
wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[];
|
||||
waiting: { node: string }[];
|
||||
reported: { node: string; outcome: string; current: boolean }[];
|
||||
};
|
||||
try {
|
||||
state = JSON.parse(asked.out);
|
||||
} catch {
|
||||
return { reached: false, applied: false, current: false, waiting: false, raw: asked.out };
|
||||
}
|
||||
const word = state.reported.find((r) => r.node === node);
|
||||
const bad = state.wrong.find((w) => w.node === node);
|
||||
return {
|
||||
reached: true,
|
||||
applied: word?.outcome === "applied",
|
||||
current: !!word?.current,
|
||||
waiting: state.waiting.some((w) => w.node === node),
|
||||
wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined,
|
||||
raw: asked.out,
|
||||
};
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
|
||||
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
stocked = raised.images;
|
||||
|
||||
// anchor raises the substrate from its bundle, digests rewritten to the scenario registry's.
|
||||
await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
||||
await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000);
|
||||
const up = await must("anchor", `docker ps --format '{{.Names}}'`);
|
||||
for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) {
|
||||
assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`);
|
||||
}
|
||||
|
||||
// Both machines join the one mesh and run a host so they apply what they are pushed.
|
||||
for (const machine of ["anchor", NODE]) {
|
||||
await mesh(`node add ${machine}`);
|
||||
const token = tokenFrom(await mesh(`token issue --node ${machine}`));
|
||||
const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
|
||||
assert.match(said, new RegExp(`enrolled as ${machine}`), said);
|
||||
await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||
}
|
||||
}, { timeout: 2_700_000 });
|
||||
|
||||
after(async () => {
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 900_000 });
|
||||
|
||||
test("the whole novox service set resolves, installs and converges on one node in one push", {
|
||||
skip, timeout: 3_300_000,
|
||||
}, async () => {
|
||||
for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`);
|
||||
|
||||
// The overlay, so a consumer's binding `at` (the provider's private-network address) is non-empty.
|
||||
// Provider and consumers are co-located on novox, but the address the mesh writes into a consumer's
|
||||
// grant is the overlay address, so the overlay is placed on both nodes first (as two-node-db does).
|
||||
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
||||
await mesh(`overlay place ${NODE} --site lab`);
|
||||
await mesh("assign anchor networking");
|
||||
await mesh(`assign ${NODE} networking`);
|
||||
|
||||
// Add every module from its committed catalog manifest, issue the ones with a broker runtime, and
|
||||
// assign all to novox. The resolver resolves the whole set at push time regardless of order. The
|
||||
// loop is resilient: a module the node cannot host (a capability it does not advertise) is recorded
|
||||
// and skipped rather than aborting the whole run, so ONE run yields the full per-module picture.
|
||||
const issued: string[] = [];
|
||||
const assigned = new Set<string>();
|
||||
const refused: { name: string; why: string }[] = [];
|
||||
for (const { name } of MODULES) {
|
||||
try {
|
||||
const { manifest, broker } = loadManifest(name);
|
||||
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`);
|
||||
await mesh(`module add /${name}.json`);
|
||||
if (broker) {
|
||||
await mesh(`module issue ${name} --node ${NODE}`);
|
||||
issued.push(name);
|
||||
}
|
||||
await mesh(`assign ${NODE} ${name}`);
|
||||
assigned.add(name);
|
||||
} catch (err) {
|
||||
const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | ");
|
||||
refused.push({ name, why });
|
||||
console.log(`NOT ASSIGNED ${name}: ${why}`);
|
||||
}
|
||||
}
|
||||
console.log(`issued broker accounts for: ${issued.join(", ")}`);
|
||||
if (refused.length) console.log(`refused (node cannot host): ${refused.map((r) => r.name).join(", ")}`);
|
||||
|
||||
// ONE push. Resolution happens here; a resolver rejection surfaces as a non-zero push.
|
||||
let pushError = "";
|
||||
try {
|
||||
await mesh(`push ${NODE}`, 120_000);
|
||||
} catch (err) {
|
||||
pushError = (err as Error).message;
|
||||
console.log(`PUSH REJECTED:\n${pushError}`);
|
||||
}
|
||||
|
||||
// The node cannot reach applied+current while a KNOWN_GAP node-service (firewall.load) keeps
|
||||
// failing, so convergence is measured directly: wait until every CORE container is up (the node
|
||||
// still pulls ~14GiB first), bounded. `settle` is used only to read the node's own verdict for the
|
||||
// report — the wait is on the containers.
|
||||
const coreContainers = MODULES.filter((m) => CORE.has(m.name) && assigned.has(m.name))
|
||||
.flatMap((m) => m.containers);
|
||||
const psNames = async (): Promise<Map<string, string>> => {
|
||||
const out = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
||||
const map = new Map<string, string>();
|
||||
for (const line of out.split("\n")) {
|
||||
const [n, ...rest] = line.split("\t");
|
||||
if (n) map.set(n.trim(), rest.join("\t").trim());
|
||||
}
|
||||
return map;
|
||||
};
|
||||
let psMap = new Map<string, string>();
|
||||
if (!pushError) {
|
||||
const until = Date.now() + 2_400_000;
|
||||
while (Date.now() < until) {
|
||||
psMap = await psNames();
|
||||
if (coreContainers.every((c) => (psMap.get(c) ?? "").startsWith("Up"))) break;
|
||||
await new Promise((r) => setTimeout(r, 8000));
|
||||
}
|
||||
// A moment for first-boot bounces to settle before the crash-loop check below.
|
||||
await new Promise((r) => setTimeout(r, 15000));
|
||||
}
|
||||
psMap = await psNames();
|
||||
const final = await nodeState(NODE);
|
||||
const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up");
|
||||
|
||||
const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out;
|
||||
const nft = (await on(NODE, `systemctl is-active nftables 2>&1`)).out;
|
||||
|
||||
// ================================================================================================
|
||||
// The per-module report — this run's deliverable.
|
||||
// ================================================================================================
|
||||
const report: string[] = [];
|
||||
report.push("================ WHOLE-MESH novox CONVERGENCE ================");
|
||||
report.push(`node reached=${final.reached} applied=${final.applied} current=${final.current} waiting=${final.waiting}`);
|
||||
if (pushError) report.push(`PUSH REJECTED (resolver): ${pushError.split("\n").slice(0, 6).join("\n ")}`);
|
||||
const failedResources = final.wrong?.failed ?? [];
|
||||
if (final.wrong) {
|
||||
report.push(`NODE WRONG: outcome=${final.wrong.outcome} refused=${final.wrong.refused ?? "-"}`);
|
||||
for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`);
|
||||
}
|
||||
if (DROPPED.length) {
|
||||
report.push("---- DROPPED (not assignable on any node) ----");
|
||||
for (const d of DROPPED) report.push(` ${d.name.padEnd(14)} ${d.why}`);
|
||||
}
|
||||
if (refused.length) {
|
||||
report.push("---- REFUSED at assign ----");
|
||||
for (const r of refused) report.push(` ${r.name.padEnd(14)} ${r.why}`);
|
||||
}
|
||||
report.push("---- CORE (gates green) ----");
|
||||
const coreFailures: string[] = [];
|
||||
const gapStatus: string[] = [];
|
||||
for (const mod of MODULES) {
|
||||
if (!assigned.has(mod.name)) continue;
|
||||
const line = mod.node
|
||||
? `${mod.name.padEnd(14)} node-service nftables=${nft.trim()}`
|
||||
: (() => {
|
||||
const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`);
|
||||
const allUp = mod.containers.every(running);
|
||||
return `${mod.name.padEnd(14)} ${allUp ? "OK " : "GAP "} ${states.join(" ")}`;
|
||||
})();
|
||||
if (CORE.has(mod.name)) {
|
||||
const ok = !mod.node && mod.containers.every(running);
|
||||
report.push(` ${line}`);
|
||||
if (!ok) coreFailures.push(mod.name);
|
||||
} else {
|
||||
gapStatus.push(` ${line}`);
|
||||
}
|
||||
}
|
||||
report.push("---- KNOWN GAPS (reported, escalated, do NOT gate green) ----");
|
||||
for (const l of gapStatus) report.push(l);
|
||||
report.push("---- broker accounts (issued modules) ----");
|
||||
for (const name of issued) {
|
||||
const present = new RegExp(`${NODE}-${name}\\b`).test(users);
|
||||
report.push(` ${name.padEnd(14)} account ${present ? "present" : "MISSING"}`);
|
||||
}
|
||||
const summary = report.join("\n");
|
||||
console.log(summary);
|
||||
|
||||
// Live diagnostics for the KNOWN_GAP failures, so the report carries the exact cause each run.
|
||||
console.log(`\n---- firewall diagnostics ----\n${(await on(NODE, `systemctl status nftables --no-pager 2>&1 | head -12; echo '--- nftables.conf ---'; sed -n '1,20p' /etc/nftables.conf 2>&1; echo '--- journal ---'; journalctl -u nftables --no-pager -n 15 2>&1`)).out}`);
|
||||
for (const mod of MODULES.filter((m) => KNOWN_GAPS.has(m.name) && !m.node && assigned.has(m.name))) {
|
||||
for (const c of mod.containers) {
|
||||
if (psMap.has(c) && !running(c)) {
|
||||
console.log(`\n---- logs: ${c} (${psMap.get(c)}) ----\n${(await on(NODE, `docker logs ${c} 2>&1 | tail -20`)).out}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================================================
|
||||
// GREEN = the whole set RESOLVED (push accepted, resources applied), every CORE module converged
|
||||
// whole, and NO core resource failed to apply. The KNOWN_GAPS (umami, photos, mailu, firewall) and
|
||||
// DROPPED (fail2ban) are reported and escalated but do not gate — the gap is in the catalog/host.
|
||||
// ================================================================================================
|
||||
assert.equal(pushError, "", `the whole set did not resolve — push was rejected:\n${pushError}`);
|
||||
const coreResourceFailures = failedResources.filter((f) => {
|
||||
const mod = f.id.split(".")[0] ?? "";
|
||||
return CORE.has(mod);
|
||||
});
|
||||
assert.deepEqual(coreResourceFailures, [],
|
||||
`a CORE resource failed to apply:\n${coreResourceFailures.map((f) => `${f.id}: ${f.error}`).join("\n")}\n${summary}`);
|
||||
assert.deepEqual(coreFailures, [],
|
||||
`these CORE modules did not converge whole: ${coreFailures.join(", ")}\n${summary}`);
|
||||
});
|
||||
Reference in New Issue
Block a user