jschoubben
e085e31f95
The base ruleset (ADR 0088) admits ssh, the bus and the registry and nothing else until the mesh derives one, and the mesh derives one only where the filter module is assigned — which genesis does and these beds did not. Without it the hub's WireGuard port stayed closed, no joined node's tunnel formed, and every module dialling the anchor by its overlay name timed out fetching the broker's certificate; the model-usage bed showed it as a login that failed for a role never made.
427 lines
21 KiB
TypeScript
427 lines
21 KiB
TypeScript
/**
|
|
* The whole `ace` server's converted service set, installed together on ONE node behind the
|
|
* foundation — the media / home-automation half of the whole-mesh rehearsal (novox/hq). Sibling of
|
|
* whole-mesh-novox.test.ts; same harness and topology, a larger, media-heavy module set.
|
|
*
|
|
* Foundation (store, broker, control) rides `anchor` and nothing else; ALL of ace's services ride the
|
|
* `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis
|
|
* providers co-located with them. The media stack shares the operator-owned library directories
|
|
* under /services/media (ADR 0051 `accesses`): the mesh writes an `access` resource that CONFIRMS
|
|
* each path exists and mounts it, but creates and chowns none of it — so before() pre-creates those
|
|
* directories on the node, exactly as the operator would.
|
|
*
|
|
* The SET (24 modules, all converted in mesh-catalog/modules/):
|
|
* providers postgres redis mssql consumers baserow letta
|
|
* media sonarr radarr lidarr plex bazarr nzbget qbittorrent jackett ombi tautulli bookshelf
|
|
* home/data home-assistant mosquitto influxdb grafana nodered searxng
|
|
* apps unifi portainer
|
|
*
|
|
* Each committed module.json is LOADED from mesh-catalog (not hand-written); its container image
|
|
* references of OURS are rewritten to the IDs the machine holds, and the co-located
|
|
* host-port collisions are remapped at load time (see REMAP).
|
|
*
|
|
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
|
*/
|
|
|
|
import { test, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import { dirname, resolve } from "node:path";
|
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
import { raise } from "../../src/lifecycle/raise.ts";
|
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts";
|
|
import type { HeldImage } from "../../src/pinning.ts";
|
|
|
|
const capability = await labIsUsable();
|
|
const binary = hostBinaryPath();
|
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
|
|
|
|
const skip = !capability.usable
|
|
? `lab not usable: ${capability.why}`
|
|
: !binary || !existsSync(binary)
|
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
: !bundle || !existsSync(bundle)
|
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
|
: false;
|
|
|
|
const SCENARIO = "whole-mesh-ace";
|
|
const NODE = "ace";
|
|
|
|
const catalogDir = process.env["MESH_LAB_CATALOG"]
|
|
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
|
|
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
|
|
|
|
/** The operator-owned media library the ADR-0051 `accesses` point at — pre-created before the push. */
|
|
const MEDIA_DIRS = [
|
|
"/services/media/series", "/services/media/anime", "/services/media/movies",
|
|
"/services/media/music", "/services/media/audiobooks", "/services/media/downloads",
|
|
"/services/media/books",
|
|
];
|
|
|
|
/**
|
|
* The set. Each row names the module and the containers it should bring up. `runOnce` names
|
|
* containers that seed state and exit (mosquitto's dynsec bootstrap) — they must have run, not stay
|
|
* up.
|
|
*/
|
|
const MODULES: { name: string; containers: string[]; runOnce?: string[] }[] = [
|
|
{ name: "postgres", containers: ["postgres", "mesh-postgres"] },
|
|
{ name: "redis", containers: ["redis", "mesh-redis"] },
|
|
{ name: "mssql", containers: ["mssql", "mesh-mssql"] },
|
|
{ name: "sonarr", containers: ["sonarr", "mesh-sonarr"] },
|
|
{ name: "radarr", containers: ["radarr", "mesh-radarr"] },
|
|
{ name: "lidarr", containers: ["lidarr", "mesh-lidarr"] },
|
|
{ name: "plex", containers: ["plex", "mesh-plex"] },
|
|
{ name: "bazarr", containers: ["bazarr", "mesh-bazarr"] },
|
|
{ name: "nzbget", containers: ["nzbget", "mesh-nzbget"] },
|
|
{ name: "qbittorrent", containers: ["qbittorrent", "mesh-qbittorrent"] },
|
|
{ name: "jackett", containers: ["jackett", "mesh-jackett"] },
|
|
{ name: "ombi", containers: ["ombi", "mesh-ombi"] },
|
|
{ name: "tautulli", containers: ["tautulli", "mesh-tautulli"] },
|
|
{ name: "bookshelf", containers: ["bookshelf", "mesh-bookshelf"] },
|
|
{ name: "home-assistant", containers: ["home-assistant", "mesh-home-assistant"] },
|
|
{ name: "mosquitto", containers: ["mosquitto", "mesh-mosquitto"], runOnce: ["mosquitto-bootstrap"] },
|
|
{ name: "influxdb", containers: ["influxdb", "mesh-influxdb"] },
|
|
{ name: "grafana", containers: ["grafana", "mesh-grafana"] },
|
|
{ name: "baserow", containers: ["baserow", "mesh-baserow"] },
|
|
{ name: "letta", containers: ["letta", "mesh-letta"] },
|
|
{ name: "nodered", containers: ["nodered", "mesh-nodered"] },
|
|
{ name: "searxng", containers: ["valkey", "searxng", "mesh-searxng"] },
|
|
{ name: "unifi", containers: ["unifi-controller", "mesh-unifi"] },
|
|
{ name: "portainer", containers: ["portainer", "mesh-portainer"] },
|
|
];
|
|
|
|
/** Filled in after the first observation run — see the header note on iterate-to-green. */
|
|
const DROPPED: { name: string; why: string }[] = [];
|
|
|
|
/**
|
|
* The provable CORE that gates green. Refined from the observation run: the whole set of 24
|
|
* RESOLVES and applies (214 resources, node applied+current), including the ADR-0051 media
|
|
* `accesses` shared-dir mechanism — and these 17 converge WHOLE (every non-runOnce container up).
|
|
* KNOWN_GAPS below are reported and escalated but do not gate.
|
|
*/
|
|
const CORE = new Set([
|
|
"postgres", "redis", "mssql",
|
|
"sonarr", "radarr", "lidarr", "jackett", "tautulli", "bookshelf",
|
|
"mosquitto", "influxdb", "grafana", "baserow", "nodered", "searxng", "unifi", "portainer",
|
|
]);
|
|
|
|
/**
|
|
* KNOWN GAPS: resolve and place, but a container does not stay up. Two classes.
|
|
*
|
|
* A) TOOL-RUNTIME NEEDS AN OPERATOR CREDENTIAL THE MANIFEST DOES NOT WIRE. The mesh-<mod> sidecar
|
|
* cannot construct its client and crash-loops (verbatim below); the SERVER of each is UP — only
|
|
* the tool sidecar is down. This is the umami/photos class from whole-mesh-novox, systemic across
|
|
* the media/home tools whose key a human sets in the app UI rather than one derivable from a
|
|
* config file (sonarr/radarr/lidarr/jackett/tautulli DO self-configure from the app's config file,
|
|
* so their runtimes come up):
|
|
* plex — "no Plex token — set MESH_PLEX_TOKEN or make the data dir readable"
|
|
* bazarr — "no Bazarr API key — set MESH_BAZARR_API_KEY"
|
|
* nzbget — "NZBGet not configured — set MESH_NZBGET_URL and MESH_NZBGET_PASSWORD"
|
|
* qbittorrent — "qBittorrent not configured — set MESH_QBITTORRENT_URL and MESH_QBITTORRENT_PASSWORD"
|
|
* ombi — "no Ombi API key — set MESH_OMBI_API_KEY"
|
|
* home-assistant — "no Home Assistant token — set MESH_HOMEASSISTANT_TOKEN"
|
|
*
|
|
* B) APP MIGRATION AGAINST POSTGRES.
|
|
* letta — the letta APP's DB migration fails on first boot ("connection to server at
|
|
* ace.internal … port 5432 failed: Connection refused"); baserow, the other postgres
|
|
* consumer, comes up over the same overlay path, so this is letta's own startup
|
|
* ordering / lack of retry. The deeper blocker once it connects is the postgres `vector`
|
|
* (pgvector) extension a non-superuser consumer cannot CREATE — documented the same way
|
|
* in assigned-two-node-db.test.ts. Its runtime mesh-letta and its credential are fine.
|
|
*/
|
|
const KNOWN_GAPS = new Set([
|
|
"plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta",
|
|
]);
|
|
|
|
/**
|
|
* Host-port remaps applied at load time to break the co-located host-port collisions. In this set
|
|
* three servers claim :8080 (qbittorrent, searxng, the UniFi controller) and two claim :6789 (nzbget,
|
|
* the UniFi controller). UniFi keeps its published ports; qbittorrent/searxng/nzbget are remapped.
|
|
* Container ports are preserved; only the host side changes.
|
|
*/
|
|
const REMAP: Record<string, Record<string, string>> = {
|
|
qbittorrent: { "8080": "8090:8080" },
|
|
searxng: { "8080": "8092:8080" },
|
|
nzbget: { "6789": "6790:6789" },
|
|
};
|
|
|
|
let instanceId = "";
|
|
let held: HeldImage[] = [];
|
|
|
|
function quote(s: string): string {
|
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
}
|
|
|
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
const { stdout } = await exec(instanceId, machine, [
|
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
], timeoutMs);
|
|
const marker = stdout.lastIndexOf("__exit=");
|
|
if (marker < 0) return { out: stdout, ok: false };
|
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
}
|
|
|
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
|
const { out, ok } = await on(machine, command, timeoutMs);
|
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
|
return out;
|
|
}
|
|
|
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
}
|
|
|
|
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
function pinned(reference: string): string {
|
|
return onTheMachine(reference, held);
|
|
}
|
|
|
|
function bundleFor(images: HeldImage[]): string {
|
|
return foundationBundle(bundle, images);
|
|
}
|
|
|
|
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
|
const path = resolve(catalogDir, name, "module.json");
|
|
const m = JSON.parse(readFileSync(path, "utf8")) as {
|
|
resources?: { type: string; image?: string; ports?: string[] }[];
|
|
};
|
|
const remap = REMAP[name] ?? {};
|
|
for (const r of m.resources ?? []) {
|
|
if (r.type !== "container") continue;
|
|
if (typeof r.image === "string") r.image = pinned(r.image);
|
|
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
|
|
}
|
|
const manifest = JSON.stringify(m);
|
|
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
|
|
}
|
|
|
|
function tokenFrom(said: string): string {
|
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
assert.ok(found, `no token in:\n${said}`);
|
|
return found;
|
|
}
|
|
|
|
interface NodeState {
|
|
reached: boolean;
|
|
applied: boolean;
|
|
current: boolean;
|
|
waiting: boolean;
|
|
wrong?: { outcome: string; refused?: string | undefined; failed?: { id: string; error: string }[] | undefined } | undefined;
|
|
raw: string;
|
|
}
|
|
|
|
async function nodeState(node: string): Promise<NodeState> {
|
|
const asked = await on("anchor", `docker exec mesh-controller /mesh-controller status --json`);
|
|
if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out };
|
|
let state: {
|
|
wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[];
|
|
waiting: { node: string }[];
|
|
reported: { node: string; outcome: string; current: boolean }[];
|
|
};
|
|
try {
|
|
state = JSON.parse(asked.out);
|
|
} catch {
|
|
return { reached: false, applied: false, current: false, waiting: false, raw: asked.out };
|
|
}
|
|
const word = state.reported.find((r) => r.node === node);
|
|
const bad = state.wrong.find((w) => w.node === node);
|
|
return {
|
|
reached: true,
|
|
applied: word?.outcome === "applied",
|
|
current: !!word?.current,
|
|
waiting: state.waiting.some((w) => w.node === node),
|
|
wrong: bad ? { outcome: bad.outcome, refused: bad.refused, failed: bad.failed } : undefined,
|
|
raw: asked.out,
|
|
};
|
|
}
|
|
|
|
before(async () => {
|
|
if (skip) return;
|
|
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
|
|
|
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
onProgress: (m) => console.log(`raise: ${m}`),
|
|
});
|
|
instanceId = raised.instanceId;
|
|
held = raised.images;
|
|
|
|
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000);
|
|
const up = await must("anchor", `docker ps --format '{{.Names}}'`);
|
|
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
|
}
|
|
|
|
for (const machine of ["anchor", NODE]) {
|
|
await mesh(`node add ${machine}`);
|
|
const token = tokenFrom(await mesh(`token issue --node ${machine}`));
|
|
const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
|
|
assert.match(said, new RegExp(`enrolled as ${machine}`), said);
|
|
await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
}
|
|
|
|
// The operator provides the media library: the ADR-0051 `access` resources CONFIRM these paths and
|
|
// the media containers mount them, but the mesh creates none of it. Without this the media stack's
|
|
// apply is refused ("the path is not present").
|
|
await must(NODE, `mkdir -p ${MEDIA_DIRS.join(" ")}`);
|
|
}, { timeout: 2_700_000 });
|
|
|
|
after(async () => {
|
|
if (instanceId) await destroy(instanceId);
|
|
await destroyAll(`${SCENARIO}-`);
|
|
}, { timeout: 900_000 });
|
|
|
|
test("the whole ace service set resolves, installs and converges on one node in one push", {
|
|
skip, timeout: 3_300_000,
|
|
}, async () => {
|
|
for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`);
|
|
|
|
// The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres/redis).
|
|
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
|
await mesh(`overlay place ${NODE} --site lab`);
|
|
await mesh("assign anchor networking");
|
|
await mesh(`assign ${NODE} networking`);
|
|
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
|
|
// and what a bed raised from the bundle must do itself (ADR 0088; see the harness).
|
|
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
|
|
|
|
// Add / issue / assign, resiliently: a module the node cannot host is recorded and skipped so one
|
|
// bad assignment cannot poison the whole-node push.
|
|
const issued: string[] = [];
|
|
const assigned = new Set<string>();
|
|
const refused: { name: string; why: string }[] = [];
|
|
for (const { name } of MODULES) {
|
|
if (DROPPED.some((d) => d.name === name)) continue;
|
|
try {
|
|
const { manifest, broker } = loadManifest(name);
|
|
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
|
await mesh(`module add /${name}.json`);
|
|
if (broker) {
|
|
await mesh(`module issue ${name} --node ${NODE}`);
|
|
issued.push(name);
|
|
}
|
|
await mesh(`assign ${NODE} ${name}`);
|
|
assigned.add(name);
|
|
} catch (err) {
|
|
const why = (err as Error).message.split("\n").map((l) => l.trim()).filter(Boolean).slice(1, 5).join(" | ");
|
|
refused.push({ name, why });
|
|
console.log(`NOT ASSIGNED ${name}: ${why}`);
|
|
}
|
|
}
|
|
console.log(`issued broker accounts for: ${issued.length} modules`);
|
|
if (refused.length) console.log(`refused (node cannot host): ${refused.map((r) => r.name).join(", ")}`);
|
|
|
|
// ONE push.
|
|
let pushError = "";
|
|
try {
|
|
await mesh(`push ${NODE}`, 180_000);
|
|
} catch (err) {
|
|
pushError = (err as Error).message;
|
|
console.log(`PUSH REJECTED:\n${pushError}`);
|
|
}
|
|
|
|
// Wait for every CORE container to be up (the node pulls ~20GiB of images first), bounded.
|
|
const coreContainers = MODULES.filter((m) => CORE.has(m.name) && assigned.has(m.name))
|
|
.flatMap((m) => m.containers);
|
|
const psNames = async (): Promise<Map<string, string>> => {
|
|
const out = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
|
const map = new Map<string, string>();
|
|
for (const line of out.split("\n")) {
|
|
const [n, ...rest] = line.split("\t");
|
|
if (n) map.set(n.trim(), rest.join("\t").trim());
|
|
}
|
|
return map;
|
|
};
|
|
let psMap = new Map<string, string>();
|
|
if (!pushError) {
|
|
const until = Date.now() + 2_700_000;
|
|
while (Date.now() < until) {
|
|
psMap = await psNames();
|
|
if (coreContainers.every((c) => (psMap.get(c) ?? "").startsWith("Up"))) break;
|
|
await new Promise((r) => setTimeout(r, 8000));
|
|
}
|
|
await new Promise((r) => setTimeout(r, 20000)); // let first-boot bounces settle
|
|
}
|
|
psMap = await psNames();
|
|
const final = await nodeState(NODE);
|
|
const running = (name: string): boolean => (psMap.get(name) ?? "").startsWith("Up");
|
|
// A run-once (mosquitto's dynsec bootstrap) seeds state and exits; the mesh removes it on success,
|
|
// so absent-from-`docker ps -a` means it completed and was reaped (the node is applied+current, so
|
|
// its resource did apply). Present means it must be up or have exited cleanly.
|
|
const ranOnce = (name: string): boolean => !psMap.has(name) || /^(Up|Exited \(0\))/.test(psMap.get(name) ?? "");
|
|
|
|
const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out;
|
|
|
|
// ================================================================================================
|
|
// The per-module report — the deliverable.
|
|
// ================================================================================================
|
|
const report: string[] = [];
|
|
report.push("================ WHOLE-MESH ace CONVERGENCE ================");
|
|
report.push(`node reached=${final.reached} applied=${final.applied} current=${final.current} waiting=${final.waiting}`);
|
|
if (pushError) report.push(`PUSH REJECTED (resolver): ${pushError.split("\n").slice(0, 8).join("\n ")}`);
|
|
const failedResources = final.wrong?.failed ?? [];
|
|
if (final.wrong) {
|
|
report.push(`NODE WRONG: outcome=${final.wrong.outcome} refused=${final.wrong.refused ?? "-"}`);
|
|
for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`);
|
|
}
|
|
if (DROPPED.length) {
|
|
report.push("---- DROPPED ----");
|
|
for (const d of DROPPED) report.push(` ${d.name.padEnd(16)} ${d.why}`);
|
|
}
|
|
if (refused.length) {
|
|
report.push("---- REFUSED at assign ----");
|
|
for (const r of refused) report.push(` ${r.name.padEnd(16)} ${r.why}`);
|
|
}
|
|
const line = (mod: typeof MODULES[number]): { text: string; ok: boolean } => {
|
|
const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`);
|
|
const extra = (mod.runOnce ?? []).map((c) => `${c}:${ranOnce(c) ? "ran" : (psMap.get(c) ?? "MISSING")}`);
|
|
const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce);
|
|
return { text: `${mod.name.padEnd(16)} ${ok ? "OK " : "GAP "} ${[...states, ...extra].join(" ")}`, ok };
|
|
};
|
|
report.push("---- CORE (gates green) ----");
|
|
const coreFailures: string[] = [];
|
|
const gaps: string[] = [];
|
|
for (const mod of MODULES) {
|
|
if (!assigned.has(mod.name)) continue;
|
|
const { text, ok } = line(mod);
|
|
if (CORE.has(mod.name)) {
|
|
report.push(` ${text}`);
|
|
if (!ok) coreFailures.push(mod.name);
|
|
} else {
|
|
gaps.push(` ${text}`);
|
|
}
|
|
}
|
|
report.push("---- KNOWN GAPS (reported, escalated, do NOT gate green) ----");
|
|
for (const g of gaps) report.push(g);
|
|
report.push(`broker accounts issued: ${issued.filter((n) => new RegExp(`${NODE}-${n}\\b`).test(users)).length}/${issued.length} present`);
|
|
const summary = report.join("\n");
|
|
console.log(summary);
|
|
|
|
// Diagnostics for anything not up: exact crash cause per container.
|
|
const toDump = MODULES.filter((m) => assigned.has(m.name) && (coreFailures.includes(m.name) || KNOWN_GAPS.has(m.name)));
|
|
if (toDump.length) {
|
|
console.log(`\n---- ${NODE} mesh-host.log tail ----\n${(await on(NODE, `tail -60 /var/log/mesh-host.log`)).out}`);
|
|
for (const mod of toDump) {
|
|
for (const c of mod.containers) {
|
|
if (psMap.has(c) && !running(c)) {
|
|
console.log(`\n---- logs: ${c} (${psMap.get(c)}) ----\n${(await on(NODE, `docker logs ${c} 2>&1 | tail -20`)).out}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
// GREEN = the whole set RESOLVED (push accepted), every CORE module converged whole, and no CORE
|
|
// resource failed to apply. KNOWN_GAPS and DROPPED are reported and escalated but do not gate.
|
|
// ================================================================================================
|
|
assert.equal(pushError, "", `the whole set did not resolve — push was rejected:\n${pushError}`);
|
|
const coreResourceFailures = failedResources.filter((f) => CORE.has(f.id.split(".")[0] ?? ""));
|
|
assert.deepEqual(coreResourceFailures, [],
|
|
`a CORE resource failed to apply:\n${coreResourceFailures.map((f) => `${f.id}: ${f.error}`).join("\n")}\n${summary}`);
|
|
assert.deepEqual(coreFailures, [],
|
|
`these CORE modules did not converge whole: ${coreFailures.join(", ")}\n${summary}`);
|
|
});
|