/** * A MACHINE IN USE IS ADOPTED BEFORE IT IS CONVERGED (novox/hq ADR 0100, and ADR 0101 on what * "in use" ignores). * * The mesh replaces a predecessor running on the same machines. This bed prepares a machine the * way the predecessor leaves one — the record's own words, "How it is checked": * * - its firewall (ufw) allowing a served port and denying the rest, incoming and routed, with the * predecessor's published container ports filtered through it (the ufw-docker arrangement); * - a service container, `hello-web`, listening on that port under a name the catalogue's * `hello-web` module also uses, and a file at a path that module declares; * - a stand-in for the predecessor's control that rewrites that file, stopped by the operator * before adoption as the record prescribes, and started again later to play one forgotten; * - a container holding the registry's port. * * Then it asks, in the record's order: a converged genesis refuses; an adopted one refuses the held * registry port and comes up on another; nothing that serves changed; the store is unreachable * from outside and reachable where it must be; the mesh works through the found firewall, across a * reload and a reboot; a predecessor still writing is caught; assigning prepares and taking cuts * over; converging previews, refuses while a found container is held, flips, and returns. * * **The module under migration is the catalogue's `hello-web` with its route requirement taken * off**, read from the catalogue (never a copy): the route needs a proxy module and a public name, * neither of which is what adoption is about. Its names — the container, the file, the port — are * the catalogue's, which is the point: they are what the predecessor also uses. * * **The forge is in the lab.** Genesis builds the control plane and the catalogue from a * repository and a commit; this bed serves the checkouts it was pointed at (their HEADs) from the * `outsider` machine, so the run builds exactly the code under test and nothing on the workstation * listens for the lab. * * Each step is recorded rather than allowed to throw; a step whose dependency failed is not * attempted, and the report says which. * * MESH_LAB_INCUS='sudo -n incus' * MESH_LAB_HOST_BINARY=/mesh-host MESH_LAB_BOOTSTRAP_BINARY=/mesh-bootstrap * MESH_LAB_BUNDLE=/examples/foundation-first-node.lock * MESH_LAB_CATALOG=/modules MESH_LAB_MODULES=/examples/modules * MESH_TOOLS= MESH_SDK= (default: the checkouts beside this one) * MESH_LAB_KEEP=1 leave it standing MESH_LAB_WARM=1 iterate from the adopted foundation */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { execFileSync } from "node:child_process"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec, push, instanceNameOf } from "../../src/lifecycle/operate.ts"; import { bootstrapBinaryPath, hostBinaryPath, placeBootstrap, HOST_PATH } from "../../src/lifecycle/place.ts"; import { waitUntilAllUsable } from "../../src/lifecycle/ready.ts"; import { incus } from "../../src/incus/client.ts"; import { catalogueRoot } from "../../src/repos.ts"; import { ready, returnTo, keep } from "../../src/warm.ts"; import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueManifest } from "./harness.ts"; import { genesis, type GenesisOptions } from "./genesis.ts"; const SCENARIO = "adoption"; const CONTROL = "anchor"; const JOINER = "joiner"; const OUTSIDER = "outsider"; const ANCHOR = "192.0.2.10"; const JOINER_ADDRESS = "192.0.2.20"; const FORGE = "192.0.2.30"; /** The port the predecessor serves, and the module that also names its container and file. */ const SERVED = 8080; const SERVICE = "hello-web"; const SERVICE_FILE = "/var/lib/hello-web/index.html"; const PREDECESSOR_PAGE = "hello from the predecessor\n"; /** The registry's port, which the predecessor holds — and the one the mesh is given instead. */ const HELD_REGISTRY = 5000; const REGISTRY_PORT = 5100; const STORE_PORT = 5432; const BUS_PORT = 5671; const HUB_PORT = 51820; const NETWORK_MODULE = "networking"; const FILTER_MODULE = "nftables"; /** Upstream images, pinned as the catalogue pins them (the harness's table). */ const ALPINE = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"; const REGISTRY_IMAGE = "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const installer = bootstrapBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; const modulesDir = process.env["MESH_LAB_MODULES"] ?? ""; const KEEP = !!process.env["MESH_LAB_KEEP"]; const WARM = !!process.env["MESH_LAB_WARM"]; const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "adoption-live" : undefined); /** The checkouts this run builds from, served by the lab's forge. */ const REPOS: Record = { "mesh-controller": modulesDir ? dirname(dirname(modulesDir)) : "", "mesh-catalog": catalogDir ? catalogueRoot(catalogDir) : "", "mesh-tools": process.env["MESH_TOOLS"] ?? resolve(process.cwd(), "..", "mesh-tools"), "mesh-sdk": process.env["MESH_SDK"] ?? resolve(process.cwd(), "..", "mesh-sdk"), }; const skip = !capability.usable ? capability.why : !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" : !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" : !modulesDir ? "MESH_LAB_MODULES is not set, so there is no control-plane checkout to build from" : Object.entries(REPOS).find(([, p]) => !p || !existsSync(resolve(p, ".git"))) ?.map((x) => `no checkout of ${x[0]} at ${x[1]}`)[0] ?? false; let instanceId = ""; // ---- talking to the machines ------------------------------------------------------------------ function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { const { stdout } = await exec(instanceId, machine, [ "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, ], timeoutMs); const marker = stdout.lastIndexOf("__exit="); if (marker < 0) return { out: stdout, ok: false }; return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; } async function must(machine: string, command: string, timeoutMs?: number): Promise { const { out, ok } = await on(machine, command, timeoutMs); if (!ok) throw new Error(`${machine}: ${command}\n${out}`); return out; } /** The control plane, retried across the brief windows in which the mesh recreates it. */ async function meshSays(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { const deadline = Date.now() + (timeoutMs ?? 120_000); for (;;) { const r = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); if (r.ok) return r; if (/is not running|No such container|No such exec instance|Cannot connect to the Docker daemon|is restarting/i.test(r.out) && Date.now() < deadline) { await sleep(2_000); continue; } return r; } } async function mesh(command: string, timeoutMs?: number): Promise { const r = await meshSays(command, timeoutMs); if (!r.ok) throw new Error(`${CONTROL}: mesh-controller ${command}\n${r.out}`); return r.out; } function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } /** Poll until `probe` returns a value, or fail naming the last thing it saw. */ async function until(what: string, seconds: number, probe: () => Promise, last: () => string): Promise { const deadline = Date.now() + seconds * 1000; for (;;) { const got = await probe(); if (got !== null) return got; if (Date.now() > deadline) throw new Error(`${what} — not within ${seconds}s. Last:\n${last()}`); await sleep(5_000); } } /** Whether a TCP connection from `machine` to address:port opens within three seconds. */ async function connects(machine: string, address: string, port: number): Promise { return (await on(machine, `timeout 4 bash -c ${quote(` { const local = join(tmpdir(), `mesh-lab-adoption-${process.pid}-${module}.json`); writeFileSync(local, manifest); await push(instanceId, CONTROL, local, `/tmp/${module}.json`); await must(CONTROL, `docker cp /tmp/${module}.json mesh-controller:/${module}.json`); return mesh(`module add /${module}.json`); } async function nodeShow(node: string): Promise { return mesh(`node show ${node}`); } /** The anchor's address on the private network. */ async function meshAddressOf(machine: string): Promise { const out = await must(machine, `ip -4 -o addr show dev mesh0`); const found = out.match(/inet (\d+\.\d+\.\d+\.\d+)\//)?.[1]; assert.ok(found, `${machine} has no address on mesh0:\n${out}`); return found; } /** The rules ufw was given, one per line, as `ufw show added` prints them. */ async function ufwAdded(): Promise { const out = await must(CONTROL, `ufw show added`); return out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw ")); } function marked(rule: string): boolean { return /comment 'mesh-host /.test(rule); } async function restartMachine(machine: string): Promise { const name = await instanceNameOf(instanceId, machine); await incus(["restart", name], 180_000); await waitUntilAllUsable([name], 300, (m) => console.log(` restart: ${m}`)); } /** Until the anchor reports what it was last sent as applied and current. */ async function settled(node = CONTROL, withinMs = 300_000): Promise { let last = ""; await until(`${node} reports the declaration it was sent as applied and current`, withinMs / 1000, async () => { const asked = await meshSays(`status --json`); last = asked.out; if (!asked.ok) return null; try { const state = JSON.parse(asked.out) as { wrong: { node: string; outcome: string }[]; waiting: { node: string }[]; reported: { node: string; outcome: string; current: boolean }[]; }; const bad = state.wrong.find((w) => w.node === node); if (bad) throw new Error(`${node} did not apply what it was sent: ${bad.outcome}\n${asked.out}`); const word = state.reported.find((r) => r.node === node); return !state.waiting.some((w) => w.node === node) && word?.outcome === "applied" && word.current ? true : null; } catch (err) { if (err instanceof Error && err.message.includes("did not apply")) throw err; return null; } }, () => last); } /** Send a node what it should be, and wait until it says it applied it. */ async function pushAndSettle(node: string): Promise { const said = await mesh(`push ${node}`, 600_000); await settled(node); return said; } function tokenFrom(said: string): string { const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); assert.ok(found, `no token in:\n${said}`); return found; } // ---- the lab's forge --------------------------------------------------------------------------- /** * Serve the checkouts under test from the outsider machine, over git's own protocol. * * Each checkout's HEAD is pushed into a bare repository as `main` and `lab`, the lot is carried in, * and a git daemon answers on the outsider's scenario address — which the anchor's builder reaches * over the hosting segment. Returns the commit each repository is served at. */ async function raiseForge(): Promise> { const dir = mkdtempSync(join(tmpdir(), "mesh-lab-forge-")); const heads: Record = {}; try { for (const [name, checkout] of Object.entries(REPOS)) { const bare = join(dir, `${name}.git`); execFileSync("git", ["init", "-q", "--bare", bare]); execFileSync("git", ["-C", checkout, "push", "-q", "--force", bare, "HEAD:refs/heads/main", "HEAD:refs/heads/lab"], { stdio: "pipe" }); heads[name] = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); } const tar = join(tmpdir(), `mesh-lab-forge-${process.pid}.tar`); execFileSync("tar", ["-cf", tar, "-C", dir, "."]); await push(instanceId, OUTSIDER, tar, "/tmp/forge.tar"); rmSync(tar, { force: true }); } finally { rmSync(dir, { recursive: true, force: true }); } await must(OUTSIDER, `command -v git >/dev/null || pacman -S --noconfirm --needed git`, 600_000); // Owned by the daemon's user: extracted as the workstation's uid, git refuses to serve a // repository someone else owns ("dubious ownership"), and the clone fails with no reason given. await must(OUTSIDER, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` + `git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE} --pid-file=/run/git-daemon.pid`); await must(OUTSIDER, `git ls-remote git://${FORGE}/mesh-controller.git lab`); await until("the lab's forge answers the anchor", 60, async () => (await connects(CONTROL, FORGE, 9418)) ? true : null, () => "no connection to 9418"); return heads; } const forgeUrl = (repo: string) => `git://${FORGE}/${repo}.git`; // ---- the predecessor --------------------------------------------------------------------------- /** * The ufw-docker arrangement: published container ports pass through ufw's route rules, and * traffic from private ranges is let through. It is how a ufw machine filters what docker publishes * at all — without it docker's own rules bypass ufw entirely (novox/hq research 012 measured 52 * forwarding rules on the control-node, one per served port). */ const UFW_DOCKER = ` # BEGIN UFW AND DOCKER *filter :ufw-user-forward - [0:0] :ufw-docker-logging-deny - [0:0] :DOCKER-USER - [0:0] -A DOCKER-USER -j ufw-user-forward -A DOCKER-USER -j RETURN -s 10.0.0.0/8 -A DOCKER-USER -j RETURN -s 172.16.0.0/12 -A DOCKER-USER -j RETURN -s 192.168.0.0/16 -A DOCKER-USER -p udp -m udp --sport 53 --dport 1024:65535 -j RETURN -A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 192.168.0.0/16 -A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 10.0.0.0/8 -A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 172.16.0.0/12 -A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 192.168.0.0/16 -A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 10.0.0.0/8 -A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 172.16.0.0/12 -A DOCKER-USER -j RETURN -A ufw-docker-logging-deny -j DROP COMMIT # END UFW AND DOCKER `; /** The stand-in for the predecessor's configuration sync: it rewrites the file every ten seconds. */ const STAND_IN = `[Unit] Description=Stand-in for the predecessor's configuration sync: rewrites a file a catalogue module declares [Service] ExecStart=/bin/sh -c 'while true; do printf "hello from the predecessor, synced %%s\\\\n" "$(date +%%s)" > ${SERVICE_FILE}; sleep 10; done' `; /** The page the predecessor's service loop serves — the catalogue module's own loop, verbatim. */ const SERVE_LOOP = "while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done"; /** Where the bed keeps what the machine looked like before the mesh arrived — on the machine, so a warm restore keeps it. */ const BEFORE = "/root/predecessor"; /** A predecessor container with no restart policy: a runtime restart would lose it. */ const NO_POLICY = "predecessor-nopolicy"; /** The broker's plaintext port: published on every interface, and the filter admits it from the mesh only. */ const AMQP_PORT = 5672; async function preparePredecessor(): Promise { const said: string[] = []; await must(CONTROL, `pacman -S --noconfirm --needed ufw`, 600_000); const rules = join(tmpdir(), `mesh-lab-ufw-docker-${process.pid}`); writeFileSync(rules, UFW_DOCKER); await push(instanceId, CONTROL, rules, "/tmp/ufw-docker.rules"); await must(CONTROL, [ `grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules || cat /tmp/ufw-docker.rules >> /etc/ufw/after.rules`, `ufw default deny incoming`, `ufw default allow outgoing`, `ufw default deny routed`, `ufw allow 22/tcp`, `ufw allow ${SERVED}/tcp`, `ufw route allow proto tcp from any to any port ${SERVED}`, `ufw --force enable`, `systemctl enable ufw`, ].join(" && ")); said.push(` firewall ufw: deny incoming and routed; allow 22 and ${SERVED}; ufw-docker after.rules`); await must(CONTROL, `mkdir -p /var/lib/hello-web && printf %s ${quote(PREDECESSOR_PAGE)} > ${SERVICE_FILE}`); await must(CONTROL, `docker run -d --name ${SERVICE} --restart unless-stopped -p ${SERVED}:${SERVED} ` + `-v ${SERVICE_FILE}:/www/index.html:ro ${ALPINE} sh -c ${quote(SERVE_LOOP)}`, 600_000); await must(CONTROL, `docker run -d --name predecessor-registry --restart unless-stopped -p ${HELD_REGISTRY}:5000 ${REGISTRY_IMAGE}`, 600_000); // A container the predecessor left running with no restart policy: a restart of the runtime // would not bring it back, which is what ADR 0102 forbids the mesh from causing. await must(CONTROL, `docker run -d --name ${NO_POLICY} ${ALPINE} sleep infinity`, 600_000); // And a setting of the machine's own in the runtime's file, beside the registries it ships with. await must(CONTROL, `python3 - <<'EOF' import json f="/etc/docker/daemon.json" d=json.load(open(f)) d["log-opts"]={"max-size":"7m"} json.dump(d,open(f,"w"),indent=2) EOF systemctl reload docker`); said.push(` containers ${SERVICE} on ${SERVED}, predecessor-registry on ${HELD_REGISTRY}`); // The predecessor's control, which the operator stops before adopting (the record's words). const unit = join(tmpdir(), `mesh-lab-stand-in-${process.pid}`); writeFileSync(unit, STAND_IN); await push(instanceId, CONTROL, unit, "/etc/systemd/system/predecessor-sync.service"); await must(CONTROL, `systemctl daemon-reload && systemctl start predecessor-sync && sleep 12 && systemctl stop predecessor-sync`); said.push(` stand-in predecessor-sync rewrote ${SERVICE_FILE}, then the operator stopped it`); // What the machine was, recorded ON the machine so a restored warm instance still has it. await must(CONTROL, [ `mkdir -p ${BEFORE}`, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1 > ${BEFORE}/file.sha256`, `cp ${SERVICE_FILE} ${BEFORE}/file`, `docker inspect -f '{{.Id}}' ${SERVICE} > ${BEFORE}/container.id`, `docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY} > ${BEFORE}/nopolicy.id`, `ufw show added > ${BEFORE}/ufw-added.txt`, ].join(" && ")); await until(`the predecessor's ${SERVICE} answers the joiner`, 60, async () => (await on(JOINER, `curl -s --max-time 3 http://${ANCHOR}:${SERVED}/`)).out.includes("hello from the predecessor") ? true : null, () => "no answer"); said.push((await must(CONTROL, `ufw status verbose`)).trim()); said.push((await must(CONTROL, `docker ps --format '{{.Names}}\t{{.Ports}}'`)).trim()); return said.join("\n"); } // ---- steps, recorded rather than thrown -------------------------------------------------------- interface Step { code: string; title: string; ok: boolean; why: string; said: string; seconds: number; warm?: boolean } const steps = new Map(); async function step(code: string, needs: string[], fn: () => Promise): Promise { const title = TITLE[code]!; const missing = needs.filter((n) => !steps.get(n)?.ok); if (missing.length) { steps.set(code, { code, title, ok: false, seconds: 0, said: "", why: `not attempted — ${missing.join(", ")} did not succeed` }); console.log(`[${code}] SKIP ${title}`); return; } console.log(`\n[${code}] ---- ${title} ----`); const began = Date.now(); const took = () => Math.round((Date.now() - began) / 1000); try { const said = await fn(); steps.set(code, { code, title, ok: true, why: "", said, seconds: took() }); console.log(`${said}\n[${code}] PASS ${title} (${took()}s)`); } catch (err) { const why = (err as Error).message; steps.set(code, { code, title, ok: false, why, said: "", seconds: took() }); console.log(`[${code}] FAIL ${title} (${took()}s)\n${why.split("\n").slice(0, 40).join("\n")}`); } } /** The plan, in the record's order. Codes are stable; titles may be reworded. */ const TITLE: Record = { P0: "the machine is prepared the way the predecessor leaves one", F0: "a converged genesis on a FRESH machine is not refused — its own resolver does not make it in use (ADR 0101)", A1: "a converged genesis refuses the machine in use, naming every container and listener it counted", A2: "an adopted genesis refuses the registry's held port, naming what holds it", A3: "given another registry port, the adopted foundation comes up — and stays on that port as modules", B1: "nothing that serves changed: the service answers, its file and container are untouched, the firewall gained only the mesh's marked rules", B2: "the store is unreachable from outside, before and after the found firewall reloads; the bus answers a machine not yet enrolled", B4: "the guard lets the machine's own containers reach the store (with the found firewall admitting them)", C1: "a second machine enrols through the found firewall and joins the private network", B3: "the store is reachable over the private network", C2: "after the found firewall reloads, the openings are there and the mesh still works", C3: "after the machine reboots, the openings are there and the mesh still works", D1: "assigning prepares: the module holds the found container and file, and neither changes", D2: "a predecessor still writing is caught: the held file's change is reported, and not reverted", D3: "converging refuses while the service's module holds its found container", D4: "taking cuts over: the found container and file are replaced, the original kept, the port reachable", E1: "converging previews: the service's port, a published port no rule mentions, and the modules it takes", E2: "the flip: the derived filter loaded, the found firewall disabled with its configuration on disk, the declared port open and the undeclared closed", E3: "returned to adopted: the found firewall enabled again, the derived filter gone, the openings back", F1: "unassigning takes the mesh's opening away and leaves the operator's own rule", }; function stateOutcome(): void { console.log(`\n================ ADOPTION: WHAT WAS ESTABLISHED ================`); let established = 0; for (const code of Object.keys(TITLE)) { const s = steps.get(code); const mark = !s ? "NEVER" : s.warm ? "WARM" : s.ok ? "PASS" : s.why.startsWith("not attempted") ? "SKIP" : "FAIL"; if (s?.ok) established++; console.log(` ${code.padEnd(3)} ${mark.padEnd(5)} ${TITLE[code]}${s?.seconds ? ` (${s.seconds}s)` : ""}`); } console.log(` ${established}/${Object.keys(TITLE).length} established.`); } // ---- the run ----------------------------------------------------------------------------------- before(async () => { if (skip) return; console.log(`\n================ THE PLAN ================`); for (const [code, title] of Object.entries(TITLE)) console.log(` ${code.padEnd(3)} ${title}`); const verdict = WARM ? await ready(SCENARIO) : { use: "raise" as const, why: "not asked to be warm" }; let restored = false; if (verdict.use === "restore") { instanceId = verdict.instanceId; const seconds = await returnTo(instanceId, (m) => console.log(`warm: ${m}`)); console.log(`WARM: restored ${instanceId} to the adopted foundation in ${seconds}s`); restored = true; for (const code of ["P0", "F0", "A1", "A2", "A3"]) { steps.set(code, { code, title: TITLE[code]!, ok: true, warm: true, seconds: 0, why: "", said: "restored from the warm snapshot — not re-run; only a fresh run proves it" }); } } else { if (WARM) console.log(`WARM: raising — ${verdict.why}`); const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), }); instanceId = bed.instanceId; } console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); let heads: Record = {}; const genesisOn = (node: string, o: Partial): Promise extends Promise ? R : never> => genesis({ instanceId, node, installer: installer as string, catalogDir, bundleTemplate: foundationBundle(bundle, []), registry: `${ANCHOR}:${HELD_REGISTRY}`, source: forgeUrl("mesh-controller"), sourceRef: heads["mesh-controller"] ?? "", toolsSource: forgeUrl("mesh-tools"), toolsRef: "lab", catalogSource: forgeUrl("mesh-catalog"), catalogRef: "lab", sdkSource: forgeUrl("mesh-sdk"), sdkRef: "lab", site: "hosting", hostService: false, ...(binary ? { hostBinary: binary } : {}), log: (m) => console.log(m), ...o, }); if (!restored) { await step("P0", [], async () => { heads = await raiseForge(); const said = [` forge git://${FORGE}/ serving ${Object.entries(heads).map(([n, h]) => `${n}@${h.slice(0, 8)}`).join(", ")}`]; said.push(await preparePredecessor()); return said.join("\n"); }); // ADR 0101: the joiner is a freshly installed machine — nothing but its operating system, a // container runtime and the host binary. A converged genesis there must not be refused. Asked // as a dry run: the question is the preflight's, and a real raise would make it a second mesh. await step("F0", ["P0"], async () => { const ran = await genesisOn(JOINER, { flags: ["--dry-run"], attempts: 1, verify: false }); assert.doesNotMatch(ran.said, /this machine is in use/, `a converged genesis refused a fresh machine as in use:\n${ran.said}`); assert.match(ran.said, /in use\s+no: no container runs and nothing listens beyond ssh/, `the installer never said the fresh machine is not in use:\n${ran.said}`); const listening = (await must(JOINER, `ss -Hltunp`)).trim(); return ` in use no — the installer went on (${ran.ok ? "dry run finished" : `dry run stopped later, at ${ran.step}`})\n` + ` what listens on the fresh machine:\n${listening.split("\n").map((l) => ` ${l}`).join("\n")}`; }); await step("A1", ["P0"], async () => { const ran = await genesisOn(CONTROL, { attempts: 1, verify: false }); assert.ok(!ran.ok, `a converged genesis went ahead on a machine in use:\n${ran.said}`); assert.match(ran.step, /preflight/, `it was refused, but not before changing anything (at ${ran.step}):\n${ran.said}`); for (const want of [/container hello-web/, /container predecessor-registry/, new RegExp(`tcp \\S+:${SERVED} by`), new RegExp(`tcp \\S+:${HELD_REGISTRY} by`)]) { assert.match(ran.said, want, `the refusal does not name ${want}:\n${ran.said}`); } assert.match(ran.said, /--adopted/, `the refusal does not say how to raise it adopted`); // And nothing was changed: the predecessor as it was, no table of the mesh's. const ps = await must(CONTROL, `docker ps --format '{{.Names}}'`); assert.deepEqual(ps.trim().split("\n").sort(), [SERVICE, NO_POLICY, "predecessor-registry"].sort(), `containers changed:\n${ps}`); assert.match(await must(CONTROL, `ufw status`), /Status: active/); assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`); return ran.said.split("\n").filter((l) => /in use|^\s+- /.test(l)).join("\n"); }); await step("A2", ["A1"], async () => { const ran = await genesisOn(CONTROL, { adopted: true, attempts: 1, verify: false }); assert.ok(!ran.ok, `an adopted genesis went ahead with the registry's port held:\n${ran.said}`); assert.match(ran.said, new RegExp(`registry's port tcp/${HELD_REGISTRY} is held by [^\\n]*predecessor-registry`), `the refusal does not name what holds the registry's port:\n${ran.said.split("\n").slice(-15).join("\n")}`); assert.match(ran.said, /--registry-port/, `the refusal does not say which flag gives another port`); const id = (await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(); assert.equal(id, (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the predecessor's container was replaced`); assert.match(await must(CONTROL, `ufw status`), /Status: active/); assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`); return ` refused at ${ran.step}\n` + ran.said.split("\n").filter((l) => /held by|port|adopted/.test(l)).slice(-8).join("\n"); }); await step("A3", ["A2"], async () => { const ran = await genesisOn(CONTROL, { adopted: true, registry: `${ANCHOR}:${REGISTRY_PORT}` }); if (!ran.ok) throw new Error(`${ran.step}: ${ran.why}\n\n${ran.report.join("\n")}`); await settled(); const said = [ran.report.join("\n")]; const bindings = await must(CONTROL, `docker inspect -f '{{json .HostConfig.PortBindings}}' mesh-registry`); assert.match(bindings, new RegExp(`"HostPort":"${REGISTRY_PORT}"`), `the registry is not on ${REGISTRY_PORT}: ${bindings}`); assert.match(await must(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' mesh-registry`), /\S/, `mesh-registry is not the host's: the foundation was not adopted as a module`); assert.match(await mesh(`module list`), /^distribution\b/m, `the registry is not a module the mesh holds`); // The node's own port, in what the mesh would send it — not the catalogue's default. const plan = await mesh(`plan ${CONTROL} --json`); assert.match(plan, new RegExp(`"${REGISTRY_PORT}:5000"`), `the registry's module does not publish ${REGISTRY_PORT}`); assert.doesNotMatch(plan, /"5000:5000"/, `the plan still publishes the catalogue's 5000`); said.push(` registry on ${REGISTRY_PORT}, as the module the mesh holds: ${bindings.trim()}`); const show = await nodeShow(CONTROL); assert.match(show, /mode\s+adopted since/, `the anchor is not reported adopted:\n${show}`); assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `the foundation's dropping table was loaded on an adopted node`); const guard = await must(CONTROL, `nft list table inet mesh_guard`); // The guard's port set is the controller's to derive; what the record fixes is that it refuses // the store's port from outside and holds nothing but refusals. assert.match(guard, new RegExp(`dport (\\{[^}]*\\b${STORE_PORT}\\b[^}]*\\}|${STORE_PORT}) drop`), `the guard does not refuse the store's port:\n${guard}`); assert.doesNotMatch(guard, /accept\s*$/m, `the guard holds an accept:\n${guard}`); assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force`); assert.match(await must(CONTROL, `curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${HELD_REGISTRY}/v2/`), /200/, `the predecessor's registry stopped answering`); said.push(show.trim(), guard.trim()); return said.join("\n"); }); if (WARM && steps.get("A3")?.ok) { await keep(SCENARIO, instanceId); console.log(`WARM: kept ${instanceId} at the adopted foundation`); } } // ---- nothing that serves changed ------------------------------------------------------------- await step("B1", ["A3"], async () => { const said: string[] = []; const want = await must(CONTROL, `cat ${BEFORE}/file`); let page = ""; await until(`the service answers the joiner as it did`, 120, async () => { page = (await on(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`)).out; return page === want ? true : null; }, () => page); said.push(` ${SERVICE} answers the joiner on ${SERVED} with the predecessor's page`); assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(), (await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `${SERVICE_FILE} changed`); assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(), (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the ${SERVICE} container was replaced`); said.push(` file, container byte for byte / the same id as before the mesh`); const was = (await must(CONTROL, `cat ${BEFORE}/ufw-added.txt`)).split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw ")); const now = await ufwAdded(); const lost = was.filter((r) => !now.includes(r)); const added = now.filter((r) => !was.includes(r)); assert.deepEqual(lost, [], `the found firewall lost rules:\n${lost.join("\n")}`); const unmarked = added.filter((r) => !marked(r)); assert.deepEqual(unmarked, [], `the found firewall gained rules not marked as the mesh's:\n${unmarked.join("\n")}`); assert.ok(added.length > 0, `the mesh opened nothing through the found firewall`); // ADR 0102: the runtime's file is written into, never over, and the runtime is reloaded. const daemon = JSON.parse(await must(CONTROL, `cat /etc/docker/daemon.json`)) as { "log-opts"?: Record; "insecure-registries"?: string[] }; assert.equal(daemon["log-opts"]?.["max-size"], "7m", `the machine's own runtime setting was replaced: ${JSON.stringify(daemon)}`); assert.ok((daemon["insecure-registries"] ?? []).some((r) => r.includes(":")), `the mesh's registry trust is not in the runtime's file: ${JSON.stringify(daemon)}`); assert.ok((daemon["insecure-registries"] ?? []).length > 1, `the machine's own registries were replaced rather than added to: ${JSON.stringify(daemon)}`); const stillUp = (await must(CONTROL, `docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY}`)).trim(); assert.match(stillUp, /^true /, `the container with no restart policy is not running: ${stillUp}`); assert.equal(stillUp, (await must(CONTROL, `cat ${BEFORE}/nopolicy.id`)).trim(), `the container with no restart policy was restarted or replaced`); said.push(` runtime file the machine's log-opts kept, the mesh's registry added; ${NO_POLICY} still up`); said.push(` firewall ${was.length} rule(s) kept, ${added.length} added, every one marked:`, ...added.map((r) => ` ${r}`)); return said.join("\n"); }); await step("B2", ["A3"], async () => { const said: string[] = []; assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers a machine off the private network`); await must(CONTROL, `ufw reload`); await sleep(3_000); assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the found firewall reloaded`); said.push(` outsider -> ${STORE_PORT} refused, before and after \`ufw reload\``); assert.ok(await connects(OUTSIDER, ANCHOR, BUS_PORT), `the bus does not answer a machine that has not enrolled`); said.push(` outsider -> ${BUS_PORT} the bus answers`); // **What the guard is for** (ADR 0100, 0103): the store must be unreachable from outside // *whatever the found firewall does*. With ufw admitting both ports, only the guard is left // between the outsider and the store — and with the guard gone they are reachable, which is // what makes this a test of the guard rather than of ufw. const admit = [STORE_PORT, AMQP_PORT].map((p) => `ufw route allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`); try { await must(CONTROL, admit.join(" && ")); await sleep(2_000); assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once the found firewall admits it — the guard refuses nothing`); assert.ok(!(await connects(OUTSIDER, ANCHOR, AMQP_PORT)), `the broker's plaintext port answers from outside once the found firewall admits it`); said.push(` outsider -> ${STORE_PORT}, ${AMQP_PORT} still refused with the found firewall admitting both — the guard's own refusal`); // The positive control: without the guard, both answer. Anything else would mean the probe // could not have failed. await must(CONTROL, `nft delete table inet mesh_guard`); await sleep(2_000); const openNow = (await connects(OUTSIDER, ANCHOR, STORE_PORT)) || (await connects(OUTSIDER, ANCHOR, AMQP_PORT)); await must(CONTROL, `systemctl reload mesh-guard.service || systemctl restart mesh-guard.service`); await sleep(2_000); assert.ok(openNow, `neither port answered with the guard deleted, so the guard is not what refuses them`); assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store stayed open after the guard was loaded again`); said.push(` guard deleted both answered; loaded again, both refused`); } finally { await on(CONTROL, [STORE_PORT, AMQP_PORT].map((p) => `ufw route delete allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`).join("; ")); } return said.join("\n"); }); // Its own step: it asks something different of the found firewall — a container on the machine // reaches a published port through the runtime's proxy, on the incoming path, not the forwarded. await step("B4", ["A3"], async () => { const said: string[] = []; // What this asks is the guard's promise: it never refuses the machine's own containers. The // found firewall stays in force (ADR 0100) and denies inbound by default, and a container on // the store's own network reaches its published port through the runtime's proxy — inbound, // not forwarded — so the operator's firewall has to admit the container interface for any // container to get there, on an adopted node as on a converged one. The probe admits it for // itself alone, and takes the rule away again. await must(CONTROL, `ufw allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`); let fromContainer: { ok: boolean; out: string }; try { fromContainer = await on(CONTROL, `docker run --rm ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`, 180_000); } finally { await on(CONTROL, `ufw delete allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`); } if (!fromContainer.ok) { // Evidence, so the cause can be read from this run rather than guessed at the next one. const evidence = await on(CONTROL, [ `echo '--- published'; docker ps --format '{{.Names}} {{.Ports}}' | grep -i ${STORE_PORT}`, `echo '--- from the host'; nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`, `echo '--- from the host network'; docker run --rm --network host ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`, `echo '--- iptables FORWARD, DOCKER-USER, isolation'; iptables -S FORWARD; iptables -S DOCKER-USER; iptables -S | grep -i isolation`, `echo '--- the guard'; nft list table inet mesh_guard`, `echo '--- nat for the port'; iptables -t nat -S | grep ${STORE_PORT}`, ].join("; "), 120_000); assert.fail(`a container on the node cannot reach the store:\n${fromContainer.out}\n${evidence.out}`); } said.push(` container -> ${STORE_PORT} reachable from a container on the node itself`); return said.join("\n"); }); // ---- the mesh works through the found firewall ----------------------------------------------- let anchorOnMesh = ""; await step("C1", ["B2"], async () => { const said: string[] = []; await mesh(`node add ${JOINER}`); const token = tokenFrom(await mesh(`token issue --node ${JOINER}`)); const out = await must(JOINER, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); assert.match(out, new RegExp(`enrolled as ${JOINER}`), out); await must(JOINER, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); said.push(` ${JOINER} enrolled over the bus, through the anchor's ufw`); await mesh(`overlay place ${JOINER} --site hosting`); await mesh(`assign ${JOINER} ${NETWORK_MODULE}`); await pushAndSettle(JOINER); // The hub's peer list changed: the anchor has to be sent it too. await pushAndSettle(CONTROL); anchorOnMesh = await meshAddressOf(CONTROL); await until(`the joiner reaches the anchor over mesh0`, 180, async () => (await on(JOINER, `ping -c1 -W2 ${anchorOnMesh}`)).ok ? true : null, () => "no ping reply"); said.push(` private network the joiner reaches the anchor at ${anchorOnMesh}`); said.push((await must(CONTROL, `wg show mesh0 latest-handshakes`)).trim()); return said.join("\n"); }); await step("B3", ["C1"], async () => { assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store does not answer over the private network`); return ` joiner -> ${anchorOnMesh}:${STORE_PORT} reachable over mesh0`; }); /** The mesh's own rules in the found firewall. */ const openings = async (): Promise => (await ufwAdded()).filter(marked); const assertOpenings = (rules: string[]) => { const text = rules.join("\n"); // By the opening's id, which names the machine's port: a forwarded rule names the CONTAINER's // port (ufw's route rules match after the runtime's translation), so the text may say another. for (const port of [BUS_PORT, REGISTRY_PORT, HUB_PORT, STORE_PORT]) { assert.match(text, new RegExp(`opening-(tcp|udp)-${port}-`), `no opening for ${port} among the mesh's rules:\n${text}`); } }; await step("C2", ["B3"], async () => { const before = await openings(); assertOpenings(before); await must(CONTROL, `ufw reload`); await sleep(3_000); const after = await openings(); assert.deepEqual(after.sort(), before.sort(), `the openings changed across a reload`); assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store stopped answering over mesh0 after the reload`); assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reload`); await pushAndSettle(JOINER); return ` after ufw reload ${after.length} opening(s) in place; the joiner reaches the store over mesh0 and takes a push\n` + after.map((r) => ` ${r}`).join("\n"); }); await step("C3", ["C2"], async () => { const before = await openings(); await restartMachine(CONTROL); await until(`the control plane answers after the reboot`, 300, async () => (await meshSays(`status`)).ok ? true : null, () => "no answer"); assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force after the reboot`); assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard did not come back`); const after = await until(`the openings are there again`, 420, async () => { const now = await openings(); return before.every((r) => now.includes(r)) ? now : null; }, () => "not all openings"); assertOpenings(after); assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reboot`); await until(`the joiner reaches the store over mesh0 again`, 300, async () => (await connects(JOINER, anchorOnMesh, STORE_PORT)) ? true : null, () => "no connection"); await pushAndSettle(JOINER); const page = await must(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`); assert.match(page, /hello from the predecessor/, `the predecessor's service did not come back: ${page}`); return ` after a reboot ufw active, the guard loaded, ${after.length} opening(s); the joiner reaches the store and takes a push`; }); // ---- assigning prepares, taking cuts over ---------------------------------------------------- let kept = ""; await step("D1", ["B1"], async () => { const said: string[] = []; // The catalogue's module, read from the catalogue, with the route requirement taken off: the // route needs a proxy module and a public name, and neither is what adoption is about. const manifest = JSON.parse(catalogueModule(SERVICE, [])) as Record; delete manifest["requires"]; delete manifest["contributes"]; delete manifest["binds"]; await registerModule(SERVICE, JSON.stringify(manifest)); await mesh(`assign ${CONTROL} ${SERVICE}`); await pushAndSettle(CONTROL); const show = await until(`the anchor reports holding ${SERVICE}'s container and file`, 120, async () => { const s = await nodeShow(CONTROL); return /holds container\s+hello-web\b/.test(s) && /holds file\s+\/var\/lib\/hello-web\/index\.html/.test(s) ? s : null; }, () => ""); kept = show.match(/holds file\s+\/var\/lib\/hello-web\/index\.html[^\n]*\n\s*original kept at (\S+)/)?.[1] ?? ""; assert.ok(kept, `the held file's original is not reported kept:\n${show}`); assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(), (await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `assigning changed ${SERVICE_FILE}`); assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(), (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `assigning replaced the ${SERVICE} container`); assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the kept original is not the file as found`); said.push(show.trim()); return said.join("\n"); }); await step("D2", ["D1"], async () => { await must(CONTROL, `systemctl start predecessor-sync`); try { await sleep(15_000); await pushAndSettle(CONTROL); const show = await until(`the anchor reports the held file changed`, 120, async () => { const s = await nodeShow(CONTROL); return /hello-web\/index\.html[^\n]*REWRITTEN/i.test(s) ? s : null; }, () => ""); // Stop the writer first, then hash: while it rewrites every ten seconds, a file the host // reverted in between would still read as the predecessor's a moment later. await must(CONTROL, `systemctl stop predecessor-sync`); const was = await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`); await pushAndSettle(CONTROL); await sleep(5_000); const now = await must(CONTROL, `cat ${SERVICE_FILE}`); assert.match(now, /synced \d+/, `the host reverted what the predecessor wrote:\n${now}`); assert.equal(await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`), was, `the held file changed under a push after the predecessor stopped writing`); return show.trim(); } finally { await on(CONTROL, `systemctl stop predecessor-sync`); } }); await step("D3", ["D1"], async () => { await pushAndSettle(CONTROL); const r = await meshSays(`converge ${CONTROL}`); assert.ok(!r.ok, `converge went ahead while ${SERVICE} holds its found container:\n${r.out}`); assert.match(r.out, new RegExp(`hello-web holds the found container hello-web`), `the refusal does not name the held container:\n${r.out}`); assert.match(r.out, new RegExp(`take ${CONTROL} hello-web`), `the refusal does not say what to do:\n${r.out}`); return r.out.trim(); }); await step("D4", ["D1"], async () => { const said: string[] = []; said.push((await mesh(`take ${CONTROL} ${SERVICE}`)).trim()); await pushAndSettle(CONTROL); const label = await until(`the ${SERVICE} container is the mesh's`, 180, async () => { const l = (await on(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' ${SERVICE}`)).out.trim(); return l && l !== "" ? l : null; }, () => ""); assert.notEqual((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(), (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the found container was not replaced`); const catalogue = (JSON.parse(catalogueModule(SERVICE, [])) as { resources: { id: string; content?: string }[] }) .resources.find((r) => r.id === "page")?.content ?? ""; assert.equal(await must(CONTROL, `cat ${SERVICE_FILE}`), catalogue, `the found file was not replaced with the module's`); assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the original was not kept`); said.push(` replaced container (spec ${label.slice(0, 12)}…) and ${SERVICE_FILE}; original still at ${kept}`); // Either the mesh opened the port, or the predecessor's own rule already admits it — then the // mesh adds nothing and will remove nothing (ADR 0103), and the operator's rule stays theirs. const opened = (await openings()).filter((r) => new RegExp(`opening-tcp-${SERVED}-`).test(r)); const operators = (await ufwAdded()).filter((r) => !marked(r) && new RegExp(`\\b${SERVED}\\b`).test(r)); assert.ok(opened.length > 0 || operators.length > 0, `no opening for ${SERVED} once ${SERVICE} was taken, and no rule of the operator's admits it:\n${(await ufwAdded()).join("\n")}`); assert.ok(operators.length > 0, `the operator's own rule for ${SERVED} is gone:\n${(await ufwAdded()).join("\n")}`); said.push(...opened.map((r) => ` opened ${r}`)); if (opened.length === 0) said.push(` opening ${SERVED} satisfied by the operator's own rule: ${operators.join(" | ")}`); const page = await until(`the taken ${SERVICE} answers over the private network`, 120, async () => { const p = await on(JOINER, `curl -s --max-time 3 http://${anchorOnMesh}:${SERVED}/`); return p.ok && p.out === catalogue ? p.out : null; }, () => ""); said.push(` joiner -> ${SERVED} ${page.trim()}`); const show = await nodeShow(CONTROL); assert.doesNotMatch(show, /holds (container|file)\s+\S*hello-web/, `the anchor still holds what was taken:\n${show}`); return said.join("\n"); }); // ---- converging previews, then changes ------------------------------------------------------- await step("E1", ["D4"], async () => { if (!/^nftables\b/m.test(await mesh(`module list`))) { await registerModule(FILTER_MODULE, JSON.stringify(JSON.parse(catalogueModule(FILTER_MODULE, [])))); } // What the flip will close must be reachable now, or its closing proves nothing. assert.ok(await connects(JOINER, anchorOnMesh, HELD_REGISTRY), `the predecessor's published ${HELD_REGISTRY} is not reachable over the private network before the flip`); await pushAndSettle(CONTROL); const preview = await mesh(`converge ${CONTROL}`); assert.match(preview, new RegExp(`tcp/${SERVED}\\b[^\\n]*declared by hello-web`), `the preview does not name the service's port as declared:\n${preview}`); assert.match(preview, new RegExp(`tcp/${HELD_REGISTRY}\\b[^\\n]*published[^\\n]*WILL CLOSE`), `the preview does not name the published ${HELD_REGISTRY} as closing:\n${preview}`); assert.match(preview, /the flip takes:\n(\s{4}\S+\n?)+/, `the preview names no module the flip takes:\n${preview}`); assert.match(preview, new RegExp(`\\n\\s{4}${NETWORK_MODULE}\\b`), `the preview does not say the flip takes ${NETWORK_MODULE}:\n${preview}`); assert.match(preview, /Nothing has changed/, preview); assert.match(await must(CONTROL, `ufw status`), /Status: active/, `previewing changed the firewall`); return preview.trim(); }); await step("E2", ["E1"], async () => { const said: string[] = []; // The flip as the preview says to make it — whatever the preview binds `--yes` to. const preview = await mesh(`converge ${CONTROL}`); const flip = preview.match(new RegExp(`\`(converge ${CONTROL} --yes[^\`]*)\``))?.[1]; assert.ok(flip, `the preview does not say how to make the flip:\n${preview}`); said.push((await mesh(flip, 300_000)).trim().split("\n").slice(-3).join("\n")); await settled(); const table = await until(`the derived filter is loaded`, 300, async () => { const t = await on(CONTROL, `nft list table inet mesh`); return t.ok && /policy drop/.test(t.out) ? t.out : null; }, () => ""); const status = await until(`the found firewall is disabled`, 180, async () => { const s = (await on(CONTROL, `ufw status`)).out; return /Status: inactive/.test(s) ? s : null; }, () => ""); assert.ok((await on(CONTROL, `test -s /etc/ufw/user.rules && grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules`)).ok, `the found firewall's configuration is not on disk any more`); assert.match(await nodeShow(CONTROL), /mode\s+converged/, `the anchor is not reported converged`); said.push(` ufw ${status.trim()} — /etc/ufw/user.rules and after.rules still on disk`); await until(`the declared ${SERVED} answers over the private network`, 120, async () => (await connects(JOINER, anchorOnMesh, SERVED)) ? true : null, () => ""); assert.ok(!(await connects(JOINER, anchorOnMesh, HELD_REGISTRY)), `the undeclared ${HELD_REGISTRY} is still open`); assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once converged`); said.push(` ports ${SERVED} open over the private network; ${HELD_REGISTRY} closed; the store still closed from outside`); said.push(table.split("\n").slice(0, 30).join("\n")); return said.join("\n"); }); await step("E3", ["E2"], async () => { const said = (await mesh(`adopt ${CONTROL}`, 300_000)).trim(); await settled(); await until(`the found firewall is enabled again`, 180, async () => /Status: active/.test((await on(CONTROL, `ufw status`)).out) ? true : null, () => ""); await until(`the derived filter is gone`, 180, async () => !(await on(CONTROL, `nft list table inet mesh`)).ok ? true : null, () => ""); assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard is not restored`); assertOpenings(await until(`the openings are back`, 180, async () => { const o = await openings(); return o.length ? o : null; }, () => "")); assert.match(await nodeShow(CONTROL), /mode\s+adopted since/, `the anchor is not reported adopted`); assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once adopted again`); return `${said}\n ufw active, table inet mesh gone, the guard and the openings back`; }); // ---- what the mesh added, it takes back; what it found, it leaves --------------------------- await step("F1", ["E3"], async () => { const said: string[] = []; const before = await ufwAdded(); const operators = before.filter((r) => !marked(r) && new RegExp(`\\b${SERVED}\\b`).test(r)); assert.ok(operators.length > 0, `the operator's own rules for ${SERVED} are already gone:\n${before.join("\n")}`); await mesh(`unassign ${CONTROL} ${SERVICE}`); await pushAndSettle(CONTROL); let lastRules: string[] = before; const after = await until(`the mesh's own rules for ${SERVED} are gone`, 180, async () => { const rules = await ufwAdded(); lastRules = rules; return rules.some((r) => marked(r) && new RegExp(`opening-tcp-${SERVED}-`).test(r)) ? null : rules; }, () => lastRules.join("\n")); for (const rule of operators) { assert.ok(after.includes(rule), `the operator's own rule went with the mesh's opening: ${rule}\n${after.join("\n")}`); } said.push(` kept ${operators.length} rule(s) of the operator's, unmarked, after the module was unassigned`); said.push(...operators.map((r) => ` ${r}`)); return said.join("\n"); }); stateOutcome(); }, { timeout: 10_800_000 }); after(async () => { if (KEEP || WARM) { console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (${KEEP ? "MESH_LAB_KEEP" : "MESH_LAB_WARM"}).`); return; } if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 900_000 }); for (const [code, title] of Object.entries(TITLE)) { test(`${code} ${title}`, { skip, timeout: 60_000 }, () => { const s = steps.get(code); assert.ok(s?.ok, s ? `${s.why}` : `${code} never ran`); }); }