diff --git a/scenarios/adoption.yml b/scenarios/adoption.yml new file mode 100644 index 0000000..0625b40 --- /dev/null +++ b/scenarios/adoption.yml @@ -0,0 +1,56 @@ +# A MACHINE IN USE, ADOPTED — and then converged, and returned (novox/hq ADR 0100, ADR 0101). +# +# The mesh replaces a predecessor that is running on the same machines. The anchor here is +# prepared the way the predecessor leaves one: its own firewall (ufw) allowing a served port and +# denying the rest, a service container on that port under a name a catalogue module also uses, a +# file at a path that module declares, a stand-in for the predecessor's configuration sync that +# rewrites the file, and a container holding the registry's port. The bed then raises the mesh on +# it, adopted, and walks the migration the record decides. +# +# hosting (public) +# anchor 192.0.2.10 the machine in use: the predecessor, then the mesh adopted on it +# joiner 192.0.2.20 a fresh machine: the "second machine" that reaches the service and +# enrols through the found firewall; also where a converged genesis on a +# FRESH machine is asked (ADR 0101) +# outsider 192.0.2.30 never enrolled, never on the private network: the probe from outside, +# and the lab's forge — the bed serves the checkouts under test to the +# anchor's builder from here, so nothing on the workstation listens +# +# inbound: allow on every machine — the anchor's firewall is the predecessor's, installed by the +# bed; `inbound: deny` would load the lab's own table beside it and make that the thing under test. +scenario: adoption + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + # Sized like the one-node bed's anchor: genesis builds the control plane, the base and the + # catalogue's modules here, beside the predecessor's two containers. + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 12GiB + cpus: 6 + disk: 60GiB + + joiner: + at: { segment: hosting, address: [192.0.2.20] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + disk: 20GiB + + outsider: + at: { segment: hosting, address: [192.0.2.30] } + egress: true + inbound: allow + memory: 2GiB + cpus: 2 + disk: 15GiB + +place: + all: [host, runtime] diff --git a/test/integration/adoption.test.ts b/test/integration/adoption.test.ts new file mode 100644 index 0000000..ff5dbda --- /dev/null +++ b/test/integration/adoption.test.ts @@ -0,0 +1,851 @@ +/** + * A MACHINE IN USE IS ADOPTED BEFORE IT IS CONVERGED (novox/hq ADR 0100, and ADR 0101 on what + * "in use" ignores). + * + * The mesh replaces a predecessor running on the same machines. This bed prepares a machine the + * way the predecessor leaves one — the record's own words, "How it is checked": + * + * - its firewall (ufw) allowing a served port and denying the rest, incoming and routed, with the + * predecessor's published container ports filtered through it (the ufw-docker arrangement); + * - a service container, `hello-web`, listening on that port under a name the catalogue's + * `hello-web` module also uses, and a file at a path that module declares; + * - a stand-in for the predecessor's control that rewrites that file, stopped by the operator + * before adoption as the record prescribes, and started again later to play one forgotten; + * - a container holding the registry's port. + * + * Then it asks, in the record's order: a converged genesis refuses; an adopted one refuses the held + * registry port and comes up on another; nothing that serves changed; the store is unreachable + * from outside and reachable where it must be; the mesh works through the found firewall, across a + * reload and a reboot; a predecessor still writing is caught; assigning prepares and taking cuts + * over; converging previews, refuses while a found container is held, flips, and returns. + * + * **The module under migration is the catalogue's `hello-web` with its route requirement taken + * off**, read from the catalogue (never a copy): the route needs a proxy module and a public name, + * neither of which is what adoption is about. Its names — the container, the file, the port — are + * the catalogue's, which is the point: they are what the predecessor also uses. + * + * **The forge is in the lab.** Genesis builds the control plane and the catalogue from a + * repository and a commit; this bed serves the checkouts it was pointed at (their HEADs) from the + * `outsider` machine, so the run builds exactly the code under test and nothing on the workstation + * listens for the lab. + * + * Each step is recorded rather than allowed to throw; a step whose dependency failed is not + * attempted, and the report says which. + * + * MESH_LAB_INCUS='sudo -n incus' + * MESH_LAB_HOST_BINARY=/mesh-host MESH_LAB_BOOTSTRAP_BINARY=/mesh-bootstrap + * MESH_LAB_BUNDLE=/examples/foundation-first-node.lock + * MESH_LAB_CATALOG=/modules MESH_LAB_MODULES=/examples/modules + * MESH_TOOLS= MESH_SDK= (default: the checkouts beside this one) + * MESH_LAB_KEEP=1 leave it standing MESH_LAB_WARM=1 iterate from the adopted foundation + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec, push, instanceNameOf } from "../../src/lifecycle/operate.ts"; +import { bootstrapBinaryPath, hostBinaryPath, placeBootstrap, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { waitUntilAllUsable } from "../../src/lifecycle/ready.ts"; +import { incus } from "../../src/incus/client.ts"; +import { catalogueRoot } from "../../src/repos.ts"; +import { ready, returnTo, keep } from "../../src/warm.ts"; +import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueManifest } from "./harness.ts"; +import { genesis, type GenesisOptions } from "./genesis.ts"; + +const SCENARIO = "adoption"; +const CONTROL = "anchor"; +const JOINER = "joiner"; +const OUTSIDER = "outsider"; +const ANCHOR = "192.0.2.10"; +const JOINER_ADDRESS = "192.0.2.20"; +const FORGE = "192.0.2.30"; + +/** The port the predecessor serves, and the module that also names its container and file. */ +const SERVED = 8080; +const SERVICE = "hello-web"; +const SERVICE_FILE = "/var/lib/hello-web/index.html"; +const PREDECESSOR_PAGE = "hello from the predecessor\n"; +/** The registry's port, which the predecessor holds — and the one the mesh is given instead. */ +const HELD_REGISTRY = 5000; +const REGISTRY_PORT = 5100; +const STORE_PORT = 5432; +const BUS_PORT = 5671; +const HUB_PORT = 51820; +const NETWORK_MODULE = "networking"; +const FILTER_MODULE = "nftables"; + +/** Upstream images, pinned as the catalogue pins them (the harness's table). */ +const ALPINE = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"; +const REGISTRY_IMAGE = "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; +const modulesDir = process.env["MESH_LAB_MODULES"] ?? ""; +const KEEP = !!process.env["MESH_LAB_KEEP"]; +const WARM = !!process.env["MESH_LAB_WARM"]; +const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "adoption-live" : undefined); + +/** The checkouts this run builds from, served by the lab's forge. */ +const REPOS: Record = { + "mesh-controller": modulesDir ? dirname(dirname(modulesDir)) : "", + "mesh-catalog": catalogDir ? catalogueRoot(catalogDir) : "", + "mesh-tools": process.env["MESH_TOOLS"] ?? resolve(process.cwd(), "..", "mesh-tools"), + "mesh-sdk": process.env["MESH_SDK"] ?? resolve(process.cwd(), "..", "mesh-sdk"), +}; + +const skip = + !capability.usable ? capability.why : + !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : + !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" : + !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" : + !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" : + !modulesDir ? "MESH_LAB_MODULES is not set, so there is no control-plane checkout to build from" : + Object.entries(REPOS).find(([, p]) => !p || !existsSync(resolve(p, ".git"))) + ?.map((x) => `no checkout of ${x[0]} at ${x[1]}`)[0] ?? false; + +let instanceId = ""; + +// ---- talking to the machines ------------------------------------------------------------------ + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} +/** The control plane, retried across the brief windows in which the mesh recreates it. */ +async function meshSays(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const deadline = Date.now() + (timeoutMs ?? 120_000); + for (;;) { + const r = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); + if (r.ok) return r; + if (/is not running|No such container|No such exec instance|Cannot connect to the Docker daemon|is restarting/i.test(r.out) && + Date.now() < deadline) { + await sleep(2_000); + continue; + } + return r; + } +} +async function mesh(command: string, timeoutMs?: number): Promise { + const r = await meshSays(command, timeoutMs); + if (!r.ok) throw new Error(`${CONTROL}: mesh-controller ${command}\n${r.out}`); + return r.out; +} +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} +/** Poll until `probe` returns a value, or fail naming the last thing it saw. */ +async function until(what: string, seconds: number, probe: () => Promise, last: () => string): Promise { + const deadline = Date.now() + seconds * 1000; + for (;;) { + const got = await probe(); + if (got !== null) return got; + if (Date.now() > deadline) throw new Error(`${what} — not within ${seconds}s. Last:\n${last()}`); + await sleep(5_000); + } +} +/** Whether a TCP connection from `machine` to address:port opens within three seconds. */ +async function connects(machine: string, address: string, port: number): Promise { + return (await on(machine, `timeout 4 bash -c ${quote(` { + const local = join(tmpdir(), `mesh-lab-adoption-${process.pid}-${module}.json`); + writeFileSync(local, manifest); + await push(instanceId, CONTROL, local, `/tmp/${module}.json`); + await must(CONTROL, `docker cp /tmp/${module}.json mesh-controller:/${module}.json`); + return mesh(`module add /${module}.json`); +} +async function nodeShow(node: string): Promise { + return mesh(`node show ${node}`); +} +/** The anchor's address on the private network. */ +async function meshAddressOf(machine: string): Promise { + const out = await must(machine, `ip -4 -o addr show dev mesh0`); + const found = out.match(/inet (\d+\.\d+\.\d+\.\d+)\//)?.[1]; + assert.ok(found, `${machine} has no address on mesh0:\n${out}`); + return found; +} +/** The rules ufw was given, one per line, as `ufw show added` prints them. */ +async function ufwAdded(): Promise { + const out = await must(CONTROL, `ufw show added`); + return out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw ")); +} +function marked(rule: string): boolean { + return /comment 'mesh-host /.test(rule); +} +async function restartMachine(machine: string): Promise { + const name = await instanceNameOf(instanceId, machine); + await incus(["restart", name], 180_000); + await waitUntilAllUsable([name], 300, (m) => console.log(` restart: ${m}`)); +} +/** Until the anchor reports what it was last sent as applied and current. */ +async function settled(node = CONTROL, withinMs = 300_000): Promise { + let last = ""; + await until(`${node} reports the declaration it was sent as applied and current`, withinMs / 1000, async () => { + const asked = await meshSays(`status --json`); + last = asked.out; + if (!asked.ok) return null; + try { + const state = JSON.parse(asked.out) as { + wrong: { node: string; outcome: string }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + const bad = state.wrong.find((w) => w.node === node); + if (bad) throw new Error(`${node} did not apply what it was sent: ${bad.outcome}\n${asked.out}`); + const word = state.reported.find((r) => r.node === node); + return !state.waiting.some((w) => w.node === node) && word?.outcome === "applied" && word.current ? true : null; + } catch (err) { + if (err instanceof Error && err.message.includes("did not apply")) throw err; + return null; + } + }, () => last); +} +/** Send a node what it should be, and wait until it says it applied it. */ +async function pushAndSettle(node: string): Promise { + const said = await mesh(`push ${node}`, 600_000); + await settled(node); + return said; +} +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +// ---- the lab's forge --------------------------------------------------------------------------- + +/** + * Serve the checkouts under test from the outsider machine, over git's own protocol. + * + * Each checkout's HEAD is pushed into a bare repository as `main` and `lab`, the lot is carried in, + * and a git daemon answers on the outsider's scenario address — which the anchor's builder reaches + * over the hosting segment. Returns the commit each repository is served at. + */ +async function raiseForge(): Promise> { + const dir = mkdtempSync(join(tmpdir(), "mesh-lab-forge-")); + const heads: Record = {}; + try { + for (const [name, checkout] of Object.entries(REPOS)) { + const bare = join(dir, `${name}.git`); + execFileSync("git", ["init", "-q", "--bare", bare]); + execFileSync("git", ["-C", checkout, "push", "-q", "--force", bare, + "HEAD:refs/heads/main", "HEAD:refs/heads/lab"], { stdio: "pipe" }); + heads[name] = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + } + const tar = join(tmpdir(), `mesh-lab-forge-${process.pid}.tar`); + execFileSync("tar", ["-cf", tar, "-C", dir, "."]); + await push(instanceId, OUTSIDER, tar, "/tmp/forge.tar"); + rmSync(tar, { force: true }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + await must(OUTSIDER, `command -v git >/dev/null || pacman -S --noconfirm --needed git`, 600_000); + // Owned by the daemon's user: extracted as the workstation's uid, git refuses to serve a + // repository someone else owns ("dubious ownership"), and the clone fails with no reason given. + await must(OUTSIDER, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` + + `git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE} --pid-file=/run/git-daemon.pid`); + await must(OUTSIDER, `git ls-remote git://${FORGE}/mesh-controller.git lab`); + await until("the lab's forge answers the anchor", 60, async () => + (await connects(CONTROL, FORGE, 9418)) ? true : null, () => "no connection to 9418"); + return heads; +} +const forgeUrl = (repo: string) => `git://${FORGE}/${repo}.git`; + +// ---- the predecessor --------------------------------------------------------------------------- + +/** + * The ufw-docker arrangement: published container ports pass through ufw's route rules, and + * traffic from private ranges is let through. It is how a ufw machine filters what docker publishes + * at all — without it docker's own rules bypass ufw entirely (novox/hq research 012 measured 52 + * forwarding rules on the control-node, one per served port). + */ +const UFW_DOCKER = ` +# BEGIN UFW AND DOCKER +*filter +:ufw-user-forward - [0:0] +:ufw-docker-logging-deny - [0:0] +:DOCKER-USER - [0:0] +-A DOCKER-USER -j ufw-user-forward +-A DOCKER-USER -j RETURN -s 10.0.0.0/8 +-A DOCKER-USER -j RETURN -s 172.16.0.0/12 +-A DOCKER-USER -j RETURN -s 192.168.0.0/16 +-A DOCKER-USER -p udp -m udp --sport 53 --dport 1024:65535 -j RETURN +-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 192.168.0.0/16 +-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 10.0.0.0/8 +-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 172.16.0.0/12 +-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 192.168.0.0/16 +-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 10.0.0.0/8 +-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 172.16.0.0/12 +-A DOCKER-USER -j RETURN +-A ufw-docker-logging-deny -j DROP +COMMIT +# END UFW AND DOCKER +`; + +/** The stand-in for the predecessor's configuration sync: it rewrites the file every ten seconds. */ +const STAND_IN = `[Unit] +Description=Stand-in for the predecessor's configuration sync: rewrites a file a catalogue module declares + +[Service] +ExecStart=/bin/sh -c 'while true; do printf "hello from the predecessor, synced %%s\\\\n" "$(date +%%s)" > ${SERVICE_FILE}; sleep 10; done' +`; + +/** The page the predecessor's service loop serves — the catalogue module's own loop, verbatim. */ +const SERVE_LOOP = + "while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done"; + +/** Where the bed keeps what the machine looked like before the mesh arrived — on the machine, so a warm restore keeps it. */ +const BEFORE = "/root/predecessor"; + +async function preparePredecessor(): Promise { + const said: string[] = []; + await must(CONTROL, `pacman -S --noconfirm --needed ufw`, 600_000); + const rules = join(tmpdir(), `mesh-lab-ufw-docker-${process.pid}`); + writeFileSync(rules, UFW_DOCKER); + await push(instanceId, CONTROL, rules, "/tmp/ufw-docker.rules"); + await must(CONTROL, [ + `grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules || cat /tmp/ufw-docker.rules >> /etc/ufw/after.rules`, + `ufw default deny incoming`, + `ufw default allow outgoing`, + `ufw default deny routed`, + `ufw allow 22/tcp`, + `ufw allow ${SERVED}/tcp`, + `ufw route allow proto tcp from any to any port ${SERVED}`, + `ufw --force enable`, + `systemctl enable ufw`, + ].join(" && ")); + said.push(` firewall ufw: deny incoming and routed; allow 22 and ${SERVED}; ufw-docker after.rules`); + + await must(CONTROL, `mkdir -p /var/lib/hello-web && printf %s ${quote(PREDECESSOR_PAGE)} > ${SERVICE_FILE}`); + await must(CONTROL, + `docker run -d --name ${SERVICE} --restart unless-stopped -p ${SERVED}:${SERVED} ` + + `-v ${SERVICE_FILE}:/www/index.html:ro ${ALPINE} sh -c ${quote(SERVE_LOOP)}`, 600_000); + await must(CONTROL, + `docker run -d --name predecessor-registry --restart unless-stopped -p ${HELD_REGISTRY}:5000 ${REGISTRY_IMAGE}`, 600_000); + said.push(` containers ${SERVICE} on ${SERVED}, predecessor-registry on ${HELD_REGISTRY}`); + + // The predecessor's control, which the operator stops before adopting (the record's words). + const unit = join(tmpdir(), `mesh-lab-stand-in-${process.pid}`); + writeFileSync(unit, STAND_IN); + await push(instanceId, CONTROL, unit, "/etc/systemd/system/predecessor-sync.service"); + await must(CONTROL, `systemctl daemon-reload && systemctl start predecessor-sync && sleep 12 && systemctl stop predecessor-sync`); + said.push(` stand-in predecessor-sync rewrote ${SERVICE_FILE}, then the operator stopped it`); + + // What the machine was, recorded ON the machine so a restored warm instance still has it. + await must(CONTROL, [ + `mkdir -p ${BEFORE}`, + `sha256sum ${SERVICE_FILE} | cut -d' ' -f1 > ${BEFORE}/file.sha256`, + `cp ${SERVICE_FILE} ${BEFORE}/file`, + `docker inspect -f '{{.Id}}' ${SERVICE} > ${BEFORE}/container.id`, + `ufw show added > ${BEFORE}/ufw-added.txt`, + ].join(" && ")); + await until(`the predecessor's ${SERVICE} answers the joiner`, 60, async () => + (await on(JOINER, `curl -s --max-time 3 http://${ANCHOR}:${SERVED}/`)).out.includes("hello from the predecessor") ? true : null, + () => "no answer"); + said.push((await must(CONTROL, `ufw status verbose`)).trim()); + said.push((await must(CONTROL, `docker ps --format '{{.Names}}\t{{.Ports}}'`)).trim()); + return said.join("\n"); +} + +// ---- steps, recorded rather than thrown -------------------------------------------------------- + +interface Step { code: string; title: string; ok: boolean; why: string; said: string; seconds: number; warm?: boolean } +const steps = new Map(); + +async function step(code: string, needs: string[], fn: () => Promise): Promise { + const title = TITLE[code]!; + const missing = needs.filter((n) => !steps.get(n)?.ok); + if (missing.length) { + steps.set(code, { code, title, ok: false, seconds: 0, said: "", + why: `not attempted — ${missing.join(", ")} did not succeed` }); + console.log(`[${code}] SKIP ${title}`); + return; + } + console.log(`\n[${code}] ---- ${title} ----`); + const began = Date.now(); + const took = () => Math.round((Date.now() - began) / 1000); + try { + const said = await fn(); + steps.set(code, { code, title, ok: true, why: "", said, seconds: took() }); + console.log(`${said}\n[${code}] PASS ${title} (${took()}s)`); + } catch (err) { + const why = (err as Error).message; + steps.set(code, { code, title, ok: false, why, said: "", seconds: took() }); + console.log(`[${code}] FAIL ${title} (${took()}s)\n${why.split("\n").slice(0, 40).join("\n")}`); + } +} + +/** The plan, in the record's order. Codes are stable; titles may be reworded. */ +const TITLE: Record = { + P0: "the machine is prepared the way the predecessor leaves one", + F0: "a converged genesis on a FRESH machine is not refused — its own resolver does not make it in use (ADR 0101)", + A1: "a converged genesis refuses the machine in use, naming every container and listener it counted", + A2: "an adopted genesis refuses the registry's held port, naming what holds it", + A3: "given another registry port, the adopted foundation comes up — and stays on that port as modules", + B1: "nothing that serves changed: the service answers, its file and container are untouched, the firewall gained only the mesh's marked rules", + B2: "the store is unreachable from outside, before and after the found firewall reloads; the bus answers a machine not yet enrolled", + B4: "the store is reachable from a container on the node itself", + C1: "a second machine enrols through the found firewall and joins the private network", + B3: "the store is reachable over the private network", + C2: "after the found firewall reloads, the openings are there and the mesh still works", + C3: "after the machine reboots, the openings are there and the mesh still works", + D1: "assigning prepares: the module holds the found container and file, and neither changes", + D2: "a predecessor still writing is caught: the held file's change is reported, and not reverted", + D3: "converging refuses while the service's module holds its found container", + D4: "taking cuts over: the found container and file are replaced, the original kept, the port opened", + E1: "converging previews: the service's port, a published port no rule mentions, and the modules it takes", + E2: "the flip: the derived filter loaded, the found firewall disabled with its configuration on disk, the declared port open and the undeclared closed", + E3: "returned to adopted: the found firewall enabled again, the derived filter gone, the openings back", +}; + +function stateOutcome(): void { + console.log(`\n================ ADOPTION: WHAT WAS ESTABLISHED ================`); + let established = 0; + for (const code of Object.keys(TITLE)) { + const s = steps.get(code); + const mark = !s ? "NEVER" : s.warm ? "WARM" : s.ok ? "PASS" : s.why.startsWith("not attempted") ? "SKIP" : "FAIL"; + if (s?.ok) established++; + console.log(` ${code.padEnd(3)} ${mark.padEnd(5)} ${TITLE[code]}${s?.seconds ? ` (${s.seconds}s)` : ""}`); + } + console.log(` ${established}/${Object.keys(TITLE).length} established.`); +} + +// ---- the run ----------------------------------------------------------------------------------- + +before(async () => { + if (skip) return; + console.log(`\n================ THE PLAN ================`); + for (const [code, title] of Object.entries(TITLE)) console.log(` ${code.padEnd(3)} ${title}`); + + const verdict = WARM ? await ready(SCENARIO) : { use: "raise" as const, why: "not asked to be warm" }; + let restored = false; + if (verdict.use === "restore") { + instanceId = verdict.instanceId; + const seconds = await returnTo(instanceId, (m) => console.log(`warm: ${m}`)); + console.log(`WARM: restored ${instanceId} to the adopted foundation in ${seconds}s`); + restored = true; + for (const code of ["P0", "F0", "A1", "A2", "A3"]) { + steps.set(code, { code, title: TITLE[code]!, ok: true, warm: true, seconds: 0, why: "", + said: "restored from the warm snapshot — not re-run; only a fresh run proves it" }); + } + } else { + if (WARM) console.log(`WARM: raising — ${verdict.why}`); + const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), + }); + instanceId = bed.instanceId; + } + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); + + let heads: Record = {}; + const genesisOn = (node: string, o: Partial): Promise extends Promise ? R : never> => + genesis({ + instanceId, node, installer: installer as string, catalogDir, + bundleTemplate: foundationBundle(bundle, []), + registry: `${ANCHOR}:${HELD_REGISTRY}`, + source: forgeUrl("mesh-controller"), sourceRef: heads["mesh-controller"] ?? "", + toolsSource: forgeUrl("mesh-tools"), toolsRef: "lab", + catalogSource: forgeUrl("mesh-catalog"), catalogRef: "lab", + sdkSource: forgeUrl("mesh-sdk"), sdkRef: "lab", + site: "hosting", + hostService: true, + ...(binary ? { hostBinary: binary } : {}), + log: (m) => console.log(m), + ...o, + }); + + if (!restored) { + await step("P0", [], async () => { + heads = await raiseForge(); + const said = [` forge git://${FORGE}/ serving ${Object.entries(heads).map(([n, h]) => `${n}@${h.slice(0, 8)}`).join(", ")}`]; + said.push(await preparePredecessor()); + return said.join("\n"); + }); + + // ADR 0101: the joiner is a freshly installed machine — nothing but its operating system, a + // container runtime and the host binary. A converged genesis there must not be refused. Asked + // as a dry run: the question is the preflight's, and a real raise would make it a second mesh. + await step("F0", ["P0"], async () => { + const ran = await genesisOn(JOINER, { flags: ["--dry-run"], attempts: 1, verify: false }); + assert.doesNotMatch(ran.said, /this machine is in use/, + `a converged genesis refused a fresh machine as in use:\n${ran.said}`); + assert.match(ran.said, /in use\s+no: no container runs and nothing listens beyond ssh/, + `the installer never said the fresh machine is not in use:\n${ran.said}`); + const listening = (await must(JOINER, `ss -Hltunp`)).trim(); + return ` in use no — the installer went on (${ran.ok ? "dry run finished" : `dry run stopped later, at ${ran.step}`})\n` + + ` what listens on the fresh machine:\n${listening.split("\n").map((l) => ` ${l}`).join("\n")}`; + }); + + await step("A1", ["P0"], async () => { + const ran = await genesisOn(CONTROL, { attempts: 1, verify: false }); + assert.ok(!ran.ok, `a converged genesis went ahead on a machine in use:\n${ran.said}`); + assert.match(ran.step, /preflight/, `it was refused, but not before changing anything (at ${ran.step}):\n${ran.said}`); + for (const want of [/container hello-web/, /container predecessor-registry/, + new RegExp(`tcp \\S+:${SERVED} by`), new RegExp(`tcp \\S+:${HELD_REGISTRY} by`)]) { + assert.match(ran.said, want, `the refusal does not name ${want}:\n${ran.said}`); + } + assert.match(ran.said, /--adopted/, `the refusal does not say how to raise it adopted`); + // And nothing was changed: the predecessor as it was, no table of the mesh's. + const ps = await must(CONTROL, `docker ps --format '{{.Names}}'`); + assert.deepEqual(ps.trim().split("\n").sort(), ["hello-web", "predecessor-registry"], `containers changed:\n${ps}`); + assert.match(await must(CONTROL, `ufw status`), /Status: active/); + assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`); + return ran.said.split("\n").filter((l) => /in use|^\s+- /.test(l)).join("\n"); + }); + + await step("A2", ["A1"], async () => { + const ran = await genesisOn(CONTROL, { adopted: true, attempts: 1, verify: false }); + assert.ok(!ran.ok, `an adopted genesis went ahead with the registry's port held:\n${ran.said}`); + assert.match(ran.said, new RegExp(`registry's port tcp/${HELD_REGISTRY} is held by [^\\n]*predecessor-registry`), + `the refusal does not name what holds the registry's port:\n${ran.said.split("\n").slice(-15).join("\n")}`); + assert.match(ran.said, /--registry-port/, `the refusal does not say which flag gives another port`); + const id = (await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(); + assert.equal(id, (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the predecessor's container was replaced`); + assert.match(await must(CONTROL, `ufw status`), /Status: active/); + assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`); + return ` refused at ${ran.step}\n` + ran.said.split("\n").filter((l) => /held by|port|adopted/.test(l)).slice(-8).join("\n"); + }); + + await step("A3", ["A2"], async () => { + const ran = await genesisOn(CONTROL, { adopted: true, registry: `${ANCHOR}:${REGISTRY_PORT}` }); + if (!ran.ok) throw new Error(`${ran.step}: ${ran.why}\n\n${ran.report.join("\n")}`); + await settled(); + const said = [ran.report.join("\n")]; + const bindings = await must(CONTROL, `docker inspect -f '{{json .HostConfig.PortBindings}}' mesh-registry`); + assert.match(bindings, new RegExp(`"HostPort":"${REGISTRY_PORT}"`), `the registry is not on ${REGISTRY_PORT}: ${bindings}`); + assert.match(await must(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' mesh-registry`), /\S/, + `mesh-registry is not the host's: the foundation was not adopted as a module`); + assert.match(await mesh(`module list`), /^distribution\b/m, `the registry is not a module the mesh holds`); + // The node's own port, in what the mesh would send it — not the catalogue's default. + const plan = await mesh(`plan ${CONTROL} --json`); + assert.match(plan, new RegExp(`"${REGISTRY_PORT}:5000"`), `the registry's module does not publish ${REGISTRY_PORT}`); + assert.doesNotMatch(plan, /"5000:5000"/, `the plan still publishes the catalogue's 5000`); + said.push(` registry on ${REGISTRY_PORT}, as the module the mesh holds: ${bindings.trim()}`); + const show = await nodeShow(CONTROL); + assert.match(show, /mode\s+adopted since/, `the anchor is not reported adopted:\n${show}`); + assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `the foundation's dropping table was loaded on an adopted node`); + const guard = await must(CONTROL, `nft list table inet mesh_guard`); + // The guard's port set is the controller's to derive; what the record fixes is that it refuses + // the store's port from outside and holds nothing but refusals. + assert.match(guard, new RegExp(`dport (\\{[^}]*\\b${STORE_PORT}\\b[^}]*\\}|${STORE_PORT}) drop`), `the guard does not refuse the store's port:\n${guard}`); + assert.doesNotMatch(guard, /accept\s*$/m, `the guard holds an accept:\n${guard}`); + assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force`); + assert.match(await must(CONTROL, `curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${HELD_REGISTRY}/v2/`), /200/, + `the predecessor's registry stopped answering`); + said.push(show.trim(), guard.trim()); + return said.join("\n"); + }); + + if (WARM && steps.get("A3")?.ok) { + await keep(SCENARIO, instanceId); + console.log(`WARM: kept ${instanceId} at the adopted foundation`); + } + } + + // ---- nothing that serves changed ------------------------------------------------------------- + await step("B1", ["A3"], async () => { + const said: string[] = []; + const want = await must(CONTROL, `cat ${BEFORE}/file`); + let page = ""; + await until(`the service answers the joiner as it did`, 120, async () => { + page = (await on(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`)).out; + return page === want ? true : null; + }, () => page); + said.push(` ${SERVICE} answers the joiner on ${SERVED} with the predecessor's page`); + assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(), + (await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `${SERVICE_FILE} changed`); + assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(), + (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the ${SERVICE} container was replaced`); + said.push(` file, container byte for byte / the same id as before the mesh`); + const was = (await must(CONTROL, `cat ${BEFORE}/ufw-added.txt`)).split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw ")); + const now = await ufwAdded(); + const lost = was.filter((r) => !now.includes(r)); + const added = now.filter((r) => !was.includes(r)); + assert.deepEqual(lost, [], `the found firewall lost rules:\n${lost.join("\n")}`); + const unmarked = added.filter((r) => !marked(r)); + assert.deepEqual(unmarked, [], `the found firewall gained rules not marked as the mesh's:\n${unmarked.join("\n")}`); + assert.ok(added.length > 0, `the mesh opened nothing through the found firewall`); + said.push(` firewall ${was.length} rule(s) kept, ${added.length} added, every one marked:`, ...added.map((r) => ` ${r}`)); + return said.join("\n"); + }); + + await step("B2", ["A3"], async () => { + const said: string[] = []; + assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers a machine off the private network`); + await must(CONTROL, `ufw reload`); + await sleep(3_000); + assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the found firewall reloaded`); + said.push(` outsider -> ${STORE_PORT} refused, before and after \`ufw reload\``); + assert.ok(await connects(OUTSIDER, ANCHOR, BUS_PORT), `the bus does not answer a machine that has not enrolled`); + said.push(` outsider -> ${BUS_PORT} the bus answers`); + return said.join("\n"); + }); + + // Its own step: it asks something different of the found firewall — a container on the machine + // reaches a published port through the runtime's proxy, on the incoming path, not the forwarded. + await step("B4", ["A3"], async () => { + const said: string[] = []; + const fromContainer = await on(CONTROL, `docker run --rm ${ALPINE} nc -z -w 3 ${ANCHOR} ${STORE_PORT}`, 180_000); + assert.ok(fromContainer.ok, `a container on the node cannot reach the store:\n${fromContainer.out}`); + said.push(` container -> ${STORE_PORT} reachable from a container on the node itself`); + return said.join("\n"); + }); + + // ---- the mesh works through the found firewall ----------------------------------------------- + let anchorOnMesh = ""; + await step("C1", ["B2"], async () => { + const said: string[] = []; + await mesh(`node add ${JOINER}`); + const token = tokenFrom(await mesh(`token issue --node ${JOINER}`)); + const out = await must(JOINER, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); + assert.match(out, new RegExp(`enrolled as ${JOINER}`), out); + await must(JOINER, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + said.push(` ${JOINER} enrolled over the bus, through the anchor's ufw`); + await mesh(`overlay place ${JOINER} --site hosting`); + await mesh(`assign ${JOINER} ${NETWORK_MODULE}`); + await pushAndSettle(JOINER); + // The hub's peer list changed: the anchor has to be sent it too. + await pushAndSettle(CONTROL); + anchorOnMesh = await meshAddressOf(CONTROL); + await until(`the joiner reaches the anchor over mesh0`, 180, async () => + (await on(JOINER, `ping -c1 -W2 ${anchorOnMesh}`)).ok ? true : null, + () => "no ping reply"); + said.push(` private network the joiner reaches the anchor at ${anchorOnMesh}`); + said.push((await must(CONTROL, `wg show mesh0 latest-handshakes`)).trim()); + return said.join("\n"); + }); + + await step("B3", ["C1"], async () => { + assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store does not answer over the private network`); + return ` joiner -> ${anchorOnMesh}:${STORE_PORT} reachable over mesh0`; + }); + + /** The mesh's own rules in the found firewall. */ + const openings = async (): Promise => (await ufwAdded()).filter(marked); + const assertOpenings = (rules: string[]) => { + const text = rules.join("\n"); + // By the opening's id, which names the machine's port: a forwarded rule names the CONTAINER's + // port (ufw's route rules match after the runtime's translation), so the text may say another. + for (const port of [BUS_PORT, REGISTRY_PORT, HUB_PORT, STORE_PORT]) { + assert.match(text, new RegExp(`opening-(tcp|udp)-${port}-`), `no opening for ${port} among the mesh's rules:\n${text}`); + } + }; + await step("C2", ["B3"], async () => { + const before = await openings(); + assertOpenings(before); + await must(CONTROL, `ufw reload`); + await sleep(3_000); + const after = await openings(); + assert.deepEqual(after.sort(), before.sort(), `the openings changed across a reload`); + assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store stopped answering over mesh0 after the reload`); + assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reload`); + await pushAndSettle(JOINER); + return ` after ufw reload ${after.length} opening(s) in place; the joiner reaches the store over mesh0 and takes a push\n` + + after.map((r) => ` ${r}`).join("\n"); + }); + + await step("C3", ["C2"], async () => { + const before = await openings(); + await restartMachine(CONTROL); + await until(`the control plane answers after the reboot`, 300, async () => + (await meshSays(`status`)).ok ? true : null, () => "no answer"); + assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force after the reboot`); + assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard did not come back`); + const after = await until(`the openings are there again`, 420, async () => { + const now = await openings(); + return before.every((r) => now.includes(r)) ? now : null; + }, () => "not all openings"); + assertOpenings(after); + assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reboot`); + await until(`the joiner reaches the store over mesh0 again`, 300, async () => + (await connects(JOINER, anchorOnMesh, STORE_PORT)) ? true : null, () => "no connection"); + await pushAndSettle(JOINER); + const page = await must(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`); + assert.match(page, /hello from the predecessor/, `the predecessor's service did not come back: ${page}`); + return ` after a reboot ufw active, the guard loaded, ${after.length} opening(s); the joiner reaches the store and takes a push`; + }); + + // ---- assigning prepares, taking cuts over ---------------------------------------------------- + let kept = ""; + await step("D1", ["B1"], async () => { + const said: string[] = []; + // The catalogue's module, read from the catalogue, with the route requirement taken off: the + // route needs a proxy module and a public name, and neither is what adoption is about. + const manifest = JSON.parse(catalogueModule(SERVICE, [])) as Record; + delete manifest["requires"]; delete manifest["contributes"]; delete manifest["binds"]; + await registerModule(SERVICE, JSON.stringify(manifest)); + await mesh(`assign ${CONTROL} ${SERVICE}`); + await pushAndSettle(CONTROL); + const show = await until(`the anchor reports holding ${SERVICE}'s container and file`, 120, async () => { + const s = await nodeShow(CONTROL); + return /holds container\s+hello-web\b/.test(s) && /holds file\s+\/var\/lib\/hello-web\/index\.html/.test(s) ? s : null; + }, () => ""); + kept = show.match(/holds file\s+\/var\/lib\/hello-web\/index\.html[^\n]*\n\s*original kept at (\S+)/)?.[1] ?? ""; + assert.ok(kept, `the held file's original is not reported kept:\n${show}`); + assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(), + (await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `assigning changed ${SERVICE_FILE}`); + assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(), + (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `assigning replaced the ${SERVICE} container`); + assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the kept original is not the file as found`); + said.push(show.trim()); + return said.join("\n"); + }); + + await step("D2", ["D1"], async () => { + await must(CONTROL, `systemctl start predecessor-sync`); + try { + await sleep(15_000); + await pushAndSettle(CONTROL); + const show = await until(`the anchor reports the held file changed`, 120, async () => { + const s = await nodeShow(CONTROL); + return /hello-web\/index\.html[^\n]*REWRITTEN/i.test(s) ? s : null; + }, () => ""); + const now = await must(CONTROL, `cat ${SERVICE_FILE}`); + assert.match(now, /synced \d+/, `the host reverted what the predecessor wrote:\n${now}`); + return show.trim(); + } finally { + await on(CONTROL, `systemctl stop predecessor-sync`); + } + }); + + await step("D3", ["D1"], async () => { + await pushAndSettle(CONTROL); + const r = await meshSays(`converge ${CONTROL}`); + assert.ok(!r.ok, `converge went ahead while ${SERVICE} holds its found container:\n${r.out}`); + assert.match(r.out, new RegExp(`hello-web holds the found container hello-web`), `the refusal does not name the held container:\n${r.out}`); + assert.match(r.out, new RegExp(`take ${CONTROL} hello-web`), `the refusal does not say what to do:\n${r.out}`); + return r.out.trim(); + }); + + await step("D4", ["D1"], async () => { + const said: string[] = []; + said.push((await mesh(`take ${CONTROL} ${SERVICE}`)).trim()); + await pushAndSettle(CONTROL); + const label = await until(`the ${SERVICE} container is the mesh's`, 180, async () => { + const l = (await on(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' ${SERVICE}`)).out.trim(); + return l && l !== "" ? l : null; + }, () => ""); + assert.notEqual((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(), + (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the found container was not replaced`); + const catalogue = (JSON.parse(catalogueModule(SERVICE, [])) as { resources: { id: string; content?: string }[] }) + .resources.find((r) => r.id === "page")?.content ?? ""; + assert.equal(await must(CONTROL, `cat ${SERVICE_FILE}`), catalogue, `the found file was not replaced with the module's`); + assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the original was not kept`); + said.push(` replaced container (spec ${label.slice(0, 12)}…) and ${SERVICE_FILE}; original still at ${kept}`); + const opened = (await openings()).filter((r) => new RegExp(`opening-tcp-${SERVED}-`).test(r)); + assert.ok(opened.length > 0, `no opening for ${SERVED} once ${SERVICE} was taken:\n${(await openings()).join("\n")}`); + said.push(...opened.map((r) => ` opened ${r}`)); + const page = await until(`the taken ${SERVICE} answers over the private network`, 120, async () => { + const p = await on(JOINER, `curl -s --max-time 3 http://${anchorOnMesh}:${SERVED}/`); + return p.ok && p.out === catalogue ? p.out : null; + }, () => ""); + said.push(` joiner -> ${SERVED} ${page.trim()}`); + const show = await nodeShow(CONTROL); + assert.doesNotMatch(show, /holds (container|file)\s+\S*hello-web/, `the anchor still holds what was taken:\n${show}`); + return said.join("\n"); + }); + + // ---- converging previews, then changes ------------------------------------------------------- + await step("E1", ["D4"], async () => { + if (!/^nftables\b/m.test(await mesh(`module list`))) { + await registerModule(FILTER_MODULE, JSON.stringify(JSON.parse(catalogueModule(FILTER_MODULE, [])))); + } + // What the flip will close must be reachable now, or its closing proves nothing. + assert.ok(await connects(JOINER, anchorOnMesh, HELD_REGISTRY), + `the predecessor's published ${HELD_REGISTRY} is not reachable over the private network before the flip`); + await pushAndSettle(CONTROL); + const preview = await mesh(`converge ${CONTROL}`); + assert.match(preview, new RegExp(`tcp/${SERVED}\\b[^\\n]*declared by hello-web`), `the preview does not name the service's port as declared:\n${preview}`); + assert.match(preview, new RegExp(`tcp/${HELD_REGISTRY}\\b[^\\n]*published[^\\n]*WILL CLOSE`), `the preview does not name the published ${HELD_REGISTRY} as closing:\n${preview}`); + assert.match(preview, /the flip takes:\n(\s{4}\S+\n?)+/, `the preview names no module the flip takes:\n${preview}`); + assert.match(preview, new RegExp(`\\n\\s{4}${NETWORK_MODULE}\\b`), `the preview does not say the flip takes ${NETWORK_MODULE}:\n${preview}`); + assert.match(preview, /Nothing has changed/, preview); + assert.match(await must(CONTROL, `ufw status`), /Status: active/, `previewing changed the firewall`); + return preview.trim(); + }); + + await step("E2", ["E1"], async () => { + const said: string[] = []; + // The flip as the preview says to make it — whatever the preview binds `--yes` to. + const preview = await mesh(`converge ${CONTROL}`); + const flip = preview.match(new RegExp(`\`(converge ${CONTROL} --yes[^\`]*)\``))?.[1]; + assert.ok(flip, `the preview does not say how to make the flip:\n${preview}`); + said.push((await mesh(flip, 300_000)).trim().split("\n").slice(-3).join("\n")); + await settled(); + const table = await until(`the derived filter is loaded`, 300, async () => { + const t = await on(CONTROL, `nft list table inet mesh`); + return t.ok && /policy drop/.test(t.out) ? t.out : null; + }, () => ""); + const status = await until(`the found firewall is disabled`, 180, async () => { + const s = (await on(CONTROL, `ufw status`)).out; + return /Status: inactive/.test(s) ? s : null; + }, () => ""); + assert.ok((await on(CONTROL, `test -s /etc/ufw/user.rules && grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules`)).ok, + `the found firewall's configuration is not on disk any more`); + assert.match(await nodeShow(CONTROL), /mode\s+converged/, `the anchor is not reported converged`); + said.push(` ufw ${status.trim()} — /etc/ufw/user.rules and after.rules still on disk`); + await until(`the declared ${SERVED} answers over the private network`, 120, async () => + (await connects(JOINER, anchorOnMesh, SERVED)) ? true : null, () => ""); + assert.ok(!(await connects(JOINER, anchorOnMesh, HELD_REGISTRY)), `the undeclared ${HELD_REGISTRY} is still open`); + assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once converged`); + said.push(` ports ${SERVED} open over the private network; ${HELD_REGISTRY} closed; the store still closed from outside`); + said.push(table.split("\n").slice(0, 30).join("\n")); + return said.join("\n"); + }); + + await step("E3", ["E2"], async () => { + const said = (await mesh(`adopt ${CONTROL}`, 300_000)).trim(); + await settled(); + await until(`the found firewall is enabled again`, 180, async () => + /Status: active/.test((await on(CONTROL, `ufw status`)).out) ? true : null, () => ""); + await until(`the derived filter is gone`, 180, async () => + !(await on(CONTROL, `nft list table inet mesh`)).ok ? true : null, () => ""); + assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard is not restored`); + assertOpenings(await until(`the openings are back`, 180, async () => { + const o = await openings(); + return o.length ? o : null; + }, () => "")); + assert.match(await nodeShow(CONTROL), /mode\s+adopted since/, `the anchor is not reported adopted`); + assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once adopted again`); + return `${said}\n ufw active, table inet mesh gone, the guard and the openings back`; + }); + + stateOutcome(); +}, { timeout: 10_800_000 }); + +after(async () => { + if (KEEP || WARM) { + console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (${KEEP ? "MESH_LAB_KEEP" : "MESH_LAB_WARM"}).`); + return; + } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +for (const [code, title] of Object.entries(TITLE)) { + test(`${code} ${title}`, { skip, timeout: 60_000 }, () => { + const s = steps.get(code); + assert.ok(s?.ok, s ? `${s.why}` : `${code} never ran`); + }); +} diff --git a/test/integration/genesis-single.test.ts b/test/integration/genesis-single.test.ts index 4880815..cfeab2c 100644 --- a/test/integration/genesis-single.test.ts +++ b/test/integration/genesis-single.test.ts @@ -93,6 +93,7 @@ before(async () => { step: "getting the machine ready to be bootstrapped — the installer never ran", why: (err as Error).message, report: [], + said: "", }; } console.log(result.report.join("\n")); diff --git a/test/integration/genesis.ts b/test/integration/genesis.ts index 9b1853d..a5a77d7 100644 --- a/test/integration/genesis.ts +++ b/test/integration/genesis.ts @@ -26,6 +26,8 @@ export interface GenesisResult { step: string; why: string; report: string[]; + /** Everything the installer printed on its last attempt — what a bed asserts a refusal names. */ + said: string; } export interface GenesisOptions { @@ -80,6 +82,23 @@ export interface GenesisOptions { hostBinary?: string; /** What of the catalogue to build. A branch under test is the usual reason this is not main. */ catalogRef?: string; + /** + * Raise the machine adopted (novox/hq ADR 0100): what it runs and its firewall are kept. Without + * it the installer raises a converged node, and refuses a machine in use. + */ + adopted?: boolean; + /** Further installer flags, as the operator would type them — `--registry-port 5100`, `--dry-run`. */ + flags?: string[]; + /** + * How many times to run the installer. Three by default, for a pull the internet rate-limited; a + * bed that expects a REFUSAL runs it once, because a refusal is the answer, not a flake. + */ + attempts?: number; + /** + * Whether to ask the machine if it became a working mesh of one afterwards. Off for a run that is + * not meant to raise one — a dry run, or a refusal the bed expects. + */ + verify?: boolean; log?: (m: string) => void; } @@ -98,9 +117,10 @@ export async function genesis(o: GenesisOptions): Promise { const log = o.log ?? (() => {}); const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`]; + let said = ""; const stop = (step: string, why: string): GenesisResult => { report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`); - return { ok: false, step, why, report }; + return { ok: false, step, why, report, said }; }; const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => { @@ -136,7 +156,9 @@ export async function genesis(o: GenesisOptions): Promise { // the lab stands in for that by copying the whole tree once. const bundleTar = join(tmpdir(), `mesh-lab-catalogue-${process.pid}-${node}.tar`); execFileSync("tar", ["-cf", bundleTar, "-C", o.catalogDir, "."]); - await must(`mkdir -p ${catalogueOnMachine}/modules`); + // Cleared first: a bed that runs genesis more than once (a refusal, then the raise) finds the last + // run's staging files, and the machine refuses to open them for the push. + await must(`rm -f /tmp/catalogue.tar /tmp/foundation-template.lock && mkdir -p ${catalogueOnMachine}/modules`); await push(o.instanceId, node, bundleTar, "/tmp/catalogue.tar"); await must(`tar -xf /tmp/catalogue.tar -C ${catalogueOnMachine}/modules`); // The three genesis itself needs must be present, or the pivot cannot even begin — checked here @@ -184,6 +206,8 @@ export async function genesis(o: GenesisOptions): Promise { `--private-network wireguard`, `--packet-filter nftables`, `--host ${HOST_PATH}`, + ...(o.adopted ? [`--adopted`] : []), + ...(o.flags ?? []), // A service when the packaging was installed above (survives a reboot); otherwise the // background process, which does not — the installer refuses to invent a unit either way. ...(o.hostService ? [] as string[] : [`--host-in-background`]), @@ -193,20 +217,24 @@ export async function genesis(o: GenesisOptions): Promise { // but because the installer is idempotent by design and says so, and because the one thing that // fails for a reason which goes away by itself is a pull: the store, broker and registry come // from the internet, and a rate-limited anonymous pull is not this mesh's fault. - let said = ""; let step = ""; - for (let attempt = 1; attempt <= 3; attempt++) { + const attempts = o.attempts ?? 3; + for (let attempt = 1; attempt <= attempts; attempt++) { const ran = await on(command, 2_400_000); said = ran.out; log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`); if (ran.ok) { step = ""; break; } - step = stepIn(said); - if (attempt < 3) { + step = stepIn(said) || "an unnamed step"; + if (attempt < attempts) { log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`); await new Promise((r) => setTimeout(r, 30_000)); } } if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n")); + if (o.verify === false) { + report.push(`\nThe installer finished; not asked whether it raised a mesh (verify: false).`); + return { ok: true, step: "", why: "", report, said }; + } // ------------------------------------------------------------------------------------------ // Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting @@ -286,5 +314,5 @@ export async function genesis(o: GenesisOptions): Promise { } report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`); - return { ok: true, step: "", why: "", report }; + return { ok: true, step: "", why: "", report, said }; }