diff --git a/test/integration/adoption.test.ts b/test/integration/adoption.test.ts index 99f5e34..d2753fe 100644 --- a/test/integration/adoption.test.ts +++ b/test/integration/adoption.test.ts @@ -317,6 +317,12 @@ const SERVE_LOOP = /** Where the bed keeps what the machine looked like before the mesh arrived — on the machine, so a warm restore keeps it. */ const BEFORE = "/root/predecessor"; +/** A predecessor container with no restart policy: a runtime restart would lose it. */ +const NO_POLICY = "predecessor-nopolicy"; + +/** The broker's plaintext port: published on every interface, and the filter admits it from the mesh only. */ +const AMQP_PORT = 5672; + async function preparePredecessor(): Promise { const said: string[] = []; await must(CONTROL, `pacman -S --noconfirm --needed ufw`, 600_000); @@ -342,6 +348,19 @@ async function preparePredecessor(): Promise { `-v ${SERVICE_FILE}:/www/index.html:ro ${ALPINE} sh -c ${quote(SERVE_LOOP)}`, 600_000); await must(CONTROL, `docker run -d --name predecessor-registry --restart unless-stopped -p ${HELD_REGISTRY}:5000 ${REGISTRY_IMAGE}`, 600_000); + // A container the predecessor left running with no restart policy: a restart of the runtime + // would not bring it back, which is what ADR 0102 forbids the mesh from causing. + await must(CONTROL, `docker run -d --name ${NO_POLICY} ${ALPINE} sleep infinity`, 600_000); + // And a setting of the machine's own in the runtime's file, beside the registries it ships with. + await must(CONTROL, + `python3 - <<'EOF' +import json +f="/etc/docker/daemon.json" +d=json.load(open(f)) +d["log-opts"]={"max-size":"7m"} +json.dump(d,open(f,"w"),indent=2) +EOF +systemctl reload docker`); said.push(` containers ${SERVICE} on ${SERVED}, predecessor-registry on ${HELD_REGISTRY}`); // The predecessor's control, which the operator stops before adopting (the record's words). @@ -357,6 +376,7 @@ async function preparePredecessor(): Promise { `sha256sum ${SERVICE_FILE} | cut -d' ' -f1 > ${BEFORE}/file.sha256`, `cp ${SERVICE_FILE} ${BEFORE}/file`, `docker inspect -f '{{.Id}}' ${SERVICE} > ${BEFORE}/container.id`, + `docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY} > ${BEFORE}/nopolicy.id`, `ufw show added > ${BEFORE}/ufw-added.txt`, ].join(" && ")); await until(`the predecessor's ${SERVICE} answers the joiner`, 60, async () => @@ -412,10 +432,11 @@ const TITLE: Record = { D1: "assigning prepares: the module holds the found container and file, and neither changes", D2: "a predecessor still writing is caught: the held file's change is reported, and not reverted", D3: "converging refuses while the service's module holds its found container", - D4: "taking cuts over: the found container and file are replaced, the original kept, the port opened", + D4: "taking cuts over: the found container and file are replaced, the original kept, the port reachable", E1: "converging previews: the service's port, a published port no rule mentions, and the modules it takes", E2: "the flip: the derived filter loaded, the found firewall disabled with its configuration on disk, the declared port open and the undeclared closed", E3: "returned to adopted: the found firewall enabled again, the derived filter gone, the openings back", + F1: "unassigning takes the mesh's opening away and leaves the operator's own rule", }; function stateOutcome(): void { @@ -469,7 +490,7 @@ before(async () => { catalogSource: forgeUrl("mesh-catalog"), catalogRef: "lab", sdkSource: forgeUrl("mesh-sdk"), sdkRef: "lab", site: "hosting", - hostService: true, + hostService: false, ...(binary ? { hostBinary: binary } : {}), log: (m) => console.log(m), ...o, @@ -586,6 +607,19 @@ before(async () => { const unmarked = added.filter((r) => !marked(r)); assert.deepEqual(unmarked, [], `the found firewall gained rules not marked as the mesh's:\n${unmarked.join("\n")}`); assert.ok(added.length > 0, `the mesh opened nothing through the found firewall`); + // ADR 0102: the runtime's file is written into, never over, and the runtime is reloaded. + const daemon = JSON.parse(await must(CONTROL, `cat /etc/docker/daemon.json`)) as + { "log-opts"?: Record; "insecure-registries"?: string[] }; + assert.equal(daemon["log-opts"]?.["max-size"], "7m", `the machine's own runtime setting was replaced: ${JSON.stringify(daemon)}`); + assert.ok((daemon["insecure-registries"] ?? []).some((r) => r.includes(":")), + `the mesh's registry trust is not in the runtime's file: ${JSON.stringify(daemon)}`); + assert.ok((daemon["insecure-registries"] ?? []).length > 1, + `the machine's own registries were replaced rather than added to: ${JSON.stringify(daemon)}`); + const stillUp = (await must(CONTROL, `docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY}`)).trim(); + assert.match(stillUp, /^true /, `the container with no restart policy is not running: ${stillUp}`); + assert.equal(stillUp, (await must(CONTROL, `cat ${BEFORE}/nopolicy.id`)).trim(), + `the container with no restart policy was restarted or replaced`); + said.push(` runtime file the machine's log-opts kept, the mesh's registry added; ${NO_POLICY} still up`); said.push(` firewall ${was.length} rule(s) kept, ${added.length} added, every one marked:`, ...added.map((r) => ` ${r}`)); return said.join("\n"); }); @@ -599,6 +633,36 @@ before(async () => { said.push(` outsider -> ${STORE_PORT} refused, before and after \`ufw reload\``); assert.ok(await connects(OUTSIDER, ANCHOR, BUS_PORT), `the bus does not answer a machine that has not enrolled`); said.push(` outsider -> ${BUS_PORT} the bus answers`); + + // **What the guard is for** (ADR 0100, 0103): the store must be unreachable from outside + // *whatever the found firewall does*. With ufw admitting both ports, only the guard is left + // between the outsider and the store — and with the guard gone they are reachable, which is + // what makes this a test of the guard rather than of ufw. + const admit = [STORE_PORT, AMQP_PORT].map((p) => + `ufw route allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`); + try { + await must(CONTROL, admit.join(" && ")); + await sleep(2_000); + assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), + `the store answers from outside once the found firewall admits it — the guard refuses nothing`); + assert.ok(!(await connects(OUTSIDER, ANCHOR, AMQP_PORT)), + `the broker's plaintext port answers from outside once the found firewall admits it`); + said.push(` outsider -> ${STORE_PORT}, ${AMQP_PORT} still refused with the found firewall admitting both — the guard's own refusal`); + + // The positive control: without the guard, both answer. Anything else would mean the probe + // could not have failed. + await must(CONTROL, `nft delete table inet mesh_guard`); + await sleep(2_000); + const openNow = (await connects(OUTSIDER, ANCHOR, STORE_PORT)) || (await connects(OUTSIDER, ANCHOR, AMQP_PORT)); + await must(CONTROL, `systemctl reload mesh-guard.service || systemctl restart mesh-guard.service`); + await sleep(2_000); + assert.ok(openNow, `neither port answered with the guard deleted, so the guard is not what refuses them`); + assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store stayed open after the guard was loaded again`); + said.push(` guard deleted both answered; loaded again, both refused`); + } finally { + await on(CONTROL, [STORE_PORT, AMQP_PORT].map((p) => + `ufw route delete allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`).join("; ")); + } return said.join("\n"); }); @@ -613,8 +677,12 @@ before(async () => { // container to get there, on an adopted node as on a converged one. The probe admits it for // itself alone, and takes the rule away again. await must(CONTROL, `ufw allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`); - const fromContainer = await on(CONTROL, `docker run --rm ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`, 180_000); - await must(CONTROL, `ufw delete allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`); + let fromContainer: { ok: boolean; out: string }; + try { + fromContainer = await on(CONTROL, `docker run --rm ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`, 180_000); + } finally { + await on(CONTROL, `ufw delete allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`); + } if (!fromContainer.ok) { // Evidence, so the cause can be read from this run rather than guessed at the next one. const evidence = await on(CONTROL, [ @@ -740,8 +808,16 @@ before(async () => { const s = await nodeShow(CONTROL); return /hello-web\/index\.html[^\n]*REWRITTEN/i.test(s) ? s : null; }, () => ""); + // Stop the writer first, then hash: while it rewrites every ten seconds, a file the host + // reverted in between would still read as the predecessor's a moment later. + await must(CONTROL, `systemctl stop predecessor-sync`); + const was = await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`); + await pushAndSettle(CONTROL); + await sleep(5_000); const now = await must(CONTROL, `cat ${SERVICE_FILE}`); assert.match(now, /synced \d+/, `the host reverted what the predecessor wrote:\n${now}`); + assert.equal(await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`), was, + `the held file changed under a push after the predecessor stopped writing`); return show.trim(); } finally { await on(CONTROL, `systemctl stop predecessor-sync`); @@ -856,6 +932,28 @@ before(async () => { return `${said}\n ufw active, table inet mesh gone, the guard and the openings back`; }); + // ---- what the mesh added, it takes back; what it found, it leaves --------------------------- + await step("F1", ["E3"], async () => { + const said: string[] = []; + const before = await ufwAdded(); + const operators = before.filter((r) => !marked(r) && new RegExp(`\\b${SERVED}\\b`).test(r)); + assert.ok(operators.length > 0, `the operator's own rules for ${SERVED} are already gone:\n${before.join("\n")}`); + await mesh(`unassign ${CONTROL} ${SERVICE}`); + await pushAndSettle(CONTROL); + let lastRules: string[] = before; + const after = await until(`the mesh's own rules for ${SERVED} are gone`, 180, async () => { + const rules = await ufwAdded(); + lastRules = rules; + return rules.some((r) => marked(r) && new RegExp(`opening-tcp-${SERVED}-`).test(r)) ? null : rules; + }, () => lastRules.join("\n")); + for (const rule of operators) { + assert.ok(after.includes(rule), `the operator's own rule went with the mesh's opening: ${rule}\n${after.join("\n")}`); + } + said.push(` kept ${operators.length} rule(s) of the operator's, unmarked, after the module was unassigned`); + said.push(...operators.map((r) => ` ${r}`)); + return said.join("\n"); + }); + stateOutcome(); }, { timeout: 10_800_000 });