|
|
|
@@ -0,0 +1,851 @@
|
|
|
|
|
/**
|
|
|
|
|
* A MACHINE IN USE IS ADOPTED BEFORE IT IS CONVERGED (novox/hq ADR 0100, and ADR 0101 on what
|
|
|
|
|
* "in use" ignores).
|
|
|
|
|
*
|
|
|
|
|
* The mesh replaces a predecessor running on the same machines. This bed prepares a machine the
|
|
|
|
|
* way the predecessor leaves one — the record's own words, "How it is checked":
|
|
|
|
|
*
|
|
|
|
|
* - its firewall (ufw) allowing a served port and denying the rest, incoming and routed, with the
|
|
|
|
|
* predecessor's published container ports filtered through it (the ufw-docker arrangement);
|
|
|
|
|
* - a service container, `hello-web`, listening on that port under a name the catalogue's
|
|
|
|
|
* `hello-web` module also uses, and a file at a path that module declares;
|
|
|
|
|
* - a stand-in for the predecessor's control that rewrites that file, stopped by the operator
|
|
|
|
|
* before adoption as the record prescribes, and started again later to play one forgotten;
|
|
|
|
|
* - a container holding the registry's port.
|
|
|
|
|
*
|
|
|
|
|
* Then it asks, in the record's order: a converged genesis refuses; an adopted one refuses the held
|
|
|
|
|
* registry port and comes up on another; nothing that serves changed; the store is unreachable
|
|
|
|
|
* from outside and reachable where it must be; the mesh works through the found firewall, across a
|
|
|
|
|
* reload and a reboot; a predecessor still writing is caught; assigning prepares and taking cuts
|
|
|
|
|
* over; converging previews, refuses while a found container is held, flips, and returns.
|
|
|
|
|
*
|
|
|
|
|
* **The module under migration is the catalogue's `hello-web` with its route requirement taken
|
|
|
|
|
* off**, read from the catalogue (never a copy): the route needs a proxy module and a public name,
|
|
|
|
|
* neither of which is what adoption is about. Its names — the container, the file, the port — are
|
|
|
|
|
* the catalogue's, which is the point: they are what the predecessor also uses.
|
|
|
|
|
*
|
|
|
|
|
* **The forge is in the lab.** Genesis builds the control plane and the catalogue from a
|
|
|
|
|
* repository and a commit; this bed serves the checkouts it was pointed at (their HEADs) from the
|
|
|
|
|
* `outsider` machine, so the run builds exactly the code under test and nothing on the workstation
|
|
|
|
|
* listens for the lab.
|
|
|
|
|
*
|
|
|
|
|
* Each step is recorded rather than allowed to throw; a step whose dependency failed is not
|
|
|
|
|
* attempted, and the report says which.
|
|
|
|
|
*
|
|
|
|
|
* MESH_LAB_INCUS='sudo -n incus'
|
|
|
|
|
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
|
|
|
|
|
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock
|
|
|
|
|
* MESH_LAB_CATALOG=<mesh-catalog>/modules MESH_LAB_MODULES=<mesh-controller>/examples/modules
|
|
|
|
|
* MESH_TOOLS=<mesh-tools> MESH_SDK=<mesh-sdk> (default: the checkouts beside this one)
|
|
|
|
|
* MESH_LAB_KEEP=1 leave it standing MESH_LAB_WARM=1 iterate from the adopted foundation
|
|
|
|
|
*/
|
|
|
|
|
import { test, before, after } from "node:test";
|
|
|
|
|
import assert from "node:assert/strict";
|
|
|
|
|
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
|
|
|
import { execFileSync } from "node:child_process";
|
|
|
|
|
import { tmpdir } from "node:os";
|
|
|
|
|
import { dirname, join, resolve } from "node:path";
|
|
|
|
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
|
|
|
import { raise } from "../../src/lifecycle/raise.ts";
|
|
|
|
|
import { destroy, exec, push, instanceNameOf } from "../../src/lifecycle/operate.ts";
|
|
|
|
|
import { bootstrapBinaryPath, hostBinaryPath, placeBootstrap, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
|
|
|
import { waitUntilAllUsable } from "../../src/lifecycle/ready.ts";
|
|
|
|
|
import { incus } from "../../src/incus/client.ts";
|
|
|
|
|
import { catalogueRoot } from "../../src/repos.ts";
|
|
|
|
|
import { ready, returnTo, keep } from "../../src/warm.ts";
|
|
|
|
|
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueManifest } from "./harness.ts";
|
|
|
|
|
import { genesis, type GenesisOptions } from "./genesis.ts";
|
|
|
|
|
|
|
|
|
|
const SCENARIO = "adoption";
|
|
|
|
|
const CONTROL = "anchor";
|
|
|
|
|
const JOINER = "joiner";
|
|
|
|
|
const OUTSIDER = "outsider";
|
|
|
|
|
const ANCHOR = "192.0.2.10";
|
|
|
|
|
const JOINER_ADDRESS = "192.0.2.20";
|
|
|
|
|
const FORGE = "192.0.2.30";
|
|
|
|
|
|
|
|
|
|
/** The port the predecessor serves, and the module that also names its container and file. */
|
|
|
|
|
const SERVED = 8080;
|
|
|
|
|
const SERVICE = "hello-web";
|
|
|
|
|
const SERVICE_FILE = "/var/lib/hello-web/index.html";
|
|
|
|
|
const PREDECESSOR_PAGE = "hello from the predecessor\n";
|
|
|
|
|
/** The registry's port, which the predecessor holds — and the one the mesh is given instead. */
|
|
|
|
|
const HELD_REGISTRY = 5000;
|
|
|
|
|
const REGISTRY_PORT = 5100;
|
|
|
|
|
const STORE_PORT = 5432;
|
|
|
|
|
const BUS_PORT = 5671;
|
|
|
|
|
const HUB_PORT = 51820;
|
|
|
|
|
const NETWORK_MODULE = "networking";
|
|
|
|
|
const FILTER_MODULE = "nftables";
|
|
|
|
|
|
|
|
|
|
/** Upstream images, pinned as the catalogue pins them (the harness's table). */
|
|
|
|
|
const ALPINE = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b";
|
|
|
|
|
const REGISTRY_IMAGE = "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
|
|
|
|
|
|
|
|
|
|
const capability = await labIsUsable();
|
|
|
|
|
const binary = hostBinaryPath();
|
|
|
|
|
const installer = bootstrapBinaryPath();
|
|
|
|
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
|
|
|
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
|
|
|
|
const modulesDir = process.env["MESH_LAB_MODULES"] ?? "";
|
|
|
|
|
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
|
|
|
|
const WARM = !!process.env["MESH_LAB_WARM"];
|
|
|
|
|
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "adoption-live" : undefined);
|
|
|
|
|
|
|
|
|
|
/** The checkouts this run builds from, served by the lab's forge. */
|
|
|
|
|
const REPOS: Record<string, string> = {
|
|
|
|
|
"mesh-controller": modulesDir ? dirname(dirname(modulesDir)) : "",
|
|
|
|
|
"mesh-catalog": catalogDir ? catalogueRoot(catalogDir) : "",
|
|
|
|
|
"mesh-tools": process.env["MESH_TOOLS"] ?? resolve(process.cwd(), "..", "mesh-tools"),
|
|
|
|
|
"mesh-sdk": process.env["MESH_SDK"] ?? resolve(process.cwd(), "..", "mesh-sdk"),
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const skip =
|
|
|
|
|
!capability.usable ? capability.why :
|
|
|
|
|
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
|
|
|
|
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
|
|
|
|
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
|
|
|
|
|
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
|
|
|
|
!modulesDir ? "MESH_LAB_MODULES is not set, so there is no control-plane checkout to build from" :
|
|
|
|
|
Object.entries(REPOS).find(([, p]) => !p || !existsSync(resolve(p, ".git")))
|
|
|
|
|
?.map((x) => `no checkout of ${x[0]} at ${x[1]}`)[0] ?? false;
|
|
|
|
|
|
|
|
|
|
let instanceId = "";
|
|
|
|
|
|
|
|
|
|
// ---- talking to the machines ------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
function quote(s: string): string {
|
|
|
|
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
|
|
|
}
|
|
|
|
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
|
|
|
const { stdout } = await exec(instanceId, machine, [
|
|
|
|
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
|
|
|
], timeoutMs);
|
|
|
|
|
const marker = stdout.lastIndexOf("__exit=");
|
|
|
|
|
if (marker < 0) return { out: stdout, ok: false };
|
|
|
|
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
|
|
|
}
|
|
|
|
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
|
|
|
|
const { out, ok } = await on(machine, command, timeoutMs);
|
|
|
|
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
/** The control plane, retried across the brief windows in which the mesh recreates it. */
|
|
|
|
|
async function meshSays(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
|
|
|
const deadline = Date.now() + (timeoutMs ?? 120_000);
|
|
|
|
|
for (;;) {
|
|
|
|
|
const r = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
|
|
|
if (r.ok) return r;
|
|
|
|
|
if (/is not running|No such container|No such exec instance|Cannot connect to the Docker daemon|is restarting/i.test(r.out) &&
|
|
|
|
|
Date.now() < deadline) {
|
|
|
|
|
await sleep(2_000);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
return r;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
|
|
|
const r = await meshSays(command, timeoutMs);
|
|
|
|
|
if (!r.ok) throw new Error(`${CONTROL}: mesh-controller ${command}\n${r.out}`);
|
|
|
|
|
return r.out;
|
|
|
|
|
}
|
|
|
|
|
function sleep(ms: number): Promise<void> {
|
|
|
|
|
return new Promise((r) => setTimeout(r, ms));
|
|
|
|
|
}
|
|
|
|
|
/** Poll until `probe` returns a value, or fail naming the last thing it saw. */
|
|
|
|
|
async function until<T>(what: string, seconds: number, probe: () => Promise<T | null>, last: () => string): Promise<T> {
|
|
|
|
|
const deadline = Date.now() + seconds * 1000;
|
|
|
|
|
for (;;) {
|
|
|
|
|
const got = await probe();
|
|
|
|
|
if (got !== null) return got;
|
|
|
|
|
if (Date.now() > deadline) throw new Error(`${what} — not within ${seconds}s. Last:\n${last()}`);
|
|
|
|
|
await sleep(5_000);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
/** Whether a TCP connection from `machine` to address:port opens within three seconds. */
|
|
|
|
|
async function connects(machine: string, address: string, port: number): Promise<boolean> {
|
|
|
|
|
return (await on(machine, `timeout 4 bash -c ${quote(`</dev/tcp/${address}/${port}`)}`)).ok;
|
|
|
|
|
}
|
|
|
|
|
/** Register a module with the control plane from a manifest's text. */
|
|
|
|
|
async function registerModule(module: string, manifest: string): Promise<string> {
|
|
|
|
|
const local = join(tmpdir(), `mesh-lab-adoption-${process.pid}-${module}.json`);
|
|
|
|
|
writeFileSync(local, manifest);
|
|
|
|
|
await push(instanceId, CONTROL, local, `/tmp/${module}.json`);
|
|
|
|
|
await must(CONTROL, `docker cp /tmp/${module}.json mesh-controller:/${module}.json`);
|
|
|
|
|
return mesh(`module add /${module}.json`);
|
|
|
|
|
}
|
|
|
|
|
async function nodeShow(node: string): Promise<string> {
|
|
|
|
|
return mesh(`node show ${node}`);
|
|
|
|
|
}
|
|
|
|
|
/** The anchor's address on the private network. */
|
|
|
|
|
async function meshAddressOf(machine: string): Promise<string> {
|
|
|
|
|
const out = await must(machine, `ip -4 -o addr show dev mesh0`);
|
|
|
|
|
const found = out.match(/inet (\d+\.\d+\.\d+\.\d+)\//)?.[1];
|
|
|
|
|
assert.ok(found, `${machine} has no address on mesh0:\n${out}`);
|
|
|
|
|
return found;
|
|
|
|
|
}
|
|
|
|
|
/** The rules ufw was given, one per line, as `ufw show added` prints them. */
|
|
|
|
|
async function ufwAdded(): Promise<string[]> {
|
|
|
|
|
const out = await must(CONTROL, `ufw show added`);
|
|
|
|
|
return out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw "));
|
|
|
|
|
}
|
|
|
|
|
function marked(rule: string): boolean {
|
|
|
|
|
return /comment 'mesh-host /.test(rule);
|
|
|
|
|
}
|
|
|
|
|
async function restartMachine(machine: string): Promise<void> {
|
|
|
|
|
const name = await instanceNameOf(instanceId, machine);
|
|
|
|
|
await incus(["restart", name], 180_000);
|
|
|
|
|
await waitUntilAllUsable([name], 300, (m) => console.log(` restart: ${m}`));
|
|
|
|
|
}
|
|
|
|
|
/** Until the anchor reports what it was last sent as applied and current. */
|
|
|
|
|
async function settled(node = CONTROL, withinMs = 300_000): Promise<void> {
|
|
|
|
|
let last = "";
|
|
|
|
|
await until(`${node} reports the declaration it was sent as applied and current`, withinMs / 1000, async () => {
|
|
|
|
|
const asked = await meshSays(`status --json`);
|
|
|
|
|
last = asked.out;
|
|
|
|
|
if (!asked.ok) return null;
|
|
|
|
|
try {
|
|
|
|
|
const state = JSON.parse(asked.out) as {
|
|
|
|
|
wrong: { node: string; outcome: string }[];
|
|
|
|
|
waiting: { node: string }[];
|
|
|
|
|
reported: { node: string; outcome: string; current: boolean }[];
|
|
|
|
|
};
|
|
|
|
|
const bad = state.wrong.find((w) => w.node === node);
|
|
|
|
|
if (bad) throw new Error(`${node} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
|
|
|
|
const word = state.reported.find((r) => r.node === node);
|
|
|
|
|
return !state.waiting.some((w) => w.node === node) && word?.outcome === "applied" && word.current ? true : null;
|
|
|
|
|
} catch (err) {
|
|
|
|
|
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
}, () => last);
|
|
|
|
|
}
|
|
|
|
|
/** Send a node what it should be, and wait until it says it applied it. */
|
|
|
|
|
async function pushAndSettle(node: string): Promise<string> {
|
|
|
|
|
const said = await mesh(`push ${node}`, 600_000);
|
|
|
|
|
await settled(node);
|
|
|
|
|
return said;
|
|
|
|
|
}
|
|
|
|
|
function tokenFrom(said: string): string {
|
|
|
|
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
|
|
|
assert.ok(found, `no token in:\n${said}`);
|
|
|
|
|
return found;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---- the lab's forge ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Serve the checkouts under test from the outsider machine, over git's own protocol.
|
|
|
|
|
*
|
|
|
|
|
* Each checkout's HEAD is pushed into a bare repository as `main` and `lab`, the lot is carried in,
|
|
|
|
|
* and a git daemon answers on the outsider's scenario address — which the anchor's builder reaches
|
|
|
|
|
* over the hosting segment. Returns the commit each repository is served at.
|
|
|
|
|
*/
|
|
|
|
|
async function raiseForge(): Promise<Record<string, string>> {
|
|
|
|
|
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-forge-"));
|
|
|
|
|
const heads: Record<string, string> = {};
|
|
|
|
|
try {
|
|
|
|
|
for (const [name, checkout] of Object.entries(REPOS)) {
|
|
|
|
|
const bare = join(dir, `${name}.git`);
|
|
|
|
|
execFileSync("git", ["init", "-q", "--bare", bare]);
|
|
|
|
|
execFileSync("git", ["-C", checkout, "push", "-q", "--force", bare,
|
|
|
|
|
"HEAD:refs/heads/main", "HEAD:refs/heads/lab"], { stdio: "pipe" });
|
|
|
|
|
heads[name] = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
|
|
|
|
}
|
|
|
|
|
const tar = join(tmpdir(), `mesh-lab-forge-${process.pid}.tar`);
|
|
|
|
|
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
|
|
|
|
|
await push(instanceId, OUTSIDER, tar, "/tmp/forge.tar");
|
|
|
|
|
rmSync(tar, { force: true });
|
|
|
|
|
} finally {
|
|
|
|
|
rmSync(dir, { recursive: true, force: true });
|
|
|
|
|
}
|
|
|
|
|
await must(OUTSIDER, `command -v git >/dev/null || pacman -S --noconfirm --needed git`, 600_000);
|
|
|
|
|
// Owned by the daemon's user: extracted as the workstation's uid, git refuses to serve a
|
|
|
|
|
// repository someone else owns ("dubious ownership"), and the clone fails with no reason given.
|
|
|
|
|
await must(OUTSIDER, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` +
|
|
|
|
|
`git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE} --pid-file=/run/git-daemon.pid`);
|
|
|
|
|
await must(OUTSIDER, `git ls-remote git://${FORGE}/mesh-controller.git lab`);
|
|
|
|
|
await until("the lab's forge answers the anchor", 60, async () =>
|
|
|
|
|
(await connects(CONTROL, FORGE, 9418)) ? true : null, () => "no connection to 9418");
|
|
|
|
|
return heads;
|
|
|
|
|
}
|
|
|
|
|
const forgeUrl = (repo: string) => `git://${FORGE}/${repo}.git`;
|
|
|
|
|
|
|
|
|
|
// ---- the predecessor ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* The ufw-docker arrangement: published container ports pass through ufw's route rules, and
|
|
|
|
|
* traffic from private ranges is let through. It is how a ufw machine filters what docker publishes
|
|
|
|
|
* at all — without it docker's own rules bypass ufw entirely (novox/hq research 012 measured 52
|
|
|
|
|
* forwarding rules on the control-node, one per served port).
|
|
|
|
|
*/
|
|
|
|
|
const UFW_DOCKER = `
|
|
|
|
|
# BEGIN UFW AND DOCKER
|
|
|
|
|
*filter
|
|
|
|
|
:ufw-user-forward - [0:0]
|
|
|
|
|
:ufw-docker-logging-deny - [0:0]
|
|
|
|
|
:DOCKER-USER - [0:0]
|
|
|
|
|
-A DOCKER-USER -j ufw-user-forward
|
|
|
|
|
-A DOCKER-USER -j RETURN -s 10.0.0.0/8
|
|
|
|
|
-A DOCKER-USER -j RETURN -s 172.16.0.0/12
|
|
|
|
|
-A DOCKER-USER -j RETURN -s 192.168.0.0/16
|
|
|
|
|
-A DOCKER-USER -p udp -m udp --sport 53 --dport 1024:65535 -j RETURN
|
|
|
|
|
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 192.168.0.0/16
|
|
|
|
|
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 10.0.0.0/8
|
|
|
|
|
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 172.16.0.0/12
|
|
|
|
|
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 192.168.0.0/16
|
|
|
|
|
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 10.0.0.0/8
|
|
|
|
|
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 172.16.0.0/12
|
|
|
|
|
-A DOCKER-USER -j RETURN
|
|
|
|
|
-A ufw-docker-logging-deny -j DROP
|
|
|
|
|
COMMIT
|
|
|
|
|
# END UFW AND DOCKER
|
|
|
|
|
`;
|
|
|
|
|
|
|
|
|
|
/** The stand-in for the predecessor's configuration sync: it rewrites the file every ten seconds. */
|
|
|
|
|
const STAND_IN = `[Unit]
|
|
|
|
|
Description=Stand-in for the predecessor's configuration sync: rewrites a file a catalogue module declares
|
|
|
|
|
|
|
|
|
|
[Service]
|
|
|
|
|
ExecStart=/bin/sh -c 'while true; do printf "hello from the predecessor, synced %%s\\\\n" "$(date +%%s)" > ${SERVICE_FILE}; sleep 10; done'
|
|
|
|
|
`;
|
|
|
|
|
|
|
|
|
|
/** The page the predecessor's service loop serves — the catalogue module's own loop, verbatim. */
|
|
|
|
|
const SERVE_LOOP =
|
|
|
|
|
"while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done";
|
|
|
|
|
|
|
|
|
|
/** Where the bed keeps what the machine looked like before the mesh arrived — on the machine, so a warm restore keeps it. */
|
|
|
|
|
const BEFORE = "/root/predecessor";
|
|
|
|
|
|
|
|
|
|
async function preparePredecessor(): Promise<string> {
|
|
|
|
|
const said: string[] = [];
|
|
|
|
|
await must(CONTROL, `pacman -S --noconfirm --needed ufw`, 600_000);
|
|
|
|
|
const rules = join(tmpdir(), `mesh-lab-ufw-docker-${process.pid}`);
|
|
|
|
|
writeFileSync(rules, UFW_DOCKER);
|
|
|
|
|
await push(instanceId, CONTROL, rules, "/tmp/ufw-docker.rules");
|
|
|
|
|
await must(CONTROL, [
|
|
|
|
|
`grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules || cat /tmp/ufw-docker.rules >> /etc/ufw/after.rules`,
|
|
|
|
|
`ufw default deny incoming`,
|
|
|
|
|
`ufw default allow outgoing`,
|
|
|
|
|
`ufw default deny routed`,
|
|
|
|
|
`ufw allow 22/tcp`,
|
|
|
|
|
`ufw allow ${SERVED}/tcp`,
|
|
|
|
|
`ufw route allow proto tcp from any to any port ${SERVED}`,
|
|
|
|
|
`ufw --force enable`,
|
|
|
|
|
`systemctl enable ufw`,
|
|
|
|
|
].join(" && "));
|
|
|
|
|
said.push(` firewall ufw: deny incoming and routed; allow 22 and ${SERVED}; ufw-docker after.rules`);
|
|
|
|
|
|
|
|
|
|
await must(CONTROL, `mkdir -p /var/lib/hello-web && printf %s ${quote(PREDECESSOR_PAGE)} > ${SERVICE_FILE}`);
|
|
|
|
|
await must(CONTROL,
|
|
|
|
|
`docker run -d --name ${SERVICE} --restart unless-stopped -p ${SERVED}:${SERVED} ` +
|
|
|
|
|
`-v ${SERVICE_FILE}:/www/index.html:ro ${ALPINE} sh -c ${quote(SERVE_LOOP)}`, 600_000);
|
|
|
|
|
await must(CONTROL,
|
|
|
|
|
`docker run -d --name predecessor-registry --restart unless-stopped -p ${HELD_REGISTRY}:5000 ${REGISTRY_IMAGE}`, 600_000);
|
|
|
|
|
said.push(` containers ${SERVICE} on ${SERVED}, predecessor-registry on ${HELD_REGISTRY}`);
|
|
|
|
|
|
|
|
|
|
// The predecessor's control, which the operator stops before adopting (the record's words).
|
|
|
|
|
const unit = join(tmpdir(), `mesh-lab-stand-in-${process.pid}`);
|
|
|
|
|
writeFileSync(unit, STAND_IN);
|
|
|
|
|
await push(instanceId, CONTROL, unit, "/etc/systemd/system/predecessor-sync.service");
|
|
|
|
|
await must(CONTROL, `systemctl daemon-reload && systemctl start predecessor-sync && sleep 12 && systemctl stop predecessor-sync`);
|
|
|
|
|
said.push(` stand-in predecessor-sync rewrote ${SERVICE_FILE}, then the operator stopped it`);
|
|
|
|
|
|
|
|
|
|
// What the machine was, recorded ON the machine so a restored warm instance still has it.
|
|
|
|
|
await must(CONTROL, [
|
|
|
|
|
`mkdir -p ${BEFORE}`,
|
|
|
|
|
`sha256sum ${SERVICE_FILE} | cut -d' ' -f1 > ${BEFORE}/file.sha256`,
|
|
|
|
|
`cp ${SERVICE_FILE} ${BEFORE}/file`,
|
|
|
|
|
`docker inspect -f '{{.Id}}' ${SERVICE} > ${BEFORE}/container.id`,
|
|
|
|
|
`ufw show added > ${BEFORE}/ufw-added.txt`,
|
|
|
|
|
].join(" && "));
|
|
|
|
|
await until(`the predecessor's ${SERVICE} answers the joiner`, 60, async () =>
|
|
|
|
|
(await on(JOINER, `curl -s --max-time 3 http://${ANCHOR}:${SERVED}/`)).out.includes("hello from the predecessor") ? true : null,
|
|
|
|
|
() => "no answer");
|
|
|
|
|
said.push((await must(CONTROL, `ufw status verbose`)).trim());
|
|
|
|
|
said.push((await must(CONTROL, `docker ps --format '{{.Names}}\t{{.Ports}}'`)).trim());
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---- steps, recorded rather than thrown --------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
interface Step { code: string; title: string; ok: boolean; why: string; said: string; seconds: number; warm?: boolean }
|
|
|
|
|
const steps = new Map<string, Step>();
|
|
|
|
|
|
|
|
|
|
async function step(code: string, needs: string[], fn: () => Promise<string>): Promise<void> {
|
|
|
|
|
const title = TITLE[code]!;
|
|
|
|
|
const missing = needs.filter((n) => !steps.get(n)?.ok);
|
|
|
|
|
if (missing.length) {
|
|
|
|
|
steps.set(code, { code, title, ok: false, seconds: 0, said: "",
|
|
|
|
|
why: `not attempted — ${missing.join(", ")} did not succeed` });
|
|
|
|
|
console.log(`[${code}] SKIP ${title}`);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
console.log(`\n[${code}] ---- ${title} ----`);
|
|
|
|
|
const began = Date.now();
|
|
|
|
|
const took = () => Math.round((Date.now() - began) / 1000);
|
|
|
|
|
try {
|
|
|
|
|
const said = await fn();
|
|
|
|
|
steps.set(code, { code, title, ok: true, why: "", said, seconds: took() });
|
|
|
|
|
console.log(`${said}\n[${code}] PASS ${title} (${took()}s)`);
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const why = (err as Error).message;
|
|
|
|
|
steps.set(code, { code, title, ok: false, why, said: "", seconds: took() });
|
|
|
|
|
console.log(`[${code}] FAIL ${title} (${took()}s)\n${why.split("\n").slice(0, 40).join("\n")}`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** The plan, in the record's order. Codes are stable; titles may be reworded. */
|
|
|
|
|
const TITLE: Record<string, string> = {
|
|
|
|
|
P0: "the machine is prepared the way the predecessor leaves one",
|
|
|
|
|
F0: "a converged genesis on a FRESH machine is not refused — its own resolver does not make it in use (ADR 0101)",
|
|
|
|
|
A1: "a converged genesis refuses the machine in use, naming every container and listener it counted",
|
|
|
|
|
A2: "an adopted genesis refuses the registry's held port, naming what holds it",
|
|
|
|
|
A3: "given another registry port, the adopted foundation comes up — and stays on that port as modules",
|
|
|
|
|
B1: "nothing that serves changed: the service answers, its file and container are untouched, the firewall gained only the mesh's marked rules",
|
|
|
|
|
B2: "the store is unreachable from outside, before and after the found firewall reloads; the bus answers a machine not yet enrolled",
|
|
|
|
|
B4: "the store is reachable from a container on the node itself",
|
|
|
|
|
C1: "a second machine enrols through the found firewall and joins the private network",
|
|
|
|
|
B3: "the store is reachable over the private network",
|
|
|
|
|
C2: "after the found firewall reloads, the openings are there and the mesh still works",
|
|
|
|
|
C3: "after the machine reboots, the openings are there and the mesh still works",
|
|
|
|
|
D1: "assigning prepares: the module holds the found container and file, and neither changes",
|
|
|
|
|
D2: "a predecessor still writing is caught: the held file's change is reported, and not reverted",
|
|
|
|
|
D3: "converging refuses while the service's module holds its found container",
|
|
|
|
|
D4: "taking cuts over: the found container and file are replaced, the original kept, the port opened",
|
|
|
|
|
E1: "converging previews: the service's port, a published port no rule mentions, and the modules it takes",
|
|
|
|
|
E2: "the flip: the derived filter loaded, the found firewall disabled with its configuration on disk, the declared port open and the undeclared closed",
|
|
|
|
|
E3: "returned to adopted: the found firewall enabled again, the derived filter gone, the openings back",
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
function stateOutcome(): void {
|
|
|
|
|
console.log(`\n================ ADOPTION: WHAT WAS ESTABLISHED ================`);
|
|
|
|
|
let established = 0;
|
|
|
|
|
for (const code of Object.keys(TITLE)) {
|
|
|
|
|
const s = steps.get(code);
|
|
|
|
|
const mark = !s ? "NEVER" : s.warm ? "WARM" : s.ok ? "PASS" : s.why.startsWith("not attempted") ? "SKIP" : "FAIL";
|
|
|
|
|
if (s?.ok) established++;
|
|
|
|
|
console.log(` ${code.padEnd(3)} ${mark.padEnd(5)} ${TITLE[code]}${s?.seconds ? ` (${s.seconds}s)` : ""}`);
|
|
|
|
|
}
|
|
|
|
|
console.log(` ${established}/${Object.keys(TITLE).length} established.`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---- the run -----------------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
before(async () => {
|
|
|
|
|
if (skip) return;
|
|
|
|
|
console.log(`\n================ THE PLAN ================`);
|
|
|
|
|
for (const [code, title] of Object.entries(TITLE)) console.log(` ${code.padEnd(3)} ${title}`);
|
|
|
|
|
|
|
|
|
|
const verdict = WARM ? await ready(SCENARIO) : { use: "raise" as const, why: "not asked to be warm" };
|
|
|
|
|
let restored = false;
|
|
|
|
|
if (verdict.use === "restore") {
|
|
|
|
|
instanceId = verdict.instanceId;
|
|
|
|
|
const seconds = await returnTo(instanceId, (m) => console.log(`warm: ${m}`));
|
|
|
|
|
console.log(`WARM: restored ${instanceId} to the adopted foundation in ${seconds}s`);
|
|
|
|
|
restored = true;
|
|
|
|
|
for (const code of ["P0", "F0", "A1", "A2", "A3"]) {
|
|
|
|
|
steps.set(code, { code, title: TITLE[code]!, ok: true, warm: true, seconds: 0, why: "",
|
|
|
|
|
said: "restored from the warm snapshot — not re-run; only a fresh run proves it" });
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (WARM) console.log(`WARM: raising — ${verdict.why}`);
|
|
|
|
|
const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
|
|
|
onProgress: (m) => console.log(`raise: ${m}`),
|
|
|
|
|
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
|
|
|
|
});
|
|
|
|
|
instanceId = bed.instanceId;
|
|
|
|
|
}
|
|
|
|
|
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
|
|
|
|
|
|
|
|
|
let heads: Record<string, string> = {};
|
|
|
|
|
const genesisOn = (node: string, o: Partial<GenesisOptions>): Promise<ReturnType<typeof genesis> extends Promise<infer R> ? R : never> =>
|
|
|
|
|
genesis({
|
|
|
|
|
instanceId, node, installer: installer as string, catalogDir,
|
|
|
|
|
bundleTemplate: foundationBundle(bundle, []),
|
|
|
|
|
registry: `${ANCHOR}:${HELD_REGISTRY}`,
|
|
|
|
|
source: forgeUrl("mesh-controller"), sourceRef: heads["mesh-controller"] ?? "",
|
|
|
|
|
toolsSource: forgeUrl("mesh-tools"), toolsRef: "lab",
|
|
|
|
|
catalogSource: forgeUrl("mesh-catalog"), catalogRef: "lab",
|
|
|
|
|
sdkSource: forgeUrl("mesh-sdk"), sdkRef: "lab",
|
|
|
|
|
site: "hosting",
|
|
|
|
|
hostService: true,
|
|
|
|
|
...(binary ? { hostBinary: binary } : {}),
|
|
|
|
|
log: (m) => console.log(m),
|
|
|
|
|
...o,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (!restored) {
|
|
|
|
|
await step("P0", [], async () => {
|
|
|
|
|
heads = await raiseForge();
|
|
|
|
|
const said = [` forge git://${FORGE}/ serving ${Object.entries(heads).map(([n, h]) => `${n}@${h.slice(0, 8)}`).join(", ")}`];
|
|
|
|
|
said.push(await preparePredecessor());
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// ADR 0101: the joiner is a freshly installed machine — nothing but its operating system, a
|
|
|
|
|
// container runtime and the host binary. A converged genesis there must not be refused. Asked
|
|
|
|
|
// as a dry run: the question is the preflight's, and a real raise would make it a second mesh.
|
|
|
|
|
await step("F0", ["P0"], async () => {
|
|
|
|
|
const ran = await genesisOn(JOINER, { flags: ["--dry-run"], attempts: 1, verify: false });
|
|
|
|
|
assert.doesNotMatch(ran.said, /this machine is in use/,
|
|
|
|
|
`a converged genesis refused a fresh machine as in use:\n${ran.said}`);
|
|
|
|
|
assert.match(ran.said, /in use\s+no: no container runs and nothing listens beyond ssh/,
|
|
|
|
|
`the installer never said the fresh machine is not in use:\n${ran.said}`);
|
|
|
|
|
const listening = (await must(JOINER, `ss -Hltunp`)).trim();
|
|
|
|
|
return ` in use no — the installer went on (${ran.ok ? "dry run finished" : `dry run stopped later, at ${ran.step}`})\n` +
|
|
|
|
|
` what listens on the fresh machine:\n${listening.split("\n").map((l) => ` ${l}`).join("\n")}`;
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("A1", ["P0"], async () => {
|
|
|
|
|
const ran = await genesisOn(CONTROL, { attempts: 1, verify: false });
|
|
|
|
|
assert.ok(!ran.ok, `a converged genesis went ahead on a machine in use:\n${ran.said}`);
|
|
|
|
|
assert.match(ran.step, /preflight/, `it was refused, but not before changing anything (at ${ran.step}):\n${ran.said}`);
|
|
|
|
|
for (const want of [/container hello-web/, /container predecessor-registry/,
|
|
|
|
|
new RegExp(`tcp \\S+:${SERVED} by`), new RegExp(`tcp \\S+:${HELD_REGISTRY} by`)]) {
|
|
|
|
|
assert.match(ran.said, want, `the refusal does not name ${want}:\n${ran.said}`);
|
|
|
|
|
}
|
|
|
|
|
assert.match(ran.said, /--adopted/, `the refusal does not say how to raise it adopted`);
|
|
|
|
|
// And nothing was changed: the predecessor as it was, no table of the mesh's.
|
|
|
|
|
const ps = await must(CONTROL, `docker ps --format '{{.Names}}'`);
|
|
|
|
|
assert.deepEqual(ps.trim().split("\n").sort(), ["hello-web", "predecessor-registry"], `containers changed:\n${ps}`);
|
|
|
|
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/);
|
|
|
|
|
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`);
|
|
|
|
|
return ran.said.split("\n").filter((l) => /in use|^\s+- /.test(l)).join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("A2", ["A1"], async () => {
|
|
|
|
|
const ran = await genesisOn(CONTROL, { adopted: true, attempts: 1, verify: false });
|
|
|
|
|
assert.ok(!ran.ok, `an adopted genesis went ahead with the registry's port held:\n${ran.said}`);
|
|
|
|
|
assert.match(ran.said, new RegExp(`registry's port tcp/${HELD_REGISTRY} is held by [^\\n]*predecessor-registry`),
|
|
|
|
|
`the refusal does not name what holds the registry's port:\n${ran.said.split("\n").slice(-15).join("\n")}`);
|
|
|
|
|
assert.match(ran.said, /--registry-port/, `the refusal does not say which flag gives another port`);
|
|
|
|
|
const id = (await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim();
|
|
|
|
|
assert.equal(id, (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the predecessor's container was replaced`);
|
|
|
|
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/);
|
|
|
|
|
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`);
|
|
|
|
|
return ` refused at ${ran.step}\n` + ran.said.split("\n").filter((l) => /held by|port|adopted/.test(l)).slice(-8).join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("A3", ["A2"], async () => {
|
|
|
|
|
const ran = await genesisOn(CONTROL, { adopted: true, registry: `${ANCHOR}:${REGISTRY_PORT}` });
|
|
|
|
|
if (!ran.ok) throw new Error(`${ran.step}: ${ran.why}\n\n${ran.report.join("\n")}`);
|
|
|
|
|
await settled();
|
|
|
|
|
const said = [ran.report.join("\n")];
|
|
|
|
|
const bindings = await must(CONTROL, `docker inspect -f '{{json .HostConfig.PortBindings}}' mesh-registry`);
|
|
|
|
|
assert.match(bindings, new RegExp(`"HostPort":"${REGISTRY_PORT}"`), `the registry is not on ${REGISTRY_PORT}: ${bindings}`);
|
|
|
|
|
assert.match(await must(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' mesh-registry`), /\S/,
|
|
|
|
|
`mesh-registry is not the host's: the foundation was not adopted as a module`);
|
|
|
|
|
assert.match(await mesh(`module list`), /^distribution\b/m, `the registry is not a module the mesh holds`);
|
|
|
|
|
// The node's own port, in what the mesh would send it — not the catalogue's default.
|
|
|
|
|
const plan = await mesh(`plan ${CONTROL} --json`);
|
|
|
|
|
assert.match(plan, new RegExp(`"${REGISTRY_PORT}:5000"`), `the registry's module does not publish ${REGISTRY_PORT}`);
|
|
|
|
|
assert.doesNotMatch(plan, /"5000:5000"/, `the plan still publishes the catalogue's 5000`);
|
|
|
|
|
said.push(` registry on ${REGISTRY_PORT}, as the module the mesh holds: ${bindings.trim()}`);
|
|
|
|
|
const show = await nodeShow(CONTROL);
|
|
|
|
|
assert.match(show, /mode\s+adopted since/, `the anchor is not reported adopted:\n${show}`);
|
|
|
|
|
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `the foundation's dropping table was loaded on an adopted node`);
|
|
|
|
|
const guard = await must(CONTROL, `nft list table inet mesh_guard`);
|
|
|
|
|
// The guard's port set is the controller's to derive; what the record fixes is that it refuses
|
|
|
|
|
// the store's port from outside and holds nothing but refusals.
|
|
|
|
|
assert.match(guard, new RegExp(`dport (\\{[^}]*\\b${STORE_PORT}\\b[^}]*\\}|${STORE_PORT}) drop`), `the guard does not refuse the store's port:\n${guard}`);
|
|
|
|
|
assert.doesNotMatch(guard, /accept\s*$/m, `the guard holds an accept:\n${guard}`);
|
|
|
|
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force`);
|
|
|
|
|
assert.match(await must(CONTROL, `curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${HELD_REGISTRY}/v2/`), /200/,
|
|
|
|
|
`the predecessor's registry stopped answering`);
|
|
|
|
|
said.push(show.trim(), guard.trim());
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (WARM && steps.get("A3")?.ok) {
|
|
|
|
|
await keep(SCENARIO, instanceId);
|
|
|
|
|
console.log(`WARM: kept ${instanceId} at the adopted foundation`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---- nothing that serves changed -------------------------------------------------------------
|
|
|
|
|
await step("B1", ["A3"], async () => {
|
|
|
|
|
const said: string[] = [];
|
|
|
|
|
const want = await must(CONTROL, `cat ${BEFORE}/file`);
|
|
|
|
|
let page = "";
|
|
|
|
|
await until(`the service answers the joiner as it did`, 120, async () => {
|
|
|
|
|
page = (await on(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`)).out;
|
|
|
|
|
return page === want ? true : null;
|
|
|
|
|
}, () => page);
|
|
|
|
|
said.push(` ${SERVICE} answers the joiner on ${SERVED} with the predecessor's page`);
|
|
|
|
|
assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(),
|
|
|
|
|
(await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `${SERVICE_FILE} changed`);
|
|
|
|
|
assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
|
|
|
|
|
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the ${SERVICE} container was replaced`);
|
|
|
|
|
said.push(` file, container byte for byte / the same id as before the mesh`);
|
|
|
|
|
const was = (await must(CONTROL, `cat ${BEFORE}/ufw-added.txt`)).split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw "));
|
|
|
|
|
const now = await ufwAdded();
|
|
|
|
|
const lost = was.filter((r) => !now.includes(r));
|
|
|
|
|
const added = now.filter((r) => !was.includes(r));
|
|
|
|
|
assert.deepEqual(lost, [], `the found firewall lost rules:\n${lost.join("\n")}`);
|
|
|
|
|
const unmarked = added.filter((r) => !marked(r));
|
|
|
|
|
assert.deepEqual(unmarked, [], `the found firewall gained rules not marked as the mesh's:\n${unmarked.join("\n")}`);
|
|
|
|
|
assert.ok(added.length > 0, `the mesh opened nothing through the found firewall`);
|
|
|
|
|
said.push(` firewall ${was.length} rule(s) kept, ${added.length} added, every one marked:`, ...added.map((r) => ` ${r}`));
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("B2", ["A3"], async () => {
|
|
|
|
|
const said: string[] = [];
|
|
|
|
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers a machine off the private network`);
|
|
|
|
|
await must(CONTROL, `ufw reload`);
|
|
|
|
|
await sleep(3_000);
|
|
|
|
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the found firewall reloaded`);
|
|
|
|
|
said.push(` outsider -> ${STORE_PORT} refused, before and after \`ufw reload\``);
|
|
|
|
|
assert.ok(await connects(OUTSIDER, ANCHOR, BUS_PORT), `the bus does not answer a machine that has not enrolled`);
|
|
|
|
|
said.push(` outsider -> ${BUS_PORT} the bus answers`);
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Its own step: it asks something different of the found firewall — a container on the machine
|
|
|
|
|
// reaches a published port through the runtime's proxy, on the incoming path, not the forwarded.
|
|
|
|
|
await step("B4", ["A3"], async () => {
|
|
|
|
|
const said: string[] = [];
|
|
|
|
|
const fromContainer = await on(CONTROL, `docker run --rm ${ALPINE} nc -z -w 3 ${ANCHOR} ${STORE_PORT}`, 180_000);
|
|
|
|
|
assert.ok(fromContainer.ok, `a container on the node cannot reach the store:\n${fromContainer.out}`);
|
|
|
|
|
said.push(` container -> ${STORE_PORT} reachable from a container on the node itself`);
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// ---- the mesh works through the found firewall -----------------------------------------------
|
|
|
|
|
let anchorOnMesh = "";
|
|
|
|
|
await step("C1", ["B2"], async () => {
|
|
|
|
|
const said: string[] = [];
|
|
|
|
|
await mesh(`node add ${JOINER}`);
|
|
|
|
|
const token = tokenFrom(await mesh(`token issue --node ${JOINER}`));
|
|
|
|
|
const out = await must(JOINER, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
|
|
|
|
|
assert.match(out, new RegExp(`enrolled as ${JOINER}`), out);
|
|
|
|
|
await must(JOINER, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
|
|
|
said.push(` ${JOINER} enrolled over the bus, through the anchor's ufw`);
|
|
|
|
|
await mesh(`overlay place ${JOINER} --site hosting`);
|
|
|
|
|
await mesh(`assign ${JOINER} ${NETWORK_MODULE}`);
|
|
|
|
|
await pushAndSettle(JOINER);
|
|
|
|
|
// The hub's peer list changed: the anchor has to be sent it too.
|
|
|
|
|
await pushAndSettle(CONTROL);
|
|
|
|
|
anchorOnMesh = await meshAddressOf(CONTROL);
|
|
|
|
|
await until(`the joiner reaches the anchor over mesh0`, 180, async () =>
|
|
|
|
|
(await on(JOINER, `ping -c1 -W2 ${anchorOnMesh}`)).ok ? true : null,
|
|
|
|
|
() => "no ping reply");
|
|
|
|
|
said.push(` private network the joiner reaches the anchor at ${anchorOnMesh}`);
|
|
|
|
|
said.push((await must(CONTROL, `wg show mesh0 latest-handshakes`)).trim());
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("B3", ["C1"], async () => {
|
|
|
|
|
assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store does not answer over the private network`);
|
|
|
|
|
return ` joiner -> ${anchorOnMesh}:${STORE_PORT} reachable over mesh0`;
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
/** The mesh's own rules in the found firewall. */
|
|
|
|
|
const openings = async (): Promise<string[]> => (await ufwAdded()).filter(marked);
|
|
|
|
|
const assertOpenings = (rules: string[]) => {
|
|
|
|
|
const text = rules.join("\n");
|
|
|
|
|
// By the opening's id, which names the machine's port: a forwarded rule names the CONTAINER's
|
|
|
|
|
// port (ufw's route rules match after the runtime's translation), so the text may say another.
|
|
|
|
|
for (const port of [BUS_PORT, REGISTRY_PORT, HUB_PORT, STORE_PORT]) {
|
|
|
|
|
assert.match(text, new RegExp(`opening-(tcp|udp)-${port}-`), `no opening for ${port} among the mesh's rules:\n${text}`);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
await step("C2", ["B3"], async () => {
|
|
|
|
|
const before = await openings();
|
|
|
|
|
assertOpenings(before);
|
|
|
|
|
await must(CONTROL, `ufw reload`);
|
|
|
|
|
await sleep(3_000);
|
|
|
|
|
const after = await openings();
|
|
|
|
|
assert.deepEqual(after.sort(), before.sort(), `the openings changed across a reload`);
|
|
|
|
|
assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store stopped answering over mesh0 after the reload`);
|
|
|
|
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reload`);
|
|
|
|
|
await pushAndSettle(JOINER);
|
|
|
|
|
return ` after ufw reload ${after.length} opening(s) in place; the joiner reaches the store over mesh0 and takes a push\n` +
|
|
|
|
|
after.map((r) => ` ${r}`).join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("C3", ["C2"], async () => {
|
|
|
|
|
const before = await openings();
|
|
|
|
|
await restartMachine(CONTROL);
|
|
|
|
|
await until(`the control plane answers after the reboot`, 300, async () =>
|
|
|
|
|
(await meshSays(`status`)).ok ? true : null, () => "no answer");
|
|
|
|
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force after the reboot`);
|
|
|
|
|
assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard did not come back`);
|
|
|
|
|
const after = await until(`the openings are there again`, 420, async () => {
|
|
|
|
|
const now = await openings();
|
|
|
|
|
return before.every((r) => now.includes(r)) ? now : null;
|
|
|
|
|
}, () => "not all openings");
|
|
|
|
|
assertOpenings(after);
|
|
|
|
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reboot`);
|
|
|
|
|
await until(`the joiner reaches the store over mesh0 again`, 300, async () =>
|
|
|
|
|
(await connects(JOINER, anchorOnMesh, STORE_PORT)) ? true : null, () => "no connection");
|
|
|
|
|
await pushAndSettle(JOINER);
|
|
|
|
|
const page = await must(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`);
|
|
|
|
|
assert.match(page, /hello from the predecessor/, `the predecessor's service did not come back: ${page}`);
|
|
|
|
|
return ` after a reboot ufw active, the guard loaded, ${after.length} opening(s); the joiner reaches the store and takes a push`;
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// ---- assigning prepares, taking cuts over ----------------------------------------------------
|
|
|
|
|
let kept = "";
|
|
|
|
|
await step("D1", ["B1"], async () => {
|
|
|
|
|
const said: string[] = [];
|
|
|
|
|
// The catalogue's module, read from the catalogue, with the route requirement taken off: the
|
|
|
|
|
// route needs a proxy module and a public name, and neither is what adoption is about.
|
|
|
|
|
const manifest = JSON.parse(catalogueModule(SERVICE, [])) as Record<string, unknown>;
|
|
|
|
|
delete manifest["requires"]; delete manifest["contributes"]; delete manifest["binds"];
|
|
|
|
|
await registerModule(SERVICE, JSON.stringify(manifest));
|
|
|
|
|
await mesh(`assign ${CONTROL} ${SERVICE}`);
|
|
|
|
|
await pushAndSettle(CONTROL);
|
|
|
|
|
const show = await until(`the anchor reports holding ${SERVICE}'s container and file`, 120, async () => {
|
|
|
|
|
const s = await nodeShow(CONTROL);
|
|
|
|
|
return /holds container\s+hello-web\b/.test(s) && /holds file\s+\/var\/lib\/hello-web\/index\.html/.test(s) ? s : null;
|
|
|
|
|
}, () => "");
|
|
|
|
|
kept = show.match(/holds file\s+\/var\/lib\/hello-web\/index\.html[^\n]*\n\s*original kept at (\S+)/)?.[1] ?? "";
|
|
|
|
|
assert.ok(kept, `the held file's original is not reported kept:\n${show}`);
|
|
|
|
|
assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(),
|
|
|
|
|
(await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `assigning changed ${SERVICE_FILE}`);
|
|
|
|
|
assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
|
|
|
|
|
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `assigning replaced the ${SERVICE} container`);
|
|
|
|
|
assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the kept original is not the file as found`);
|
|
|
|
|
said.push(show.trim());
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("D2", ["D1"], async () => {
|
|
|
|
|
await must(CONTROL, `systemctl start predecessor-sync`);
|
|
|
|
|
try {
|
|
|
|
|
await sleep(15_000);
|
|
|
|
|
await pushAndSettle(CONTROL);
|
|
|
|
|
const show = await until(`the anchor reports the held file changed`, 120, async () => {
|
|
|
|
|
const s = await nodeShow(CONTROL);
|
|
|
|
|
return /hello-web\/index\.html[^\n]*REWRITTEN/i.test(s) ? s : null;
|
|
|
|
|
}, () => "");
|
|
|
|
|
const now = await must(CONTROL, `cat ${SERVICE_FILE}`);
|
|
|
|
|
assert.match(now, /synced \d+/, `the host reverted what the predecessor wrote:\n${now}`);
|
|
|
|
|
return show.trim();
|
|
|
|
|
} finally {
|
|
|
|
|
await on(CONTROL, `systemctl stop predecessor-sync`);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("D3", ["D1"], async () => {
|
|
|
|
|
await pushAndSettle(CONTROL);
|
|
|
|
|
const r = await meshSays(`converge ${CONTROL}`);
|
|
|
|
|
assert.ok(!r.ok, `converge went ahead while ${SERVICE} holds its found container:\n${r.out}`);
|
|
|
|
|
assert.match(r.out, new RegExp(`hello-web holds the found container hello-web`), `the refusal does not name the held container:\n${r.out}`);
|
|
|
|
|
assert.match(r.out, new RegExp(`take ${CONTROL} hello-web`), `the refusal does not say what to do:\n${r.out}`);
|
|
|
|
|
return r.out.trim();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("D4", ["D1"], async () => {
|
|
|
|
|
const said: string[] = [];
|
|
|
|
|
said.push((await mesh(`take ${CONTROL} ${SERVICE}`)).trim());
|
|
|
|
|
await pushAndSettle(CONTROL);
|
|
|
|
|
const label = await until(`the ${SERVICE} container is the mesh's`, 180, async () => {
|
|
|
|
|
const l = (await on(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' ${SERVICE}`)).out.trim();
|
|
|
|
|
return l && l !== "<no value>" ? l : null;
|
|
|
|
|
}, () => "");
|
|
|
|
|
assert.notEqual((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
|
|
|
|
|
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the found container was not replaced`);
|
|
|
|
|
const catalogue = (JSON.parse(catalogueModule(SERVICE, [])) as { resources: { id: string; content?: string }[] })
|
|
|
|
|
.resources.find((r) => r.id === "page")?.content ?? "";
|
|
|
|
|
assert.equal(await must(CONTROL, `cat ${SERVICE_FILE}`), catalogue, `the found file was not replaced with the module's`);
|
|
|
|
|
assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the original was not kept`);
|
|
|
|
|
said.push(` replaced container (spec ${label.slice(0, 12)}…) and ${SERVICE_FILE}; original still at ${kept}`);
|
|
|
|
|
const opened = (await openings()).filter((r) => new RegExp(`opening-tcp-${SERVED}-`).test(r));
|
|
|
|
|
assert.ok(opened.length > 0, `no opening for ${SERVED} once ${SERVICE} was taken:\n${(await openings()).join("\n")}`);
|
|
|
|
|
said.push(...opened.map((r) => ` opened ${r}`));
|
|
|
|
|
const page = await until(`the taken ${SERVICE} answers over the private network`, 120, async () => {
|
|
|
|
|
const p = await on(JOINER, `curl -s --max-time 3 http://${anchorOnMesh}:${SERVED}/`);
|
|
|
|
|
return p.ok && p.out === catalogue ? p.out : null;
|
|
|
|
|
}, () => "");
|
|
|
|
|
said.push(` joiner -> ${SERVED} ${page.trim()}`);
|
|
|
|
|
const show = await nodeShow(CONTROL);
|
|
|
|
|
assert.doesNotMatch(show, /holds (container|file)\s+\S*hello-web/, `the anchor still holds what was taken:\n${show}`);
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// ---- converging previews, then changes -------------------------------------------------------
|
|
|
|
|
await step("E1", ["D4"], async () => {
|
|
|
|
|
if (!/^nftables\b/m.test(await mesh(`module list`))) {
|
|
|
|
|
await registerModule(FILTER_MODULE, JSON.stringify(JSON.parse(catalogueModule(FILTER_MODULE, []))));
|
|
|
|
|
}
|
|
|
|
|
// What the flip will close must be reachable now, or its closing proves nothing.
|
|
|
|
|
assert.ok(await connects(JOINER, anchorOnMesh, HELD_REGISTRY),
|
|
|
|
|
`the predecessor's published ${HELD_REGISTRY} is not reachable over the private network before the flip`);
|
|
|
|
|
await pushAndSettle(CONTROL);
|
|
|
|
|
const preview = await mesh(`converge ${CONTROL}`);
|
|
|
|
|
assert.match(preview, new RegExp(`tcp/${SERVED}\\b[^\\n]*declared by hello-web`), `the preview does not name the service's port as declared:\n${preview}`);
|
|
|
|
|
assert.match(preview, new RegExp(`tcp/${HELD_REGISTRY}\\b[^\\n]*published[^\\n]*WILL CLOSE`), `the preview does not name the published ${HELD_REGISTRY} as closing:\n${preview}`);
|
|
|
|
|
assert.match(preview, /the flip takes:\n(\s{4}\S+\n?)+/, `the preview names no module the flip takes:\n${preview}`);
|
|
|
|
|
assert.match(preview, new RegExp(`\\n\\s{4}${NETWORK_MODULE}\\b`), `the preview does not say the flip takes ${NETWORK_MODULE}:\n${preview}`);
|
|
|
|
|
assert.match(preview, /Nothing has changed/, preview);
|
|
|
|
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `previewing changed the firewall`);
|
|
|
|
|
return preview.trim();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("E2", ["E1"], async () => {
|
|
|
|
|
const said: string[] = [];
|
|
|
|
|
// The flip as the preview says to make it — whatever the preview binds `--yes` to.
|
|
|
|
|
const preview = await mesh(`converge ${CONTROL}`);
|
|
|
|
|
const flip = preview.match(new RegExp(`\`(converge ${CONTROL} --yes[^\`]*)\``))?.[1];
|
|
|
|
|
assert.ok(flip, `the preview does not say how to make the flip:\n${preview}`);
|
|
|
|
|
said.push((await mesh(flip, 300_000)).trim().split("\n").slice(-3).join("\n"));
|
|
|
|
|
await settled();
|
|
|
|
|
const table = await until(`the derived filter is loaded`, 300, async () => {
|
|
|
|
|
const t = await on(CONTROL, `nft list table inet mesh`);
|
|
|
|
|
return t.ok && /policy drop/.test(t.out) ? t.out : null;
|
|
|
|
|
}, () => "");
|
|
|
|
|
const status = await until(`the found firewall is disabled`, 180, async () => {
|
|
|
|
|
const s = (await on(CONTROL, `ufw status`)).out;
|
|
|
|
|
return /Status: inactive/.test(s) ? s : null;
|
|
|
|
|
}, () => "");
|
|
|
|
|
assert.ok((await on(CONTROL, `test -s /etc/ufw/user.rules && grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules`)).ok,
|
|
|
|
|
`the found firewall's configuration is not on disk any more`);
|
|
|
|
|
assert.match(await nodeShow(CONTROL), /mode\s+converged/, `the anchor is not reported converged`);
|
|
|
|
|
said.push(` ufw ${status.trim()} — /etc/ufw/user.rules and after.rules still on disk`);
|
|
|
|
|
await until(`the declared ${SERVED} answers over the private network`, 120, async () =>
|
|
|
|
|
(await connects(JOINER, anchorOnMesh, SERVED)) ? true : null, () => "");
|
|
|
|
|
assert.ok(!(await connects(JOINER, anchorOnMesh, HELD_REGISTRY)), `the undeclared ${HELD_REGISTRY} is still open`);
|
|
|
|
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once converged`);
|
|
|
|
|
said.push(` ports ${SERVED} open over the private network; ${HELD_REGISTRY} closed; the store still closed from outside`);
|
|
|
|
|
said.push(table.split("\n").slice(0, 30).join("\n"));
|
|
|
|
|
return said.join("\n");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await step("E3", ["E2"], async () => {
|
|
|
|
|
const said = (await mesh(`adopt ${CONTROL}`, 300_000)).trim();
|
|
|
|
|
await settled();
|
|
|
|
|
await until(`the found firewall is enabled again`, 180, async () =>
|
|
|
|
|
/Status: active/.test((await on(CONTROL, `ufw status`)).out) ? true : null, () => "");
|
|
|
|
|
await until(`the derived filter is gone`, 180, async () =>
|
|
|
|
|
!(await on(CONTROL, `nft list table inet mesh`)).ok ? true : null, () => "");
|
|
|
|
|
assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard is not restored`);
|
|
|
|
|
assertOpenings(await until(`the openings are back`, 180, async () => {
|
|
|
|
|
const o = await openings();
|
|
|
|
|
return o.length ? o : null;
|
|
|
|
|
}, () => ""));
|
|
|
|
|
assert.match(await nodeShow(CONTROL), /mode\s+adopted since/, `the anchor is not reported adopted`);
|
|
|
|
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once adopted again`);
|
|
|
|
|
return `${said}\n ufw active, table inet mesh gone, the guard and the openings back`;
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
stateOutcome();
|
|
|
|
|
}, { timeout: 10_800_000 });
|
|
|
|
|
|
|
|
|
|
after(async () => {
|
|
|
|
|
if (KEEP || WARM) {
|
|
|
|
|
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (${KEEP ? "MESH_LAB_KEEP" : "MESH_LAB_WARM"}).`);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (instanceId) await destroy(instanceId);
|
|
|
|
|
await destroyAll(`${SCENARIO}-`);
|
|
|
|
|
}, { timeout: 900_000 });
|
|
|
|
|
|
|
|
|
|
for (const [code, title] of Object.entries(TITLE)) {
|
|
|
|
|
test(`${code} ${title}`, { skip, timeout: 60_000 }, () => {
|
|
|
|
|
const s = steps.get(code);
|
|
|
|
|
assert.ok(s?.ok, s ? `${s.why}` : `${code} never ran`);
|
|
|
|
|
});
|
|
|
|
|
}
|
