diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index bf2414f..195f4a4 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -24,6 +24,10 @@ images: - postgres:17-alpine - cloudamqp/lavinmq:latest - mesh-control:development + # So a module can mirror one into a registry of the mesh's own. The scenario's registry serves + # what the mesh's registry is built from — the same chicken-and-egg the bootstrap has, resolved + # the same way. + - registry:2 place: all: [host, runtime] diff --git a/src/lifecycle/base.ts b/src/lifecycle/base.ts index 4a3f88d..fd7833f 100644 --- a/src/lifecycle/base.ts +++ b/src/lifecycle/base.ts @@ -71,6 +71,13 @@ export async function buildBaseImage( log(" installing git, so a machine can build modules"); await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "git"], 600_000); + // And nftables, because the mesh computes a machine's filtering and delivers it as a file + // that a service reflects — and neither the file nor the service can install what loads it. + // Installed and NOT enabled: whether a machine filters is the mesh's decision, and a lab that + // turned it on itself would be testing its own setup. + log(" installing nftables, so a machine can enforce what the mesh computed"); + await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "nftables"], 600_000); + // Trust the documentation ranges as plain-HTTP registries. // // A scenario's registry is scenery inside the scenario, serving over HTTP, and a runtime @@ -108,6 +115,18 @@ export async function buildBaseImage( } log(` ${git.trim()}`); + // The same again, for nftables. A machine that cannot load a rule set applies the mesh's + // filtering, reports success, and filters nothing — which is precisely the fault the whole + // derivation exists to remove, reappearing in the lab. + const nft = await incusOk(["exec", BUILDER, "--", "nft", "--version"], 60_000); + if (!nft?.trim()) { + throw new BaseImageError( + `nftables was installed in ${BUILDER} and \`nft\` does not answer. Publishing this would ` + + `give every scenario a machine that cannot enforce what the mesh computed for it.`, + ); + } + log(` ${nft.trim()}`); + // Read back from the runtime, not from the package manager. An installed package is not a // capability (novox/hq 04-ISSUES/007), and this is the one place to catch that — after // publishing, every scenario pays for it instead. diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index eb62edb..e7d143a 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -44,6 +44,22 @@ const SCENARIO = "two-nodes"; let instanceId = ""; /** The scenario's own registry, which serves the images a module may mirror. */ let registry = ""; +/** What that registry actually serves, by repository. */ +let stocked: string[] = []; + +/** + * The pinned reference for one of the scenario's images. + * + * By digest, because the lab's registry drops tags when it stocks: `registry:2` is not there and + * asking for it fails with "not found", which reads like a missing image rather than a naming + * convention. A digest is also what a declaration pins, so this is the reference a module would + * really carry. + */ +function pinned(repository: string): string { + const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); + assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); + return found; +} function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -104,6 +120,7 @@ before(async () => { // because the digests are this registry's and are not known until it is up. await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`); + stocked = raised.images; const first = raised.images[0]; assert.ok(first, "the scenario stocked no images, so nothing can be mirrored"); registry = first.slice(0, first.indexOf("/")); @@ -407,12 +424,164 @@ test("a machine that fell behind catches up without being named", { skip, timeou assert.match(await mesh("push --behind"), /every machine is doing what it was told/); }); -// The mesh running its own artifact store is proven in its own scenario, not this one. -// -// It was here, and adding the image it mirrors to this scenario made the bootstrap fail: the -// store container did not come up within three minutes, with no output at all from its own -// readiness check — which says the container was not running rather than that the database was -// slow. Four images on a machine this size is the difference. -// -// Left as a note rather than a silently deleted test: what it asserted is worth asserting, and -// where it belongs is a scenario with room for it (novox/hq 04-ISSUES/012). +test("the mesh runs its own artifact store", { + skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false), + timeout: 900_000, +}, async () => { + // Artifacts go to a registry, and the only registries that existed were raised by the lab or by + // the bootstrap bundle. A mesh had no way to run its own. + // + // Chicken and egg, resolved the way the bootstrap's is: the scenario's registry serves the image + // the module mirrors, and the module then runs a registry of the mesh's own. + await must("anchor", `mkdir -p /root/registry && printf %s '{"module":"registry","version":"1",` + + `"provides":[{"name":"artifact-store","scope":"mesh"}],` + + `"capabilities":["container-runtime"],` + + `"claims":[{"name":"the-artifact-store","scope":"node"}],` + + `"serves":{"artifact-store":{"port":5000}},` + + `"build":{"artifacts":[{"name":"registry","kind":"upstream","from":"${pinned("registry")}"}]},` + + `"resources":[` + + `{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` + + `{"id":"store","type":"container","name":"mesh-registry","artifact":"registry",` + + `"ports":["5000:5000"],"volumes":["mesh-registry-data:/var/lib/registry"]}]}' ` + + `> /root/registry/module.json`); + await must("anchor", `cd /root/registry && git init -q . && git add -A && ` + + `git -c user.email=lab -c user.name=lab commit -qm registry`); + + await mesh("build /root/registry --wait 300s", 420_000); + await mesh("assign anchor registry"); + await mesh("push anchor"); + await new Promise((r) => setTimeout(r, 12_000)); + + // Running, and answering — a container that is up is not a registry that replies. + assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/); + let answers = false; + for (let i = 0; i < 20 && !answers; i++) { + answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok; + if (!answers) await new Promise((r) => setTimeout(r, 2000)); + } + assert.ok(answers, "the mesh's own registry is running and does not answer"); + + // And reachable from another machine over the private network, which is the whole point of an + // artifact store being a mesh-scoped provision. + assert.ok((await on("laptop", `curl -sf http://anchor.internal:5000/v2/ -o /dev/null`)).ok, + "the artifact store is not reachable from another machine, so nothing else can use it"); +}); + +test("a machine serves its internal name with a certificate the mesh issued", { + skip, timeout: 900_000, +}, async () => { + // The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity). + // Asserted with a real handshake: a certificate that parses and does not chain fails at the + // moment something connects, which is the worst place to find out. + await must("anchor", `printf %s '{"module":"served","version":"1",` + + `"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` + + `"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` + + `> /tmp/served.json`); + await must("anchor", `docker cp /tmp/served.json mesh-control:/served.json`); + await mesh("module add /served.json"); + await mesh("assign anchor served"); + await mesh("push anchor"); + await new Promise((r) => setTimeout(r, 8000)); + + assert.ok((await on("anchor", `test -s /etc/mesh/serving.crt`)).ok, "no certificate arrived"); + assert.ok((await on("anchor", `test -s /etc/mesh/authority.crt`)).ok, "no authority arrived"); + + // The name it was issued for is the one the mesh gave this machine. + const named = await must("anchor", + `openssl x509 -in /etc/mesh/serving.crt -noout -ext subjectAltName 2>/dev/null || ` + + `docker run --rm -v /etc/mesh:/m ${pinned("registry")} sh -c ` + + `"apk add --no-cache openssl >/dev/null 2>&1; openssl x509 -in /m/serving.crt -noout -text" | grep -A1 'Alternative'`); + assert.match(named, /anchor\.internal/, `the certificate is not for this machine's name:\n${named}`); + + // And a real handshake: the machine serves TLS with the key it generated, and another machine + // verifies it against the mesh's authority and nothing else. + await must("anchor", `openssl s_server -cert /etc/mesh/serving.crt ` + + `-key /var/lib/mesh-host/serving.key -accept 8443 -naccept 1 -quiet ` + + `> /var/log/tls.log 2>&1 & sleep 2`); + await must("laptop", `mkdir -p /etc/mesh`); + const authority = await must("anchor", `cat /etc/mesh/authority.crt`); + await must("laptop", `cat > /etc/mesh/authority.crt <<'MESHCA'\n${authority}\nMESHCA`); + + const shook = await on("laptop", + `echo | openssl s_client -connect anchor.internal:8443 ` + + `-CAfile /etc/mesh/authority.crt -verify_return_error -brief 2>&1`); + assert.ok(shook.ok, `the handshake failed:\n${shook.out}`); + assert.match(shook.out, /Verification: OK/, shook.out); +}); + +test("a machine filters exactly what its modules declared, and nothing else", { + skip, timeout: 900_000, +}, async () => { + // The rule set is derived from what is assigned, not kept in step by hand — and the proof that + // matters is not that a file arrived but that packets are treated differently because of it. + // A rule nothing enforces is the fault this mechanism exists to remove (novox/hq 04-ISSUES/003). + // + // Note what the module cannot contain: an action. The link may not carry one (novox/hq ADR 0005), + // so the mesh writes the rule set and declares that a service must reflect it. `restart-on` is + // the shape that rule leaves, and this is the first thing to use it for its real purpose. + await must("laptop", `nohup sh -c 'while true; do python3 -c "` + + `import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9101));` + + `s.listen(1);c,_=s.accept();c.send(b\"declared\");c.close()"; done' ` + + `> /var/log/declared.log 2>&1 & sleep 2`); + await must("laptop", `nohup sh -c 'while true; do python3 -c "` + + `import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9102));` + + `s.listen(1);c,_=s.accept();c.send(b\"undeclared\");c.close()"; done' ` + + `> /var/log/undeclared.log 2>&1 & sleep 2`); + + // Reachable before any rule set exists, so what changes afterwards is the rule set and not the + // listener. Without this the test would pass against a service that never started. + const reach = async (port: number) => + (await on("anchor", `timeout 5 python3 -c "` + + `import socket;s=socket.create_connection((\"192.0.2.20\",${port}),4);print(s.recv(32));s.close()"`)).ok; + assert.ok(await reach(9101), "the declared port never opened, so nothing below tests anything"); + assert.ok(await reach(9102), "the undeclared port never opened"); + + await must("anchor", `printf %s '{"module":"talker","version":"1",` + + `"listens":[{"port":9101,"from":"mesh","why":"the thing this test is about"}],` + + `"resources":[]}' > /tmp/talker.json`); + // The rule set goes where this machine's nftables unit reads from, and the unit is declared to + // reflect it. No command anywhere. + await must("anchor", `printf %s '{"module":"firewall","version":"1",` + + `"filtering":{"into":"/etc/nftables.conf"},` + + `"resources":[{"id":"nftables","type":"package","package":"nftables"},` + + `{"id":"filter","type":"service","unit":"nftables.service","state":"running",` + + `"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/firewall.json`); + for (const f of ["talker", "firewall"]) { + await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`); + await mesh(`module add /${f}.json`); + } + await mesh("assign laptop talker"); + await mesh("assign laptop firewall"); + await mesh("push laptop"); + await new Promise((r) => setTimeout(r, 20_000)); + + const written = await must("laptop", `cat /etc/nftables.conf`); + // A rule names its source. Not decoration: it is the only thing that answers "why is this open". + assert.match(written, /# talker . the thing this test is about/, + `the rule does not name what caused it:\n${written}`); + assert.match(written, /192\.0\.2\.\d+/, `"from the mesh" resolved to nothing:\n${written}`); + assert.doesNotMatch(written, /dport 9102/, `a port no module declared was opened:\n${written}`); + + // Loaded, not merely written. The service was restarted because a file it reflects changed. + const table = await must("laptop", `nft list table inet mesh`); + assert.match(table, /dport 9101 accept/, `the rule set was never loaded:\n${table}`); + + // And it filters. The declared port answers from another machine; the undeclared one does not. + assert.ok(await reach(9101), + "the declared port is closed, so the machine is filtering more than it was told to"); + assert.ok(!(await reach(9102)), + "a port no module declared is still reachable, so the rule set restricts nothing"); + + // The machine did not lock itself out of the mesh: it is still taking declarations. + assert.doesNotMatch(await mesh("status"), /laptop\s+(failed|refused)/, + "the machine stopped doing what it was told after applying its own rule set"); + + // Removing the module that wanted the port closes it, with nobody editing a rule. This is the + // whole claim of a derived firewall, and it is also the second load — which must replace the + // table rather than add to it. + await mesh("unassign laptop talker"); + await mesh("push laptop"); + await new Promise((r) => setTimeout(r, 20_000)); + assert.ok(!(await reach(9101)), + "the port stayed open after the module that wanted it was removed"); +});