diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index 195f4a4..2e34d6a 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -28,6 +28,9 @@ images: # what the mesh's registry is built from — the same chicken-and-egg the bootstrap has, resolved # the same way. - registry:2 + # And the builder, because it is a module the mesh assigns rather than a program somebody + # starts by hand — which is the only way its credential can be one the mesh delivered. + - mesh-builder:development place: all: [host, runtime] diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index e7d143a..dc88160 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -505,7 +505,8 @@ test("a machine serves its internal name with a certificate the mesh issued", { const shook = await on("laptop", `echo | openssl s_client -connect anchor.internal:8443 ` + `-CAfile /etc/mesh/authority.crt -verify_return_error -brief 2>&1`); - assert.ok(shook.ok, `the handshake failed:\n${shook.out}`); + assert.ok(shook.ok, `the handshake failed:\n${shook.out}\n` + + `what the server said:\n${(await on("anchor", `cat /var/log/tls.log`)).out}`); assert.match(shook.out, /Verification: OK/, shook.out); }); @@ -519,20 +520,35 @@ test("a machine filters exactly what its modules declared, and nothing else", { // Note what the module cannot contain: an action. The link may not carry one (novox/hq ADR 0005), // so the mesh writes the rule set and declares that a service must reflect it. `restart-on` is // the shape that rule leaves, and this is the first thing to use it for its real purpose. - await must("laptop", `nohup sh -c 'while true; do python3 -c "` + - `import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9101));` + - `s.listen(1);c,_=s.accept();c.send(b\"declared\");c.close()"; done' ` + - `> /var/log/declared.log 2>&1 & sleep 2`); - await must("laptop", `nohup sh -c 'while true; do python3 -c "` + - `import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9102));` + - `s.listen(1);c,_=s.accept();c.send(b\"undeclared\");c.close()"; done' ` + - `> /var/log/undeclared.log 2>&1 & sleep 2`); + // A listener is written to a file rather than squeezed through three levels of shell quoting. + // The first attempt did the latter, never started, and the test failed on its own setup — + // which reads exactly like the firewall working. + await must("laptop", `cat > /root/listen.py <<'LISTENER'\n` + + `import socket, sys, threading\n` + + `def serve(port):\n` + + ` s = socket.socket()\n` + + ` s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n` + + ` s.bind(("0.0.0.0", port))\n` + + ` s.listen(8)\n` + + ` while True:\n` + + ` c, _ = s.accept()\n` + + ` c.send(str(port).encode())\n` + + ` c.close()\n` + + `for p in (9101, 9102):\n` + + ` threading.Thread(target=serve, args=(p,), daemon=True).start()\n` + + `threading.Event().wait()\n` + + `LISTENER`); + await must("laptop", `nohup python3 /root/listen.py > /var/log/listen.log 2>&1 & sleep 2`); + + const reach = async (port: number) => { + const said = await on("anchor", + `timeout 5 python3 -c "import socket;s=socket.create_connection(('192.0.2.20',${port}),4);` + + `print(s.recv(32).decode());s.close()"`); + return said.ok; + }; // Reachable before any rule set exists, so what changes afterwards is the rule set and not the // listener. Without this the test would pass against a service that never started. - const reach = async (port: number) => - (await on("anchor", `timeout 5 python3 -c "` + - `import socket;s=socket.create_connection((\"192.0.2.20\",${port}),4);print(s.recv(32));s.close()"`)).ok; assert.ok(await reach(9101), "the declared port never opened, so nothing below tests anything"); assert.ok(await reach(9102), "the undeclared port never opened"); @@ -542,6 +558,7 @@ test("a machine filters exactly what its modules declared, and nothing else", { // The rule set goes where this machine's nftables unit reads from, and the unit is declared to // reflect it. No command anywhere. await must("anchor", `printf %s '{"module":"firewall","version":"1",` + + `"capabilities":["firewall"],` + `"filtering":{"into":"/etc/nftables.conf"},` + `"resources":[{"id":"nftables","type":"package","package":"nftables"},` + `{"id":"filter","type":"service","unit":"nftables.service","state":"running",` + @@ -585,3 +602,85 @@ test("a machine filters exactly what its modules declared, and nothing else", { assert.ok(!(await reach(9101)), "the port stayed open after the module that wanted it was removed"); }); + +test("the builder is a module the mesh assigns, with a credential the mesh delivered", { + skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false), + timeout: 900_000, +}, async () => { + // Until this, the builder was a program somebody started on a machine with whatever credential + // they had to hand — in practice the broker's administrative one. A program documented as + // holding its own credential and given somebody else's is worse than one with no story at all. + // + // So: the mesh issues a scoped account, seals it to the machine, and delivers it with the + // declaration. Nobody types it and the mesh cannot read it back. + await must("anchor", `mkdir -p /root/builder && printf %s '{"module":"builder","version":"1",` + + `"requires":["artifact-store"],"capabilities":["container-runtime"],` + + `"claims":[{"name":"the-build-machine","scope":"node"}],` + + `"binds":{"artifact-store":"/var/lib/mesh/builder/artifact-store.json"},` + + `"needs":{"broker":"/var/lib/mesh/builder/broker"},` + + `"build":{"artifacts":[{"name":"builder","kind":"upstream",` + + `"from":"${pinned("mesh-builder")}"}]},` + + `"resources":[` + + `{"id":"state","type":"directory","path":"/var/lib/mesh/builder","mode":"0700"},` + + `{"id":"workspace","type":"directory","path":"/var/lib/mesh/builder/workspace","mode":"0700"},` + + `{"id":"run","type":"container","name":"mesh-builder","artifact":"builder",` + + `"network":"host",` + + `"volumes":["/var/lib/mesh/builder:/var/lib/mesh/builder",` + + `"/var/run/docker.sock:/var/run/docker.sock"],` + + `"env":{"MESH_BROKER_FILE":"/var/lib/mesh/builder/broker",` + + `"MESH_BINDING":"/var/lib/mesh/builder/artifact-store.json",` + + `"MESH_WORKSPACE":"/var/lib/mesh/builder/workspace"}}]}' > /root/builder/module.json`); + await must("anchor", `cd /root/builder && git init -q . && git add -A && ` + + `git -c user.email=lab -c user.name=lab commit -qm builder`); + + // The builder's own image is built by the builder that is already running — the same + // chicken-and-egg as the registry, resolved the same way. The one started by hand does this + // last piece of work and is then replaced by the module it just built. + await mesh("build /root/builder --wait 300s", 420_000); + + // The mesh makes the account and seals the URL to this machine. Nothing is printed that would + // work if it were pasted somewhere else. + const issued = await mesh("builder issue lab-builder --node anchor"); + assert.match(issued, /sealed to anchor/, issued); + assert.doesNotMatch(issued, /amqps:\/\/lab-builder:/, + "the credential was printed, so the one copy that matters is on a terminal"); + + // Now the hand-started one goes, or two builders race for the same queue and whichever answers + // proves nothing. By process name: `pkill -f` matches the shell running it too, which kills the + // connection carrying the command and hangs the caller waiting for a reply that will never + // come. Cost an hour once, in this file. + await on("anchor", `pkill -x mesh-builder`); + await new Promise((r) => setTimeout(r, 2000)); + assert.ok(!(await on("anchor", `pgrep -x mesh-builder`)).ok, + "the hand-started builder is still running, so this would test that one"); + + await mesh("assign anchor builder"); + await mesh("push anchor"); + await new Promise((r) => setTimeout(r, 20_000)); + + const running = await must("anchor", `docker ps --format '{{.Names}}'`); + assert.match(running, /mesh-builder/, + `the builder was assigned and is not running:\n${running}\n` + + `${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`); + + // The credential arrived, is readable only by the machine, and is the scoped account rather + // than the broker's own. + assert.match(await must("anchor", `stat -c %a /var/lib/mesh/builder/broker`), /^600/); + const credential = await must("anchor", `cat /var/lib/mesh/builder/broker`); + assert.match(credential, /^amqps:\/\/lab-builder:/, + "the builder is using an account that is not its own"); + assert.doesNotMatch(credential, /guest:guest/, "the builder holds the broker's own account"); + + // And it works: the mesh asks this builder to build something, and it does. Answering is the + // only proof that the delivered credential authenticates — a container that is up with a + // credential it cannot use looks identical from outside. + await must("anchor", `mkdir -p /root/built && printf %s '{"module":"built","version":"1",` + + `"resources":[{"id":"marker","type":"file","path":"/etc/built","content":"yes","mode":"0644"}]}' ` + + `> /root/built/module.json`); + await must("anchor", `cd /root/built && git init -q . && git add -A && ` + + `git -c user.email=lab -c user.name=lab commit -qm built`); + await mesh("build /root/built --wait 300s", 420_000); + + assert.match(await mesh("builds"), /built/, + "the build was accepted and no build was recorded against the module"); +});