Build the builder before replacing the hand-started one, and listen from a file
Two setup faults, each of which looked like the thing being tested failing. The builder module was assigned without its artifact ever being built, so nothing could start — and the build has to happen while the hand-started builder is still alive. Same chicken-and-egg as the registry, resolved the same way: the builder that exists builds the one that replaces it. The firewall test's listeners were squeezed through three levels of shell quoting and never started, so the test failed on its own setup — which reads exactly like the firewall working.
This commit is contained in:
+111
-12
@@ -505,7 +505,8 @@ test("a machine serves its internal name with a certificate the mesh issued", {
|
||||
const shook = await on("laptop",
|
||||
`echo | openssl s_client -connect anchor.internal:8443 ` +
|
||||
`-CAfile /etc/mesh/authority.crt -verify_return_error -brief 2>&1`);
|
||||
assert.ok(shook.ok, `the handshake failed:\n${shook.out}`);
|
||||
assert.ok(shook.ok, `the handshake failed:\n${shook.out}\n` +
|
||||
`what the server said:\n${(await on("anchor", `cat /var/log/tls.log`)).out}`);
|
||||
assert.match(shook.out, /Verification: OK/, shook.out);
|
||||
});
|
||||
|
||||
@@ -519,20 +520,35 @@ test("a machine filters exactly what its modules declared, and nothing else", {
|
||||
// Note what the module cannot contain: an action. The link may not carry one (novox/hq ADR 0005),
|
||||
// so the mesh writes the rule set and declares that a service must reflect it. `restart-on` is
|
||||
// the shape that rule leaves, and this is the first thing to use it for its real purpose.
|
||||
await must("laptop", `nohup sh -c 'while true; do python3 -c "` +
|
||||
`import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9101));` +
|
||||
`s.listen(1);c,_=s.accept();c.send(b\"declared\");c.close()"; done' ` +
|
||||
`> /var/log/declared.log 2>&1 & sleep 2`);
|
||||
await must("laptop", `nohup sh -c 'while true; do python3 -c "` +
|
||||
`import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9102));` +
|
||||
`s.listen(1);c,_=s.accept();c.send(b\"undeclared\");c.close()"; done' ` +
|
||||
`> /var/log/undeclared.log 2>&1 & sleep 2`);
|
||||
// A listener is written to a file rather than squeezed through three levels of shell quoting.
|
||||
// The first attempt did the latter, never started, and the test failed on its own setup —
|
||||
// which reads exactly like the firewall working.
|
||||
await must("laptop", `cat > /root/listen.py <<'LISTENER'\n` +
|
||||
`import socket, sys, threading\n` +
|
||||
`def serve(port):\n` +
|
||||
` s = socket.socket()\n` +
|
||||
` s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n` +
|
||||
` s.bind(("0.0.0.0", port))\n` +
|
||||
` s.listen(8)\n` +
|
||||
` while True:\n` +
|
||||
` c, _ = s.accept()\n` +
|
||||
` c.send(str(port).encode())\n` +
|
||||
` c.close()\n` +
|
||||
`for p in (9101, 9102):\n` +
|
||||
` threading.Thread(target=serve, args=(p,), daemon=True).start()\n` +
|
||||
`threading.Event().wait()\n` +
|
||||
`LISTENER`);
|
||||
await must("laptop", `nohup python3 /root/listen.py > /var/log/listen.log 2>&1 & sleep 2`);
|
||||
|
||||
const reach = async (port: number) => {
|
||||
const said = await on("anchor",
|
||||
`timeout 5 python3 -c "import socket;s=socket.create_connection(('192.0.2.20',${port}),4);` +
|
||||
`print(s.recv(32).decode());s.close()"`);
|
||||
return said.ok;
|
||||
};
|
||||
|
||||
// Reachable before any rule set exists, so what changes afterwards is the rule set and not the
|
||||
// listener. Without this the test would pass against a service that never started.
|
||||
const reach = async (port: number) =>
|
||||
(await on("anchor", `timeout 5 python3 -c "` +
|
||||
`import socket;s=socket.create_connection((\"192.0.2.20\",${port}),4);print(s.recv(32));s.close()"`)).ok;
|
||||
assert.ok(await reach(9101), "the declared port never opened, so nothing below tests anything");
|
||||
assert.ok(await reach(9102), "the undeclared port never opened");
|
||||
|
||||
@@ -542,6 +558,7 @@ test("a machine filters exactly what its modules declared, and nothing else", {
|
||||
// The rule set goes where this machine's nftables unit reads from, and the unit is declared to
|
||||
// reflect it. No command anywhere.
|
||||
await must("anchor", `printf %s '{"module":"firewall","version":"1",` +
|
||||
`"capabilities":["firewall"],` +
|
||||
`"filtering":{"into":"/etc/nftables.conf"},` +
|
||||
`"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
|
||||
`{"id":"filter","type":"service","unit":"nftables.service","state":"running",` +
|
||||
@@ -585,3 +602,85 @@ test("a machine filters exactly what its modules declared, and nothing else", {
|
||||
assert.ok(!(await reach(9101)),
|
||||
"the port stayed open after the module that wanted it was removed");
|
||||
});
|
||||
|
||||
test("the builder is a module the mesh assigns, with a credential the mesh delivered", {
|
||||
skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
|
||||
timeout: 900_000,
|
||||
}, async () => {
|
||||
// Until this, the builder was a program somebody started on a machine with whatever credential
|
||||
// they had to hand — in practice the broker's administrative one. A program documented as
|
||||
// holding its own credential and given somebody else's is worse than one with no story at all.
|
||||
//
|
||||
// So: the mesh issues a scoped account, seals it to the machine, and delivers it with the
|
||||
// declaration. Nobody types it and the mesh cannot read it back.
|
||||
await must("anchor", `mkdir -p /root/builder && printf %s '{"module":"builder","version":"1",` +
|
||||
`"requires":["artifact-store"],"capabilities":["container-runtime"],` +
|
||||
`"claims":[{"name":"the-build-machine","scope":"node"}],` +
|
||||
`"binds":{"artifact-store":"/var/lib/mesh/builder/artifact-store.json"},` +
|
||||
`"needs":{"broker":"/var/lib/mesh/builder/broker"},` +
|
||||
`"build":{"artifacts":[{"name":"builder","kind":"upstream",` +
|
||||
`"from":"${pinned("mesh-builder")}"}]},` +
|
||||
`"resources":[` +
|
||||
`{"id":"state","type":"directory","path":"/var/lib/mesh/builder","mode":"0700"},` +
|
||||
`{"id":"workspace","type":"directory","path":"/var/lib/mesh/builder/workspace","mode":"0700"},` +
|
||||
`{"id":"run","type":"container","name":"mesh-builder","artifact":"builder",` +
|
||||
`"network":"host",` +
|
||||
`"volumes":["/var/lib/mesh/builder:/var/lib/mesh/builder",` +
|
||||
`"/var/run/docker.sock:/var/run/docker.sock"],` +
|
||||
`"env":{"MESH_BROKER_FILE":"/var/lib/mesh/builder/broker",` +
|
||||
`"MESH_BINDING":"/var/lib/mesh/builder/artifact-store.json",` +
|
||||
`"MESH_WORKSPACE":"/var/lib/mesh/builder/workspace"}}]}' > /root/builder/module.json`);
|
||||
await must("anchor", `cd /root/builder && git init -q . && git add -A && ` +
|
||||
`git -c user.email=lab -c user.name=lab commit -qm builder`);
|
||||
|
||||
// The builder's own image is built by the builder that is already running — the same
|
||||
// chicken-and-egg as the registry, resolved the same way. The one started by hand does this
|
||||
// last piece of work and is then replaced by the module it just built.
|
||||
await mesh("build /root/builder --wait 300s", 420_000);
|
||||
|
||||
// The mesh makes the account and seals the URL to this machine. Nothing is printed that would
|
||||
// work if it were pasted somewhere else.
|
||||
const issued = await mesh("builder issue lab-builder --node anchor");
|
||||
assert.match(issued, /sealed to anchor/, issued);
|
||||
assert.doesNotMatch(issued, /amqps:\/\/lab-builder:/,
|
||||
"the credential was printed, so the one copy that matters is on a terminal");
|
||||
|
||||
// Now the hand-started one goes, or two builders race for the same queue and whichever answers
|
||||
// proves nothing. By process name: `pkill -f` matches the shell running it too, which kills the
|
||||
// connection carrying the command and hangs the caller waiting for a reply that will never
|
||||
// come. Cost an hour once, in this file.
|
||||
await on("anchor", `pkill -x mesh-builder`);
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
assert.ok(!(await on("anchor", `pgrep -x mesh-builder`)).ok,
|
||||
"the hand-started builder is still running, so this would test that one");
|
||||
|
||||
await mesh("assign anchor builder");
|
||||
await mesh("push anchor");
|
||||
await new Promise((r) => setTimeout(r, 20_000));
|
||||
|
||||
const running = await must("anchor", `docker ps --format '{{.Names}}'`);
|
||||
assert.match(running, /mesh-builder/,
|
||||
`the builder was assigned and is not running:\n${running}\n` +
|
||||
`${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
|
||||
|
||||
// The credential arrived, is readable only by the machine, and is the scoped account rather
|
||||
// than the broker's own.
|
||||
assert.match(await must("anchor", `stat -c %a /var/lib/mesh/builder/broker`), /^600/);
|
||||
const credential = await must("anchor", `cat /var/lib/mesh/builder/broker`);
|
||||
assert.match(credential, /^amqps:\/\/lab-builder:/,
|
||||
"the builder is using an account that is not its own");
|
||||
assert.doesNotMatch(credential, /guest:guest/, "the builder holds the broker's own account");
|
||||
|
||||
// And it works: the mesh asks this builder to build something, and it does. Answering is the
|
||||
// only proof that the delivered credential authenticates — a container that is up with a
|
||||
// credential it cannot use looks identical from outside.
|
||||
await must("anchor", `mkdir -p /root/built && printf %s '{"module":"built","version":"1",` +
|
||||
`"resources":[{"id":"marker","type":"file","path":"/etc/built","content":"yes","mode":"0644"}]}' ` +
|
||||
`> /root/built/module.json`);
|
||||
await must("anchor", `cd /root/built && git init -q . && git add -A && ` +
|
||||
`git -c user.email=lab -c user.name=lab commit -qm built`);
|
||||
await mesh("build /root/built --wait 300s", 420_000);
|
||||
|
||||
assert.match(await mesh("builds"), /built/,
|
||||
"the build was accepted and no build was recorded against the module");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user