novox/mesh-lab
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Prove a machine filters what it was told to and nothing else

Browse Source 
Written and loaded are different things, and loaded and enforcing are different
again. The test opens two ports on a machine, declares one of them, and checks
from the other machine that the declared one answers and the undeclared one
does not — then removes the module and checks the port closes with nobody
editing a rule.

The base image gains nftables, read back through `nft --version` like the other
three: a machine that cannot load a rule set applies the mesh's filtering,
reports success and filters nothing, which is the exact fault the derivation
exists to remove.

Two earlier tests were asking for things that are not there. The lab's registry
drops tags when it stocks, so `registry:2` is not served and the mirror test
failed with "not found" — it now uses the pinned digest, which is what a
declaration carries anyway.
This commit is contained in:
jschoubben
2026-08-31 00:25:19 +02:00
parent 2f81701a13
commit 29cdaa4de3
3 changed files with 201 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scenarios/two-nodes.yml
+4
View File
@@ -24,6 +24,10 @@ images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# So a module can mirror one into a registry of the mesh's own. The scenario's registry serves
# what the mesh's registry is built from — the same chicken-and-egg the bootstrap has, resolved
# the same way.
- registry:2
place:
all: [host, runtime]
src/lifecycle/base.ts
+19
View File
@@ -71,6 +71,13 @@ export async function buildBaseImage(
log(" installing git, so a machine can build modules");
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "git"], 600_000);
// And nftables, because the mesh computes a machine's filtering and delivers it as a file
// that a service reflects — and neither the file nor the service can install what loads it.
// Installed and NOT enabled: whether a machine filters is the mesh's decision, and a lab that
// turned it on itself would be testing its own setup.
log(" installing nftables, so a machine can enforce what the mesh computed");
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "nftables"], 600_000);
// Trust the documentation ranges as plain-HTTP registries.
//
// A scenario's registry is scenery inside the scenario, serving over HTTP, and a runtime
@@ -108,6 +115,18 @@ export async function buildBaseImage(
}
log(` ${git.trim()}`);
// The same again, for nftables. A machine that cannot load a rule set applies the mesh's
// filtering, reports success, and filters nothing — which is precisely the fault the whole
// derivation exists to remove, reappearing in the lab.
const nft = await incusOk(["exec", BUILDER, "--", "nft", "--version"], 60_000);
if (!nft?.trim()) {
throw new BaseImageError(
`nftables was installed in ${BUILDER} and \`nft\` does not answer. Publishing this would ` +
`give every scenario a machine that cannot enforce what the mesh computed for it.`,
);
}
log(` ${nft.trim()}`);
// Read back from the runtime, not from the package manager. An installed package is not a
// capability (novox/hq 04-ISSUES/007), and this is the one place to catch that — after
// publishing, every scenario pays for it instead.
test/integration/mesh.test.ts
+178 -9
View File
@@ -44,6 +44,22 @@ const SCENARIO = "two-nodes";
let instanceId = "";
/** The scenario's own registry, which serves the images a module may mirror. */
let registry = "";
/** What that registry actually serves, by repository. */
let stocked: string[] = [];
/**
* The pinned reference for one of the scenario's images.
*
* By digest, because the lab's registry drops tags when it stocks: `registry:2` is not there and
* asking for it fails with "not found", which reads like a missing image rather than a naming
* convention. A digest is also what a declaration pins, so this is the reference a module would
* really carry.
*/
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
}
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -104,6 +120,7 @@ before(async () => {
// because the digests are this registry's and are not known until it is up.
await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`);
stocked = raised.images;
const first = raised.images[0];
assert.ok(first, "the scenario stocked no images, so nothing can be mirrored");
registry = first.slice(0, first.indexOf("/"));
@@ -407,12 +424,164 @@ test("a machine that fell behind catches up without being named", { skip, timeou
assert.match(await mesh("push --behind"), /every machine is doing what it was told/);
});
// The mesh running its own artifact store is proven in its own scenario, not this one.
//
// It was here, and adding the image it mirrors to this scenario made the bootstrap fail: the
// store container did not come up within three minutes, with no output at all from its own
// readiness check — which says the container was not running rather than that the database was
// slow. Four images on a machine this size is the difference.
//
// Left as a note rather than a silently deleted test: what it asserted is worth asserting, and
// where it belongs is a scenario with room for it (novox/hq 04-ISSUES/012).
test("the mesh runs its own artifact store", {
skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
timeout: 900_000,
}, async () => {
// Artifacts go to a registry, and the only registries that existed were raised by the lab or by
// the bootstrap bundle. A mesh had no way to run its own.
//
// Chicken and egg, resolved the way the bootstrap's is: the scenario's registry serves the image
// the module mirrors, and the module then runs a registry of the mesh's own.
await must("anchor", `mkdir -p /root/registry && printf %s '{"module":"registry","version":"1",` +
`"provides":[{"name":"artifact-store","scope":"mesh"}],` +
`"capabilities":["container-runtime"],` +
`"claims":[{"name":"the-artifact-store","scope":"node"}],` +
`"serves":{"artifact-store":{"port":5000}},` +
`"build":{"artifacts":[{"name":"registry","kind":"upstream","from":"${pinned("registry")}"}]},` +
`"resources":[` +
`{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
`{"id":"store","type":"container","name":"mesh-registry","artifact":"registry",` +
`"ports":["5000:5000"],"volumes":["mesh-registry-data:/var/lib/registry"]}]}' ` +
`> /root/registry/module.json`);
await must("anchor", `cd /root/registry && git init -q . && git add -A && ` +
`git -c user.email=lab -c user.name=lab commit -qm registry`);
await mesh("build /root/registry --wait 300s", 420_000);
await mesh("assign anchor registry");
await mesh("push anchor");
await new Promise((r) => setTimeout(r, 12_000));
// Running, and answering — a container that is up is not a registry that replies.
assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/);
let answers = false;
for (let i = 0; i < 20 && !answers; i++) {
answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
if (!answers) await new Promise((r) => setTimeout(r, 2000));
}
assert.ok(answers, "the mesh's own registry is running and does not answer");
// And reachable from another machine over the private network, which is the whole point of an
// artifact store being a mesh-scoped provision.
assert.ok((await on("laptop", `curl -sf http://anchor.internal:5000/v2/ -o /dev/null`)).ok,
"the artifact store is not reachable from another machine, so nothing else can use it");
});
test("a machine serves its internal name with a certificate the mesh issued", {
skip, timeout: 900_000,
}, async () => {
// The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
// Asserted with a real handshake: a certificate that parses and does not chain fails at the
// moment something connects, which is the worst place to find out.
await must("anchor", `printf %s '{"module":"served","version":"1",` +
`"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
`"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
`> /tmp/served.json`);
await must("anchor", `docker cp /tmp/served.json mesh-control:/served.json`);
await mesh("module add /served.json");
await mesh("assign anchor served");
await mesh("push anchor");
await new Promise((r) => setTimeout(r, 8000));
assert.ok((await on("anchor", `test -s /etc/mesh/serving.crt`)).ok, "no certificate arrived");
assert.ok((await on("anchor", `test -s /etc/mesh/authority.crt`)).ok, "no authority arrived");
// The name it was issued for is the one the mesh gave this machine.
const named = await must("anchor",
`openssl x509 -in /etc/mesh/serving.crt -noout -ext subjectAltName 2>/dev/null || ` +
`docker run --rm -v /etc/mesh:/m ${pinned("registry")} sh -c ` +
`"apk add --no-cache openssl >/dev/null 2>&1; openssl x509 -in /m/serving.crt -noout -text" | grep -A1 'Alternative'`);
assert.match(named, /anchor\.internal/, `the certificate is not for this machine's name:\n${named}`);
// And a real handshake: the machine serves TLS with the key it generated, and another machine
// verifies it against the mesh's authority and nothing else.
await must("anchor", `openssl s_server -cert /etc/mesh/serving.crt ` +
`-key /var/lib/mesh-host/serving.key -accept 8443 -naccept 1 -quiet ` +
`> /var/log/tls.log 2>&1 & sleep 2`);
await must("laptop", `mkdir -p /etc/mesh`);
const authority = await must("anchor", `cat /etc/mesh/authority.crt`);
await must("laptop", `cat > /etc/mesh/authority.crt <<'MESHCA'\n${authority}\nMESHCA`);
const shook = await on("laptop",
`echo | openssl s_client -connect anchor.internal:8443 ` +
`-CAfile /etc/mesh/authority.crt -verify_return_error -brief 2>&1`);
assert.ok(shook.ok, `the handshake failed:\n${shook.out}`);
assert.match(shook.out, /Verification: OK/, shook.out);
});
test("a machine filters exactly what its modules declared, and nothing else", {
skip, timeout: 900_000,
}, async () => {
// The rule set is derived from what is assigned, not kept in step by hand — and the proof that
// matters is not that a file arrived but that packets are treated differently because of it.
// A rule nothing enforces is the fault this mechanism exists to remove (novox/hq 04-ISSUES/003).
//
// Note what the module cannot contain: an action. The link may not carry one (novox/hq ADR 0005),
// so the mesh writes the rule set and declares that a service must reflect it. `restart-on` is
// the shape that rule leaves, and this is the first thing to use it for its real purpose.
await must("laptop", `nohup sh -c 'while true; do python3 -c "` +
`import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9101));` +
`s.listen(1);c,_=s.accept();c.send(b\"declared\");c.close()"; done' ` +
`> /var/log/declared.log 2>&1 & sleep 2`);
await must("laptop", `nohup sh -c 'while true; do python3 -c "` +
`import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9102));` +
`s.listen(1);c,_=s.accept();c.send(b\"undeclared\");c.close()"; done' ` +
`> /var/log/undeclared.log 2>&1 & sleep 2`);
// Reachable before any rule set exists, so what changes afterwards is the rule set and not the
// listener. Without this the test would pass against a service that never started.
const reach = async (port: number) =>
(await on("anchor", `timeout 5 python3 -c "` +
`import socket;s=socket.create_connection((\"192.0.2.20\",${port}),4);print(s.recv(32));s.close()"`)).ok;
assert.ok(await reach(9101), "the declared port never opened, so nothing below tests anything");
assert.ok(await reach(9102), "the undeclared port never opened");
await must("anchor", `printf %s '{"module":"talker","version":"1",` +
`"listens":[{"port":9101,"from":"mesh","why":"the thing this test is about"}],` +
`"resources":[]}' > /tmp/talker.json`);
// The rule set goes where this machine's nftables unit reads from, and the unit is declared to
// reflect it. No command anywhere.
await must("anchor", `printf %s '{"module":"firewall","version":"1",` +
`"filtering":{"into":"/etc/nftables.conf"},` +
`"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
`{"id":"filter","type":"service","unit":"nftables.service","state":"running",` +
`"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/firewall.json`);
for (const f of ["talker", "firewall"]) {
await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`);
await mesh(`module add /${f}.json`);
}
await mesh("assign laptop talker");
await mesh("assign laptop firewall");
await mesh("push laptop");
await new Promise((r) => setTimeout(r, 20_000));
const written = await must("laptop", `cat /etc/nftables.conf`);
// A rule names its source. Not decoration: it is the only thing that answers "why is this open".
assert.match(written, /# talker . the thing this test is about/,
`the rule does not name what caused it:\n${written}`);
assert.match(written, /192\.0\.2\.\d+/, `"from the mesh" resolved to nothing:\n${written}`);
assert.doesNotMatch(written, /dport 9102/, `a port no module declared was opened:\n${written}`);
// Loaded, not merely written. The service was restarted because a file it reflects changed.
const table = await must("laptop", `nft list table inet mesh`);
assert.match(table, /dport 9101 accept/, `the rule set was never loaded:\n${table}`);
// And it filters. The declared port answers from another machine; the undeclared one does not.
assert.ok(await reach(9101),
"the declared port is closed, so the machine is filtering more than it was told to");
assert.ok(!(await reach(9102)),
"a port no module declared is still reachable, so the rule set restricts nothing");
// The machine did not lock itself out of the mesh: it is still taking declarations.
assert.doesNotMatch(await mesh("status"), /laptop\s+(failed|refused)/,
"the machine stopped doing what it was told after applying its own rule set");
// Removing the module that wanted the port closes it, with nobody editing a rule. This is the
// whole claim of a derived firewall, and it is also the second load — which must replace the
// table rather than add to it.
await mesh("unassign laptop talker");
await mesh("push laptop");
await new Promise((r) => setTimeout(r, 20_000));
assert.ok(!(await reach(9101)),
"the port stayed open after the module that wanted it was removed");
});