scenarios/two-nodes.yml

@@ -24,6 +24,10 @@ images:
   - postgres:17-alpine
   - cloudamqp/lavinmq:latest
   - mesh-control:development
+  # So a module can mirror one into a registry of the mesh's own. The scenario's registry serves
+  # what the mesh's registry is built from — the same chicken-and-egg the bootstrap has, resolved
+  # the same way.
+  - registry:2
 
 place:
   all: [host, runtime]

src/lifecycle/base.ts

@@ -71,6 +71,13 @@ export async function buildBaseImage(
     log("  installing git, so a machine can build modules");
     await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "git"], 600_000);
 
+    // And nftables, because the mesh computes a machine's filtering and delivers it as a file
+    // that a service reflects — and neither the file nor the service can install what loads it.
+    // Installed and NOT enabled: whether a machine filters is the mesh's decision, and a lab that
+    // turned it on itself would be testing its own setup.
+    log("  installing nftables, so a machine can enforce what the mesh computed");
+    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "nftables"], 600_000);
+
     // Trust the documentation ranges as plain-HTTP registries.
     //
     // A scenario's registry is scenery inside the scenario, serving over HTTP, and a runtime
@@ -108,6 +115,18 @@ export async function buildBaseImage(
     }
     log(`  ${git.trim()}`);
 
+    // The same again, for nftables. A machine that cannot load a rule set applies the mesh's
+    // filtering, reports success, and filters nothing — which is precisely the fault the whole
+    // derivation exists to remove, reappearing in the lab.
+    const nft = await incusOk(["exec", BUILDER, "--", "nft", "--version"], 60_000);
+    if (!nft?.trim()) {
+      throw new BaseImageError(
+        `nftables was installed in ${BUILDER} and \`nft\` does not answer. Publishing this would ` +
+          `give every scenario a machine that cannot enforce what the mesh computed for it.`,
+      );
+    }
+    log(`  ${nft.trim()}`);
+
     // Read back from the runtime, not from the package manager. An installed package is not a
     // capability (novox/hq 04-ISSUES/007), and this is the one place to catch that — after
     // publishing, every scenario pays for it instead.

test/integration/mesh.test.ts

@@ -44,6 +44,22 @@ const SCENARIO = "two-nodes";
 let instanceId = "";
 /** The scenario's own registry, which serves the images a module may mirror. */
 let registry = "";
+/** What that registry actually serves, by repository. */
+let stocked: string[] = [];
+
+/**
+ * The pinned reference for one of the scenario's images.
+ *
+ * By digest, because the lab's registry drops tags when it stocks: `registry:2` is not there and
+ * asking for it fails with "not found", which reads like a missing image rather than a naming
+ * convention. A digest is also what a declaration pins, so this is the reference a module would
+ * really carry.
+ */
+function pinned(repository: string): string {
+  const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
+  assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
+  return found;
+}
 
 function quote(s: string): string {
   return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -104,6 +120,7 @@ before(async () => {
   // because the digests are this registry's and are not known until it is up.
   await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
   await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`);
+  stocked = raised.images;
   const first = raised.images[0];
   assert.ok(first, "the scenario stocked no images, so nothing can be mirrored");
   registry = first.slice(0, first.indexOf("/"));
@@ -407,12 +424,164 @@ test("a machine that fell behind catches up without being named", { skip, timeou
   assert.match(await mesh("push --behind"), /every machine is doing what it was told/);
 });
 
-// The mesh running its own artifact store is proven in its own scenario, not this one.
-//
-// It was here, and adding the image it mirrors to this scenario made the bootstrap fail: the
-// store container did not come up within three minutes, with no output at all from its own
-// readiness check — which says the container was not running rather than that the database was
-// slow. Four images on a machine this size is the difference.
-//
-// Left as a note rather than a silently deleted test: what it asserted is worth asserting, and
-// where it belongs is a scenario with room for it (novox/hq 04-ISSUES/012).
+test("the mesh runs its own artifact store", {
+  skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
+  timeout: 900_000,
+}, async () => {
+  // Artifacts go to a registry, and the only registries that existed were raised by the lab or by
+  // the bootstrap bundle. A mesh had no way to run its own.
+  //
+  // Chicken and egg, resolved the way the bootstrap's is: the scenario's registry serves the image
+  // the module mirrors, and the module then runs a registry of the mesh's own.
+  await must("anchor", `mkdir -p /root/registry && printf %s '{"module":"registry","version":"1",` +
+    `"provides":[{"name":"artifact-store","scope":"mesh"}],` +
+    `"capabilities":["container-runtime"],` +
+    `"claims":[{"name":"the-artifact-store","scope":"node"}],` +
+    `"serves":{"artifact-store":{"port":5000}},` +
+    `"build":{"artifacts":[{"name":"registry","kind":"upstream","from":"${pinned("registry")}"}]},` +
+    `"resources":[` +
+    `{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
+    `{"id":"store","type":"container","name":"mesh-registry","artifact":"registry",` +
+    `"ports":["5000:5000"],"volumes":["mesh-registry-data:/var/lib/registry"]}]}' ` +
+    `> /root/registry/module.json`);
+  await must("anchor", `cd /root/registry && git init -q . && git add -A && ` +
+    `git -c user.email=lab -c user.name=lab commit -qm registry`);
+
+  await mesh("build /root/registry --wait 300s", 420_000);
+  await mesh("assign anchor registry");
+  await mesh("push anchor");
+  await new Promise((r) => setTimeout(r, 12_000));
+
+  // Running, and answering — a container that is up is not a registry that replies.
+  assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/);
+  let answers = false;
+  for (let i = 0; i < 20 && !answers; i++) {
+    answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
+    if (!answers) await new Promise((r) => setTimeout(r, 2000));
+  }
+  assert.ok(answers, "the mesh's own registry is running and does not answer");
+
+  // And reachable from another machine over the private network, which is the whole point of an
+  // artifact store being a mesh-scoped provision.
+  assert.ok((await on("laptop", `curl -sf http://anchor.internal:5000/v2/ -o /dev/null`)).ok,
+    "the artifact store is not reachable from another machine, so nothing else can use it");
+});
+
+test("a machine serves its internal name with a certificate the mesh issued", {
+  skip, timeout: 900_000,
+}, async () => {
+  // The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
+  // Asserted with a real handshake: a certificate that parses and does not chain fails at the
+  // moment something connects, which is the worst place to find out.
+  await must("anchor", `printf %s '{"module":"served","version":"1",` +
+    `"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
+    `"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
+    `> /tmp/served.json`);
+  await must("anchor", `docker cp /tmp/served.json mesh-control:/served.json`);
+  await mesh("module add /served.json");
+  await mesh("assign anchor served");
+  await mesh("push anchor");
+  await new Promise((r) => setTimeout(r, 8000));
+
+  assert.ok((await on("anchor", `test -s /etc/mesh/serving.crt`)).ok, "no certificate arrived");
+  assert.ok((await on("anchor", `test -s /etc/mesh/authority.crt`)).ok, "no authority arrived");
+
+  // The name it was issued for is the one the mesh gave this machine.
+  const named = await must("anchor",
+    `openssl x509 -in /etc/mesh/serving.crt -noout -ext subjectAltName 2>/dev/null || ` +
+    `docker run --rm -v /etc/mesh:/m ${pinned("registry")} sh -c ` +
+    `"apk add --no-cache openssl >/dev/null 2>&1; openssl x509 -in /m/serving.crt -noout -text" | grep -A1 'Alternative'`);
+  assert.match(named, /anchor\.internal/, `the certificate is not for this machine's name:\n${named}`);
+
+  // And a real handshake: the machine serves TLS with the key it generated, and another machine
+  // verifies it against the mesh's authority and nothing else.
+  await must("anchor", `openssl s_server -cert /etc/mesh/serving.crt ` +
+    `-key /var/lib/mesh-host/serving.key -accept 8443 -naccept 1 -quiet ` +
+    `> /var/log/tls.log 2>&1 & sleep 2`);
+  await must("laptop", `mkdir -p /etc/mesh`);
+  const authority = await must("anchor", `cat /etc/mesh/authority.crt`);
+  await must("laptop", `cat > /etc/mesh/authority.crt <<'MESHCA'\n${authority}\nMESHCA`);
+
+  const shook = await on("laptop",
+    `echo | openssl s_client -connect anchor.internal:8443 ` +
+    `-CAfile /etc/mesh/authority.crt -verify_return_error -brief 2>&1`);
+  assert.ok(shook.ok, `the handshake failed:\n${shook.out}`);
+  assert.match(shook.out, /Verification: OK/, shook.out);
+});
+
+test("a machine filters exactly what its modules declared, and nothing else", {
+  skip, timeout: 900_000,
+}, async () => {
+  // The rule set is derived from what is assigned, not kept in step by hand — and the proof that
+  // matters is not that a file arrived but that packets are treated differently because of it.
+  // A rule nothing enforces is the fault this mechanism exists to remove (novox/hq 04-ISSUES/003).
+  //
+  // Note what the module cannot contain: an action. The link may not carry one (novox/hq ADR 0005),
+  // so the mesh writes the rule set and declares that a service must reflect it. `restart-on` is
+  // the shape that rule leaves, and this is the first thing to use it for its real purpose.
+  await must("laptop", `nohup sh -c 'while true; do python3 -c "` +
+    `import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9101));` +
+    `s.listen(1);c,_=s.accept();c.send(b\"declared\");c.close()"; done' ` +
+    `> /var/log/declared.log 2>&1 & sleep 2`);
+  await must("laptop", `nohup sh -c 'while true; do python3 -c "` +
+    `import socket,sys;s=socket.socket();s.setsockopt(1,2,1);s.bind((\"0.0.0.0\",9102));` +
+    `s.listen(1);c,_=s.accept();c.send(b\"undeclared\");c.close()"; done' ` +
+    `> /var/log/undeclared.log 2>&1 & sleep 2`);
+
+  // Reachable before any rule set exists, so what changes afterwards is the rule set and not the
+  // listener. Without this the test would pass against a service that never started.
+  const reach = async (port: number) =>
+    (await on("anchor", `timeout 5 python3 -c "` +
+      `import socket;s=socket.create_connection((\"192.0.2.20\",${port}),4);print(s.recv(32));s.close()"`)).ok;
+  assert.ok(await reach(9101), "the declared port never opened, so nothing below tests anything");
+  assert.ok(await reach(9102), "the undeclared port never opened");
+
+  await must("anchor", `printf %s '{"module":"talker","version":"1",` +
+    `"listens":[{"port":9101,"from":"mesh","why":"the thing this test is about"}],` +
+    `"resources":[]}' > /tmp/talker.json`);
+  // The rule set goes where this machine's nftables unit reads from, and the unit is declared to
+  // reflect it. No command anywhere.
+  await must("anchor", `printf %s '{"module":"firewall","version":"1",` +
+    `"filtering":{"into":"/etc/nftables.conf"},` +
+    `"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
+    `{"id":"filter","type":"service","unit":"nftables.service","state":"running",` +
+    `"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/firewall.json`);
+  for (const f of ["talker", "firewall"]) {
+    await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`);
+    await mesh(`module add /${f}.json`);
+  }
+  await mesh("assign laptop talker");
+  await mesh("assign laptop firewall");
+  await mesh("push laptop");
+  await new Promise((r) => setTimeout(r, 20_000));
+
+  const written = await must("laptop", `cat /etc/nftables.conf`);
+  // A rule names its source. Not decoration: it is the only thing that answers "why is this open".
+  assert.match(written, /# talker . the thing this test is about/,
+    `the rule does not name what caused it:\n${written}`);
+  assert.match(written, /192\.0\.2\.\d+/, `"from the mesh" resolved to nothing:\n${written}`);
+  assert.doesNotMatch(written, /dport 9102/, `a port no module declared was opened:\n${written}`);
+
+  // Loaded, not merely written. The service was restarted because a file it reflects changed.
+  const table = await must("laptop", `nft list table inet mesh`);
+  assert.match(table, /dport 9101 accept/, `the rule set was never loaded:\n${table}`);
+
+  // And it filters. The declared port answers from another machine; the undeclared one does not.
+  assert.ok(await reach(9101),
+    "the declared port is closed, so the machine is filtering more than it was told to");
+  assert.ok(!(await reach(9102)),
+    "a port no module declared is still reachable, so the rule set restricts nothing");
+
+  // The machine did not lock itself out of the mesh: it is still taking declarations.
+  assert.doesNotMatch(await mesh("status"), /laptop\s+(failed|refused)/,
+    "the machine stopped doing what it was told after applying its own rule set");
+
+  // Removing the module that wanted the port closes it, with nobody editing a rule. This is the
+  // whole claim of a derived firewall, and it is also the second load — which must replace the
+  // table rather than add to it.
+  await mesh("unassign laptop talker");
+  await mesh("push laptop");
+  await new Promise((r) => setTimeout(r, 20_000));
+  assert.ok(!(await reach(9101)),
+    "the port stayed open after the module that wanted it was removed");
+});