scenarios/trust-anchor.yml

@@ -0,0 +1,40 @@
+# One machine that becomes a mesh, runs the mesh's own certificate authority, and is then given the
+# module that makes it trust it — the bed for novox/hq ADR 0147 and issue 129.
+#
+# The question is narrow and the bed is shaped to answer only it: does a machine holding `ca-trust`
+# verify a certificate from the mesh's own authority with no bundle argument and no `-k`, and does
+# it stop verifying it when the module is taken away? The authority itself is what is dialled —
+# step-ca serves its own API with a leaf it issued — so nothing else has to be right for the answer
+# to mean something. No proxy, no routed name, no public issuance: those are the certificates and
+# route-forwarding beds, and a trust bed that leaned on them would pass for their reasons.
+#
+# The negative half is not optional. It is asserted BEFORE the module is assigned and again AFTER it
+# is unassigned, because an anchor bed that only ever checks the success is one that would pass on a
+# machine that already trusted everything.
+#
+#   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
+#   MESH_LAB_CATALOG=.../mesh-catalog/modules
+#   step-ca's image is upstream and pinned by the catalogue; the machine pulls it over its uplink.
+#   ca-trust carries no image at all — a script, a unit, and the machine's own systemd.
+scenario: trust-anchor
+
+segments:
+  hosting:
+    kind: public
+    cidr: [192.0.2.0/24]
+
+machines:
+  anchor:
+    at: { segment: hosting, address: [192.0.2.10] }
+    egress: true
+    inbound: allow
+    memory: 3GiB
+    cpus: 2
+
+images:
+  - mesh-controller:development
+
+place:
+  # Only the host. The authority's image comes from the internet over the machine's uplink, and the
+  # trust module has nothing to place.
+  all: [host]

test/integration/assigned-ca-trust.test.ts

@@ -0,0 +1,221 @@
+/**
+ * A machine trusts the mesh's own authority because a module put its root there — and stops when
+ * the module is taken away (novox/hq ADR 0147, 04-ISSUES/129).
+ *
+ * What is dialled is the authority itself. step-ca serves its own API with a leaf it issued, so a
+ * plain client verifying that handshake — no `-k`, no `--cacert` — is verifying exactly one thing:
+ * that this machine's trust store now contains the mesh's root. No proxy, no routed name, no public
+ * issuance. A trust bed leaning on those would pass for their reasons, which is the failure this
+ * whole sequence keeps producing.
+ *
+ * The negative half runs twice, before the module is assigned and after it is unassigned. An anchor
+ * bed that only checks the success would pass on a machine that already trusted everything, and
+ * would say nothing at all about removal — which is the half issue 129 asked for by name.
+ *
+ *   MESH_LAB_HOST_BINARY=.../mesh-host
+ *   MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
+ *   MESH_LAB_CATALOG=.../mesh-catalog/modules
+ *   step-ca's image is upstream, pinned by the catalogue, pulled by the machine over its uplink.
+ *   ca-trust carries no image: a script, a unit, and the machine's own systemd.
+ */
+
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync } from "node:fs";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec } from "../../src/lifecycle/operate.ts";
+import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
+import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueIsPresent } from "./harness.ts";
+import type { HeldImage } from "../../src/pinning.ts";
+
+const capability = await labIsUsable();
+const binary = hostBinaryPath();
+const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
+
+const skip = !capability.usable
+  ? `lab not usable: ${capability.why}`
+  : !binary || !existsSync(binary)
+    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
+    : !bundle || !existsSync(bundle)
+      ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
+      : catalogueIsPresent();
+
+const SCENARIO = "trust-anchor";
+const MACHINE = "anchor";
+/** The authority's own API, which it serves with a certificate it issued itself. */
+const AUTHORITY = "https://127.0.0.1:9000/health";
+/** Where the module puts the root, and therefore what removal must take away. */
+const ANCHOR = "/etc/ca-certificates/trust-source/anchors/mesh-internal-ca.crt";
+
+let instanceId = "";
+let held: HeldImage[] = [];
+
+function quote(s: string): string {
+  return `'${s.replaceAll("'", `'\\''`)}'`;
+}
+
+async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
+  const { stdout } = await exec(instanceId, MACHINE, [
+    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
+  ], timeoutMs);
+  const marker = stdout.lastIndexOf("__exit=");
+  if (marker < 0) return { out: stdout, ok: false };
+  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
+}
+
+async function must(command: string, timeoutMs?: number): Promise<string> {
+  const { out, ok } = await on(command, timeoutMs);
+  if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
+  return out;
+}
+
+/** The control plane, a container on the node. */
+async function mesh(command: string, timeoutMs?: number): Promise<string> {
+  return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
+}
+
+function tokenFrom(said: string): string {
+  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
+  assert.ok(found, `no token in:\n${said}`);
+  return found;
+}
+
+async function settled(withinMs = 480_000): Promise<void> {
+  const until = Date.now() + withinMs;
+  let last = "";
+  while (Date.now() < until) {
+    const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
+    if (asked.ok) {
+      try {
+        const state = JSON.parse(asked.out) as {
+          wrong: { node: string; outcome: string }[];
+          waiting: { node: string }[];
+          reported: { node: string; outcome: string; current: boolean }[];
+        };
+        const bad = state.wrong.find((w) => w.node === MACHINE);
+        if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
+        const word = state.reported.find((r) => r.node === MACHINE);
+        if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
+        last = asked.out;
+      } catch (err) {
+        if (err instanceof Error && err.message.includes("did not apply")) throw err;
+        last = asked.out;
+      }
+    }
+    await new Promise((r) => setTimeout(r, 5000));
+  }
+  throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
+}
+
+/** Install a catalogue module's manifest into the control plane, read from the catalogue. */
+async function register(module: string): Promise<void> {
+  const manifest = catalogueModule(module, held);
+  await must(
+    `printf %s ${quote(manifest)} > /tmp/${module}.json && ` +
+      `docker cp /tmp/${module}.json mesh-controller:/${module}.json`,
+  );
+  await mesh(`module add /${module}.json`);
+}
+
+/** Dial the authority the way anything on this machine would: verifying, with nothing handed to it. */
+async function verifying(): Promise<{ out: string; ok: boolean }> {
+  return on(`curl --silent --show-error --max-time 10 ${AUTHORITY}`);
+}
+
+before(async () => {
+  if (skip) return;
+
+  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
+    onProgress: (m) => console.log(`raise: ${m}`),
+  });
+  instanceId = raised.instanceId;
+  held = raised.images;
+
+  await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
+  await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
+  const up = await must(`docker ps --format '{{.Names}}'`);
+  for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
+    assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
+  }
+
+  // The control plane is a container, and a container that is restarting answers nothing. Asked
+  // until it answers rather than once, so a crash-looping control plane is reported as itself
+  // instead of as whatever command happened to be sent first.
+  let control = { out: "", ok: false };
+  const answering = Date.now() + 120_000;
+  while (Date.now() < answering) {
+    control = await on(`docker exec mesh-controller /mesh-controller status`);
+    if (control.ok) break;
+    await new Promise((r) => setTimeout(r, 3000));
+  }
+  assert.ok(control.ok,
+    `the control plane never answered:\n${control.out}\n` +
+      `${(await on(`docker logs mesh-controller 2>&1 | tail -40`)).out}`);
+
+  await mesh(`node add ${MACHINE}`);
+  const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
+  await must(`${HOST_PATH} enrol --token ${quote(token)}`);
+  await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
+}, { timeout: 1_800_000 });
+
+after(async () => {
+  if (instanceId) await destroy(instanceId);
+  await destroyAll(`${SCENARIO}-`);
+}, { timeout: 600_000 });
+
+test("the mesh's authority is verified on a machine holding ca-trust, and not on one that is not", {
+  skip, timeout: 1_800_000,
+}, async () => {
+  // The authority first, on its own. Nothing about trust yet.
+  await register("step-ca");
+  await mesh(`module issue step-ca --node ${MACHINE}`);
+  await mesh(`assign ${MACHINE} step-ca`);
+  await mesh(`push ${MACHINE}`);
+  await settled();
+
+  // It is up and answering — asked the way nothing else in this bed is allowed to ask, with
+  // verification off, because at this point in the bed no machine could verify it.
+  let answering = false;
+  const until = Date.now() + 180_000;
+  while (Date.now() < until && !answering) {
+    answering = (await on(`curl --silent --insecure --max-time 5 ${AUTHORITY}`)).ok;
+    if (!answering) await new Promise((r) => setTimeout(r, 3000));
+  }
+  assert.ok(answering, `the authority never answered:\n${(await on(`docker logs step-ca 2>&1 | tail -30`)).out}`);
+
+  // **The negative half, before anything is assigned.** If this passes, the bed is measuring
+  // something already in the machine's trust store and everything below it is worthless.
+  const before = await verifying();
+  assert.equal(before.ok, false, `the authority verified before anything anchored its root:\n${before.out}`);
+  assert.match(before.out, /certificate|issuer/i, `it failed for some other reason:\n${before.out}`);
+
+  // Now the module.
+  await register("ca-trust");
+  await mesh(`assign ${MACHINE} ca-trust`);
+  await mesh(`push ${MACHINE}`);
+  await settled();
+
+  const unit = await on(`systemctl is-active mesh-ca-trust.service`);
+  assert.ok(unit.ok, `the unit is ${unit.out.trim()}:\n${(await on(`journalctl -u mesh-ca-trust --no-pager | tail -30`)).out}`);
+  await must(`test -s ${ANCHOR}`);
+  const anchors = await must(`trust list | grep -A2 -i 'Mesh Internal CA' || trust list`);
+  assert.match(anchors, /Mesh Internal CA/, `the mesh's authority is not among the machine's anchors:\n${anchors}`);
+
+  // **The whole claim, in one command.** No -k, no --cacert: the machine's own trust store, and a
+  // certificate the mesh's authority issued.
+  const after = await verifying();
+  assert.ok(after.ok, `the authority still does not verify with the module assigned:\n${after.out}`);
+
+  // **And removal is the same unit's business.** Unassigned, the host stops it; stopping it takes
+  // the anchor away and refreshes the bundles, so the machine stops trusting the mesh.
+  await mesh(`unassign ${MACHINE} ca-trust`);
+  await mesh(`push ${MACHINE}`);
+  await settled();
+
+  const gone = await on(`test -e ${ANCHOR}`);
+  assert.equal(gone.ok, false, "the anchor is still on the machine after the module was unassigned");
+  const afterwards = await verifying();
+  assert.equal(afterwards.ok, false, `the machine still verifies the mesh's authority with nothing anchoring it:\n${afterwards.out}`);
+  assert.match(afterwards.out, /certificate|issuer/i, `it failed for some other reason:\n${afterwards.out}`);
+});

test/integration/harness.ts

@@ -40,6 +40,14 @@ const UPSTREAM_STORE =
   "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
 const UPSTREAM_BROKER =
   "cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b";
+/**
+ * And the bus, for `foundation-first-node-nats.lock` — the bundle the mesh raises since it stopped
+ * speaking AMQP (novox/hq ADR 0131). The digest is the bundle's own: what the registry served was a
+ * copy of the upstream image, so the same digest resolves on Docker Hub, and this is a prefix being
+ * removed rather than a reference being replaced.
+ */
+const UPSTREAM_BUS =
+  "nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927";
 
 /**
  * The foundation bundle as a machine should receive it.
@@ -53,6 +61,7 @@ export function foundationBundle(path: string, held: HeldImage[]): string {
   text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE);
   text = text.replaceAll(
     /[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER);
+  text = text.replaceAll(/[A-Za-z0-9_.:-]+:[0-9]+\/nats@sha256:[0-9a-f]{64}/g, UPSTREAM_BUS);
   return pinnedInto(text, held);
 }