test/integration/one-node-mesh.test.ts

@@ -132,6 +132,7 @@ const ANCHOR_RUNS = "the anchor runs the module the mesh built";
 const DESCRIBES = "the control plane can describe the mesh, and what it says is true";
 const CATALOGUED = "the catalogue holds every module this mesh built";
 const NETWORK = "the machine's networking is what the modules asked for";
+const DECLARED = "every resource the mesh declared is true on the machine";
 const FOLLOWS = "a change to a module's source reaches the machine on its own";
 const SURVIVES = "the mesh comes back after the machine reboots";
 const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping", container: "amqp-ping" };
@@ -394,6 +395,7 @@ const PLAN: { code: string; title: string }[] = [
   { code: "V1", title: DESCRIBES },
   { code: "V2", title: CATALOGUED },
   { code: "V3", title: NETWORK },
+  { code: "V4", title: DECLARED },
   { code: "E1", title: FOLLOWS },
   { code: "E2", title: SURVIVES },
 ];
@@ -827,12 +829,103 @@ before(async () => {
     return said.join("\n");
   });
 
+  // ---- V4. THE WHOLE VOCABULARY, NOT THE PART I KEPT LOOKING AT --------------------------------
+  //
+  // **Every check in this file until now asked about containers.** A container is one resource kind
+  // out of ten — directory, file, user, network, access, archive, service, package, container,
+  // action — and a module is far more often the others: the firewall is a package and a service,
+  // the mesh's names are a file, a run-once step is an action or a container that exits. Asking
+  // only about containers is how a module with no container at all went unnoticed for this long.
+  //
+  // So this takes the declaration the machine was actually sent and verifies each resource in it,
+  // by kind, on the machine. Nothing is hand-picked: whatever the installed modules declared is
+  // what gets checked, and a kind nothing declared is REPORTED as unexercised rather than quietly
+  // counted as working.
+  await step("V4", DECLARED, NETWORK, async () => {
+    const plan = JSON.parse(await mesh(`plan ${CONTROL} --json`)) as {
+      resources: Record<string, unknown>[];
+    };
+    const seen = new Map<string, number>();
+    const wrong: string[] = [];
+    const unchecked: string[] = [];
+
+    for (const r of plan.resources) {
+      const kind = String(r["type"] ?? "");
+      seen.set(kind, (seen.get(kind) ?? 0) + 1);
+      const id = String(r["id"] ?? kind);
+      const check = async (command: string, why: string) => {
+        if (!(await on(CONTROL, command)).ok) wrong.push(`${kind} ${id}: ${why}`);
+      };
+      switch (kind) {
+        case "directory":
+          await check(`test -d ${quote(String(r["path"]))}`, `no directory at ${r["path"]}`);
+          break;
+        case "file":
+          await check(`test -f ${quote(String(r["path"]))}`, `no file at ${r["path"]}`);
+          break;
+        case "access":
+          await check(`test -e ${quote(String(r["path"]))}`, `nothing at ${r["path"]}`);
+          break;
+        case "archive":
+          await check(`test -e ${quote(String(r["path"]))}`,
+            `nothing unpacked at ${r["path"]} — the archive was never fetched`);
+          break;
+        case "package":
+          await check(`command -v pacman >/dev/null && pacman -Q ${quote(String(r["package"]))}`,
+            `the package ${r["package"]} is not installed`);
+          break;
+        case "service": {
+          // A unit the host put into a state. "running" is the state worth checking; a one-shot
+          // that has done its work reports inactive and that is correct (this is the reading that
+          // made the firewall module appear broken on every machine for months).
+          const unit = String(r["unit"]);
+          if (String(r["state"]) === "running") {
+            await check(`systemctl is-active ${quote(unit)} >/dev/null || ` +
+              `systemctl show -p ExecMainStatus --value ${quote(unit)} | grep -qx 0`,
+              `the unit ${unit} is neither active nor a one-shot that succeeded`);
+          }
+          break;
+        }
+        case "network":
+          await check(`docker network inspect ${quote(String(r["name"]))} >/dev/null 2>&1`,
+            `no network named ${r["name"]}`);
+          break;
+        case "container": {
+          const name = String(r["name"]);
+          if (r["run-once"] === true || r["schedule"]) {
+            // Not expected to be running: it ran, or it runs later. What matters is that it exists
+            // and, if it ran, that it succeeded.
+            await check(`docker inspect ${quote(name)} >/dev/null 2>&1`,
+              `the container ${name} was never created`);
+          } else {
+            await check(`docker ps --format '{{.Names}}' | grep -qx ${quote(name)}`,
+              `the container ${name} is not running`);
+          }
+          break;
+        }
+        default:
+          unchecked.push(`${kind} ${id}`);
+      }
+    }
+
+    const kinds = [...seen.entries()].sort().map(([k, n]) => `${k}×${n}`).join("  ");
+    const never = ["directory", "file", "user", "network", "access", "archive", "service",
+      "package", "container", "action"].filter((k) => !seen.has(k));
+    assert.deepEqual(wrong, [],
+      `the machine is not what the mesh said it should be:\n  ${wrong.join("\n  ")}`);
+    return [
+      `  declared          ${plan.resources.length} resources — ${kinds}`,
+      unchecked.length ? `  not verified here ${unchecked.join(", ")}` : `  every kind present was verified`,
+      never.length ? `  NOT EXERCISED     ${never.join(", ")} — nothing installed here declares one` : ``,
+    ].filter(Boolean).join("\n");
+  });
+
   // ---- 11. A CHANGE REACHES THE MACHINE ON ITS OWN ----------------------------------------------
   //
   // The whole point of the mesh, and the capability the migration depends on: move a module's
   // source and the running copy follows, with nobody driving the steps. Everything above is
   // machinery; this is what the machinery is for.
-  await step("E1", FOLLOWS, NETWORK, async () => {
+  await step("E1", FOLLOWS, DECLARED, async () => {
     const before = await mesh(`builds ${MODULE.module}`);
     const was = before.match(/sha256:[0-9a-f]{64}/)?.[0] ?? "";
     assert.ok(was, `nothing is pinned to rebuild from:\n${before}`);
@@ -947,6 +1040,7 @@ for (const name of [
   DESCRIBES,
   CATALOGUED,
   NETWORK,
+  DECLARED,
   FOLLOWS,
   SURVIVES,
 ]) {