test/integration/mesh.test.ts

@@ -229,3 +229,114 @@ test("when a machine cannot do what it was told, the mesh says which and why", {
   assert.doesNotMatch(said.split("not heard from")[0] ?? said, /anchor\s+(failed|refused)/,
     "a machine that did as it was told is listed as wrong");
 });
+
+test("a declaration waits for a machine that is switched off", { skip, timeout: 900_000 }, async () => {
+  // A machine is disconnected as an ordinary situation, not an exception (novox/hq ADR 0004), so
+  // a push to one that is not listening must wait rather than vanish. The queue is durable and the
+  // message persistent, which ought to be enough — but a lost declaration is silent, and "ought to
+  // be" is not a property.
+  //
+  // The machine is not merely idle here: its host is stopped, so nothing is consuming its queue.
+
+  // Nothing this test asserts should depend on what another left behind. The machine still has a
+  // deliberately-impossible module from the test above, and while that is assigned the mesh never
+  // updates its account of what the machine holds — a partial report is not an account, on
+  // purpose (novox/hq 04-ISSUES/010).
+  await mesh("unassign laptop impossible");
+
+  // Stop listening, and prove it stopped — a test that pushed to a machine that was still running
+  // would pass having checked nothing.
+  //
+  // By process name, never by matching the command line: `pkill -f` matches the shell running it
+  // too, which kills the connection carrying the command and hangs the caller waiting for a reply
+  // that will never come. Cost an hour once, in this file.
+  await must("laptop", `pkill -x mesh-host || true; sleep 1`);
+  const listening = await on("laptop", `pgrep -x mesh-host`);
+  assert.equal(listening.ok, false, "the host is still running, so this proves nothing");
+
+  await must("anchor", `printf %s '{"module":"while-away","version":"1","resources":[` +
+    `{"id":"note","type":"file","path":"/etc/mesh-while-away","content":"waited"}]}' > /tmp/away.json`);
+  await must("anchor", `docker cp /tmp/away.json mesh-control:/away.json`);
+  await mesh("module add /away.json");
+  await mesh("assign laptop while-away");
+  await mesh("push laptop");
+
+  // Nothing has happened on the machine, because nothing is there to do it.
+  const before = await on("laptop", `test -f /etc/mesh-while-away`);
+  assert.equal(before.ok, false, "a machine with no host applied a declaration");
+
+  // And now it listens again. No second push, and nobody says anything.
+  await must("laptop", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 8`);
+  let arrived = false;
+  for (let i = 0; i < 20 && !arrived; i++) {
+    arrived = (await on("laptop", `test -f /etc/mesh-while-away`)).ok;
+    if (!arrived) await new Promise((r) => setTimeout(r, 2000));
+  }
+  if (!arrived) {
+    // Everything needed to tell "the message was never queued" from "the host never read it".
+    const log = await on("laptop", `tail -20 /var/log/mesh-host.log`);
+    const queues = await on("anchor",
+      `docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`);
+    const owned = await on("anchor",
+      `docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`);
+    assert.fail(`a declaration sent to a switched-off machine was lost\n` +
+      `--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` +
+      `--- what each machine last did ---\n${owned.out}`);
+  }
+  assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited");
+
+  // And the mesh's account of what that machine holds catches up too, or a later declaration
+  // would tell it to remove what it has just been given.
+  await new Promise((r) => setTimeout(r, 4000));
+  const owned = await must("anchor",
+    `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
+      `-c "select owned from node where name = 'laptop'"`);
+  assert.match(owned, /while-away\.note/, `the mesh does not know the machine holds it: ${owned}`);
+});
+
+test("unassigning takes away exactly what it should", { skip, timeout: 900_000 }, async () => {
+  // Removal is the half nobody tests. The mesh takes away what IT declared and no longer declares,
+  // and never what the machine raised for itself from its bundle — which is the fault that
+  // destroyed a substrate once (novox/hq 04-ISSUES/010).
+  //
+  // Two modules, so the test can tell "removed the right one" from "removed everything".
+  for (const [name, path] of [["kept", "/etc/mesh-kept"], ["going", "/etc/mesh-going"]] as const) {
+    await must("anchor", `printf %s '{"module":"${name}","version":"1","resources":[` +
+      `{"id":"note","type":"file","path":"${path}","content":"${name}"}]}' > /tmp/${name}.json`);
+    await must("anchor", `docker cp /tmp/${name}.json mesh-control:/${name}.json`);
+    await mesh(`module add /${name}.json`);
+    await mesh(`assign anchor ${name}`);
+  }
+  await mesh("push anchor");
+  await new Promise((r) => setTimeout(r, 6000));
+  assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok, "the first module did not arrive");
+  assert.ok((await on("anchor", `test -f /etc/mesh-going`)).ok, "the second module did not arrive");
+
+  await mesh("unassign anchor going");
+  await mesh("push anchor");
+  await new Promise((r) => setTimeout(r, 6000));
+
+  assert.equal((await on("anchor", `test -f /etc/mesh-going`)).ok, false,
+    "an unassigned module's file is still there");
+  assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok,
+    "unassigning one module took another one's file with it");
+
+  // And the substrate this machine raised from its own bundle is untouched. It was not declared by
+  // the mesh, so the mesh must never remove it — the machine would take its own control plane
+  // away, which is exactly what happened before origins existed.
+  const running = await must("anchor", `docker ps --format '{{.Names}}'`);
+  for (const container of ["mesh-store", "mesh-broker", "mesh-control"]) {
+    assert.match(running, new RegExp(container),
+      `${container} was removed by a declaration that never declared it`);
+  }
+});
+
+test("a machine keeps what it was given when the mesh says nothing about it", { skip, timeout: 600_000 }, async () => {
+  // The other direction of the same rule. A node that is sent a declaration mentioning none of its
+  // private network must not lose it: the network came from a module that is still assigned, and
+  // "not in this message" is not "no longer wanted".
+  assert.ok((await on("laptop", `test -f /etc/wireguard/mesh0.conf`)).ok,
+    "the private network's configuration is gone");
+  assert.ok((await on("laptop", `grep -q anchor.internal /etc/hosts`)).ok,
+    "the mesh's names are gone");
+});