diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index 571e14d..f277d16 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -229,3 +229,114 @@ test("when a machine cannot do what it was told, the mesh says which and why", { assert.doesNotMatch(said.split("not heard from")[0] ?? said, /anchor\s+(failed|refused)/, "a machine that did as it was told is listed as wrong"); }); + +test("a declaration waits for a machine that is switched off", { skip, timeout: 900_000 }, async () => { + // A machine is disconnected as an ordinary situation, not an exception (novox/hq ADR 0004), so + // a push to one that is not listening must wait rather than vanish. The queue is durable and the + // message persistent, which ought to be enough — but a lost declaration is silent, and "ought to + // be" is not a property. + // + // The machine is not merely idle here: its host is stopped, so nothing is consuming its queue. + + // Nothing this test asserts should depend on what another left behind. The machine still has a + // deliberately-impossible module from the test above, and while that is assigned the mesh never + // updates its account of what the machine holds — a partial report is not an account, on + // purpose (novox/hq 04-ISSUES/010). + await mesh("unassign laptop impossible"); + + // Stop listening, and prove it stopped — a test that pushed to a machine that was still running + // would pass having checked nothing. + // + // By process name, never by matching the command line: `pkill -f` matches the shell running it + // too, which kills the connection carrying the command and hangs the caller waiting for a reply + // that will never come. Cost an hour once, in this file. + await must("laptop", `pkill -x mesh-host || true; sleep 1`); + const listening = await on("laptop", `pgrep -x mesh-host`); + assert.equal(listening.ok, false, "the host is still running, so this proves nothing"); + + await must("anchor", `printf %s '{"module":"while-away","version":"1","resources":[` + + `{"id":"note","type":"file","path":"/etc/mesh-while-away","content":"waited"}]}' > /tmp/away.json`); + await must("anchor", `docker cp /tmp/away.json mesh-control:/away.json`); + await mesh("module add /away.json"); + await mesh("assign laptop while-away"); + await mesh("push laptop"); + + // Nothing has happened on the machine, because nothing is there to do it. + const before = await on("laptop", `test -f /etc/mesh-while-away`); + assert.equal(before.ok, false, "a machine with no host applied a declaration"); + + // And now it listens again. No second push, and nobody says anything. + await must("laptop", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 8`); + let arrived = false; + for (let i = 0; i < 20 && !arrived; i++) { + arrived = (await on("laptop", `test -f /etc/mesh-while-away`)).ok; + if (!arrived) await new Promise((r) => setTimeout(r, 2000)); + } + if (!arrived) { + // Everything needed to tell "the message was never queued" from "the host never read it". + const log = await on("laptop", `tail -20 /var/log/mesh-host.log`); + const queues = await on("anchor", + `docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`); + const owned = await on("anchor", + `docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`); + assert.fail(`a declaration sent to a switched-off machine was lost\n` + + `--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` + + `--- what each machine last did ---\n${owned.out}`); + } + assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited"); + + // And the mesh's account of what that machine holds catches up too, or a later declaration + // would tell it to remove what it has just been given. + await new Promise((r) => setTimeout(r, 4000)); + const owned = await must("anchor", + `docker exec mesh-store psql -U postgres -d inventory -qAt ` + + `-c "select owned from node where name = 'laptop'"`); + assert.match(owned, /while-away\.note/, `the mesh does not know the machine holds it: ${owned}`); +}); + +test("unassigning takes away exactly what it should", { skip, timeout: 900_000 }, async () => { + // Removal is the half nobody tests. The mesh takes away what IT declared and no longer declares, + // and never what the machine raised for itself from its bundle — which is the fault that + // destroyed a substrate once (novox/hq 04-ISSUES/010). + // + // Two modules, so the test can tell "removed the right one" from "removed everything". + for (const [name, path] of [["kept", "/etc/mesh-kept"], ["going", "/etc/mesh-going"]] as const) { + await must("anchor", `printf %s '{"module":"${name}","version":"1","resources":[` + + `{"id":"note","type":"file","path":"${path}","content":"${name}"}]}' > /tmp/${name}.json`); + await must("anchor", `docker cp /tmp/${name}.json mesh-control:/${name}.json`); + await mesh(`module add /${name}.json`); + await mesh(`assign anchor ${name}`); + } + await mesh("push anchor"); + await new Promise((r) => setTimeout(r, 6000)); + assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok, "the first module did not arrive"); + assert.ok((await on("anchor", `test -f /etc/mesh-going`)).ok, "the second module did not arrive"); + + await mesh("unassign anchor going"); + await mesh("push anchor"); + await new Promise((r) => setTimeout(r, 6000)); + + assert.equal((await on("anchor", `test -f /etc/mesh-going`)).ok, false, + "an unassigned module's file is still there"); + assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok, + "unassigning one module took another one's file with it"); + + // And the substrate this machine raised from its own bundle is untouched. It was not declared by + // the mesh, so the mesh must never remove it — the machine would take its own control plane + // away, which is exactly what happened before origins existed. + const running = await must("anchor", `docker ps --format '{{.Names}}'`); + for (const container of ["mesh-store", "mesh-broker", "mesh-control"]) { + assert.match(running, new RegExp(container), + `${container} was removed by a declaration that never declared it`); + } +}); + +test("a machine keeps what it was given when the mesh says nothing about it", { skip, timeout: 600_000 }, async () => { + // The other direction of the same rule. A node that is sent a declaration mentioning none of its + // private network must not lose it: the network came from a module that is still assigned, and + // "not in this message" is not "no longer wanted". + assert.ok((await on("laptop", `test -f /etc/wireguard/mesh0.conf`)).ok, + "the private network's configuration is gone"); + assert.ok((await on("laptop", `grep -q anchor.internal /etc/hosts`)).ok, + "the mesh's names are gone"); +});