test/integration/mesh.test.ts

@@ -1402,21 +1402,37 @@ test("a service is reached by a name under the machine it runs on", {
   for (const machine of ["anchor", "laptop"]) {
     const state = await on(machine, `systemctl is-active dnsmasq.service`);
     if (state.out.trim() === "active") continue;
-    assert.fail(
-      `the resolver is not running on ${machine} (${state.out.trim()}):\n\n` +
-      `journal:\n${(await on(machine, `journalctl -u dnsmasq -n 25 --no-pager`)).out}\n` +
+    assert.fail(`the resolver is not running on ${machine} (${state.out.trim()}):\n\n` +
       `its config:\n${(await on(machine, `cat /etc/dnsmasq.conf`)).out}\n` +
-      `what the mesh wrote:\n${(await on(machine, `cat /etc/mesh-resolver/nodes.conf`)).out}\n` +
-      `resolv.conf:\n${(await on(machine, `cat /etc/resolv.conf`)).out}\n` +
-      `what holds a loopback address:\n` +
-      `${(await on(machine, `ss -lntup | grep 127.0.0 || echo none`)).out}`);
+      `${await diagnose(machine)}`);
   }
 
+  // Everything this test could want to know, gathered in one place.
+  //
+  // Three times now a diagnostic has not run because the thing before it threw: `must` on a
+  // command that fails, and then a query that hangs long enough to take the harness's own timeout
+  // with it. A 30-second test costs fifteen minutes to re-run, so the evidence has to be gathered
+  // whether the failure is an assertion, an error, or a hang.
+  const diagnose = async (machine: string) =>
+    `--- ${machine}\n` +
+    `dnsmasq: ${(await on(machine, `systemctl is-active dnsmasq.service`)).out.trim()}\n` +
+    `${(await on(machine, `journalctl -u dnsmasq -n 12 --no-pager`)).out}\n` +
+    `resolved: ${(await on(machine, `resolvectl status | head -30`)).out}\n` +
+    `resolv.conf:\n${(await on(machine, `cat /etc/resolv.conf`)).out}\n` +
+    `what the mesh wrote:\n${(await on(machine, `cat /etc/mesh-resolver/nodes.conf`)).out}\n` +
+    `listening:\n${(await on(machine, `ss -lnup | grep :53 || echo none`)).out}\n` +
+    `asked directly:\n${(await on(machine,
+      `timeout 5 resolvectl query postgres.anchor.internal 2>&1 || echo "no answer"`)).out}`;
+
   // Through the machine's own resolver, by the path an application actually takes: nsswitch, then
   // files, then DNS. `dig` would ask a server directly and prove less — the resolv.conf module is
   // half of what is being tested, and only this path goes through it.
+  //
+  // Bounded on the machine rather than by the harness: a query that hangs is a result, and letting
+  // it run into the harness's own timeout turns it into an error with no evidence attached.
   const resolves = async (machine: string, name: string) => {
-    const said = await on(machine, `getent hosts ${name} | head -1 | cut -d' ' -f1`, 30_000);
+    const said = await on(machine,
+      `timeout 5 getent hosts ${name} | head -1 | cut -d' ' -f1`, 20_000);
     return said.out.trim();
   };
   const addressOf = async (machine: string, node: string) =>
@@ -1434,7 +1450,8 @@ test("a service is reached by a name under the machine it runs on", {
       if (!got) await new Promise((r) => setTimeout(r, 3000));
     }
     assert.equal(got, anchorAt,
-      `${machine} does not resolve a service named under anchor: ${got}`);
+      `${machine} does not resolve a service named under anchor: ${got || "(nothing)"}\n\n` +
+      `${await diagnose(machine)}`);
   }
 
   // Any name at all, which is the whole point: the mesh was never told these exist.
@@ -1445,7 +1462,7 @@ test("a service is reached by a name under the machine it runs on", {
     ["radarr.laptop.internal", laptopAt],
   ] as const) {
     assert.equal(await resolves("laptop", name), expected,
-      `${name} did not resolve to the machine it is named under`);
+      `${name} did not resolve to the machine it is named under\n\n${await diagnose("laptop")}`);
   }
 
   // The machine's own name still resolves, and to the same place. Two accounts of where a machine