From e1317c9a69ac316f8ee08322d6343146175ada08 Mon Sep 17 00:00:00 2001
From: jochen <jschoubben@gmail.com>
Date: Mon, 31 Aug 2026 13:54:04 +0200
Subject: [PATCH] Gather the evidence however the resolver test fails
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three times a diagnostic has not run because the thing before it threw: `must`
on a command that exits non-zero, and then a query that hung long enough to
take the harness's own timeout with it — which arrives as an error with no
evidence attached rather than as a failed assertion.

A thirty-second test costs fifteen minutes to re-run, so the evidence has to be
gathered whichever way it fails. One helper, used by every assertion here, and
the query is bounded on the machine rather than by the harness: a query that
hangs is a result, not an accident.
---
 test/integration/mesh.test.ts | 37 +++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts
index 09a268b..5bb3e8f 100644
--- a/test/integration/mesh.test.ts
+++ b/test/integration/mesh.test.ts
@@ -1402,21 +1402,37 @@ test("a service is reached by a name under the machine it runs on", {
   for (const machine of ["anchor", "laptop"]) {
     const state = await on(machine, `systemctl is-active dnsmasq.service`);
     if (state.out.trim() === "active") continue;
-    assert.fail(
-      `the resolver is not running on ${machine} (${state.out.trim()}):\n\n` +
-      `journal:\n${(await on(machine, `journalctl -u dnsmasq -n 25 --no-pager`)).out}\n` +
+    assert.fail(`the resolver is not running on ${machine} (${state.out.trim()}):\n\n` +
       `its config:\n${(await on(machine, `cat /etc/dnsmasq.conf`)).out}\n` +
-      `what the mesh wrote:\n${(await on(machine, `cat /etc/mesh-resolver/nodes.conf`)).out}\n` +
-      `resolv.conf:\n${(await on(machine, `cat /etc/resolv.conf`)).out}\n` +
-      `what holds a loopback address:\n` +
-      `${(await on(machine, `ss -lntup | grep 127.0.0 || echo none`)).out}`);
+      `${await diagnose(machine)}`);
   }
 
+  // Everything this test could want to know, gathered in one place.
+  //
+  // Three times now a diagnostic has not run because the thing before it threw: `must` on a
+  // command that fails, and then a query that hangs long enough to take the harness's own timeout
+  // with it. A 30-second test costs fifteen minutes to re-run, so the evidence has to be gathered
+  // whether the failure is an assertion, an error, or a hang.
+  const diagnose = async (machine: string) =>
+    `--- ${machine}\n` +
+    `dnsmasq: ${(await on(machine, `systemctl is-active dnsmasq.service`)).out.trim()}\n` +
+    `${(await on(machine, `journalctl -u dnsmasq -n 12 --no-pager`)).out}\n` +
+    `resolved: ${(await on(machine, `resolvectl status | head -30`)).out}\n` +
+    `resolv.conf:\n${(await on(machine, `cat /etc/resolv.conf`)).out}\n` +
+    `what the mesh wrote:\n${(await on(machine, `cat /etc/mesh-resolver/nodes.conf`)).out}\n` +
+    `listening:\n${(await on(machine, `ss -lnup | grep :53 || echo none`)).out}\n` +
+    `asked directly:\n${(await on(machine,
+      `timeout 5 resolvectl query postgres.anchor.internal 2>&1 || echo "no answer"`)).out}`;
+
   // Through the machine's own resolver, by the path an application actually takes: nsswitch, then
   // files, then DNS. `dig` would ask a server directly and prove less — the resolv.conf module is
   // half of what is being tested, and only this path goes through it.
+  //
+  // Bounded on the machine rather than by the harness: a query that hangs is a result, and letting
+  // it run into the harness's own timeout turns it into an error with no evidence attached.
   const resolves = async (machine: string, name: string) => {
-    const said = await on(machine, `getent hosts ${name} | head -1 | cut -d' ' -f1`, 30_000);
+    const said = await on(machine,
+      `timeout 5 getent hosts ${name} | head -1 | cut -d' ' -f1`, 20_000);
     return said.out.trim();
   };
   const addressOf = async (machine: string, node: string) =>
@@ -1434,7 +1450,8 @@ test("a service is reached by a name under the machine it runs on", {
       if (!got) await new Promise((r) => setTimeout(r, 3000));
     }
     assert.equal(got, anchorAt,
-      `${machine} does not resolve a service named under anchor: ${got}`);
+      `${machine} does not resolve a service named under anchor: ${got || "(nothing)"}\n\n` +
+      `${await diagnose(machine)}`);
   }
 
   // Any name at all, which is the whole point: the mesh was never told these exist.
@@ -1445,7 +1462,7 @@ test("a service is reached by a name under the machine it runs on", {
     ["radarr.laptop.internal", laptopAt],
   ] as const) {
     assert.equal(await resolves("laptop", name), expected,
-      `${name} did not resolve to the machine it is named under`);
+      `${name} did not resolve to the machine it is named under\n\n${await diagnose("laptop")}`);
   }
 
   // The machine's own name still resolves, and to the same place. Two accounts of where a machine