From ea0a7f7e64ab25848994c45468481fff99308b4e Mon Sep 17 00:00:00 2001
From: jochen <jschoubben@gmail.com>
Date: Mon, 14 Sep 2026 18:23:25 +0200
Subject: [PATCH] One dropped lookup should not cost a two-hour run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The egress check proves the uplink resolves and reaches the internet, and that
is the right thing to check. What it does not cover is the hours afterwards: a
bed pulls images on four machines at once, the uplink's resolver is one server
under exactly that load, and three runs have now died at a lookup timeout long
after the check passed — the path was never broken, a query just went
unanswered.

The uplink stays first and keeps proving the path. Public resolvers sit behind
it and answer only when it does not, and the retry is tightened so a silent
server costs seconds. A genuinely broken uplink still fails the check, before
any of this applies.
---
 src/lifecycle/egress.ts | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/src/lifecycle/egress.ts b/src/lifecycle/egress.ts
index ae1fd1b..386018b 100644
--- a/src/lifecycle/egress.ts
+++ b/src/lifecycle/egress.ts
@@ -76,6 +76,19 @@ export async function confirmEgress(
       );
     }
     log(`  ${machine} reaches the internet over its uplink (${UPSTREAM} answered ${code})`);
+
+    // **Now that the path is proven, stop one dropped packet from costing a whole run.**
+    //
+    // The check above deliberately uses the uplink alone, because proving that path is the point
+    // of it. What follows is a different concern: a bed runs for hours after this, pulling images
+    // on four machines at once, and the uplink's resolver is a single server under exactly that
+    // load. Three runs have died at a lookup timeout well after this check passed — not because
+    // the path was broken, but because one query went unanswered.
+    //
+    // So the uplink stays first and keeps being used; public resolvers are appended behind it, and
+    // the retry is tightened so a silent server costs seconds rather than the step. A broken uplink
+    // still fails the check above, before any of this.
+    await resilientResolver(name);
   }
 }
 
@@ -117,6 +130,23 @@ async function reaches(name: string, waitSeconds: number): Promise<string | null
  * machine with egress that is the uplink by construction, since every scenario range is routed
  * explicitly and nothing else defaults.
  */
+/**
+ * Keep the uplink as the resolver and give it company.
+ *
+ * Appended rather than replacing: the uplink is still asked first and still proves the modelled
+ * path. The fallbacks only answer when it does not, which under a four-machine image pull is a
+ * thing that happens and has ended three runs.
+ */
+async function resilientResolver(name: string): Promise<void> {
+  await incus([
+    "exec", name, "--", "sh", "-c",
+    `via=$(ip -4 route show default | awk '{print $3}' | head -n1); ` +
+      `{ [ -n "$via" ] && printf 'nameserver %s\\n' "$via"; ` +
+      `printf 'nameserver 1.1.1.1\\nnameserver 8.8.8.8\\n'; ` +
+      `printf 'options timeout:2 attempts:3\\n'; } > /etc/resolv.conf; true`,
+  ], 30_000);
+}
+
 async function pointResolverAtTheUplink(name: string): Promise<void> {
   await incus([
     "exec", name, "--", "sh", "-c",