One dropped lookup should not cost a two-hour run #24

Merged
jschoubben merged 1 commits from fix/one-dropped-lookup-should-not-cost-a-run into main 2026-09-14 16:23:49 +00:00
Showing only changes of commit ea0a7f7e64 - Show all commits
+30
View File
@@ -76,6 +76,19 @@ export async function confirmEgress(
);
}
log(` ${machine} reaches the internet over its uplink (${UPSTREAM} answered ${code})`);
// **Now that the path is proven, stop one dropped packet from costing a whole run.**
//
// The check above deliberately uses the uplink alone, because proving that path is the point
// of it. What follows is a different concern: a bed runs for hours after this, pulling images
// on four machines at once, and the uplink's resolver is a single server under exactly that
// load. Three runs have died at a lookup timeout well after this check passed — not because
// the path was broken, but because one query went unanswered.
//
// So the uplink stays first and keeps being used; public resolvers are appended behind it, and
// the retry is tightened so a silent server costs seconds rather than the step. A broken uplink
// still fails the check above, before any of this.
await resilientResolver(name);
}
}
@@ -117,6 +130,23 @@ async function reaches(name: string, waitSeconds: number): Promise<string | null
* machine with egress that is the uplink by construction, since every scenario range is routed
* explicitly and nothing else defaults.
*/
/**
* Keep the uplink as the resolver and give it company.
*
* Appended rather than replacing: the uplink is still asked first and still proves the modelled
* path. The fallbacks only answer when it does not, which under a four-machine image pull is a
* thing that happens and has ended three runs.
*/
async function resilientResolver(name: string): Promise<void> {
await incus([
"exec", name, "--", "sh", "-c",
`via=$(ip -4 route show default | awk '{print $3}' | head -n1); ` +
`{ [ -n "$via" ] && printf 'nameserver %s\\n' "$via"; ` +
`printf 'nameserver 1.1.1.1\\nnameserver 8.8.8.8\\n'; ` +
`printf 'options timeout:2 attempts:3\\n'; } > /etc/resolv.conf; true`,
], 30_000);
}
async function pointResolverAtTheUplink(name: string): Promise<void> {
await incus([
"exec", name, "--", "sh", "-c",