One dropped lookup should not cost a two-hour run #24
@@ -76,6 +76,19 @@ export async function confirmEgress(
|
||||
);
|
||||
}
|
||||
log(` ${machine} reaches the internet over its uplink (${UPSTREAM} answered ${code})`);
|
||||
|
||||
// **Now that the path is proven, stop one dropped packet from costing a whole run.**
|
||||
//
|
||||
// The check above deliberately uses the uplink alone, because proving that path is the point
|
||||
// of it. What follows is a different concern: a bed runs for hours after this, pulling images
|
||||
// on four machines at once, and the uplink's resolver is a single server under exactly that
|
||||
// load. Three runs have died at a lookup timeout well after this check passed — not because
|
||||
// the path was broken, but because one query went unanswered.
|
||||
//
|
||||
// So the uplink stays first and keeps being used; public resolvers are appended behind it, and
|
||||
// the retry is tightened so a silent server costs seconds rather than the step. A broken uplink
|
||||
// still fails the check above, before any of this.
|
||||
await resilientResolver(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,6 +130,23 @@ async function reaches(name: string, waitSeconds: number): Promise<string | null
|
||||
* machine with egress that is the uplink by construction, since every scenario range is routed
|
||||
* explicitly and nothing else defaults.
|
||||
*/
|
||||
/**
|
||||
* Keep the uplink as the resolver and give it company.
|
||||
*
|
||||
* Appended rather than replacing: the uplink is still asked first and still proves the modelled
|
||||
* path. The fallbacks only answer when it does not, which under a four-machine image pull is a
|
||||
* thing that happens and has ended three runs.
|
||||
*/
|
||||
async function resilientResolver(name: string): Promise<void> {
|
||||
await incus([
|
||||
"exec", name, "--", "sh", "-c",
|
||||
`via=$(ip -4 route show default | awk '{print $3}' | head -n1); ` +
|
||||
`{ [ -n "$via" ] && printf 'nameserver %s\\n' "$via"; ` +
|
||||
`printf 'nameserver 1.1.1.1\\nnameserver 8.8.8.8\\n'; ` +
|
||||
`printf 'options timeout:2 attempts:3\\n'; } > /etc/resolv.conf; true`,
|
||||
], 30_000);
|
||||
}
|
||||
|
||||
async function pointResolverAtTheUplink(name: string): Promise<void> {
|
||||
await incus([
|
||||
"exec", name, "--", "sh", "-c",
|
||||
|
||||
Reference in New Issue
Block a user