One dropped lookup should not cost a two-hour run #24
@@ -76,6 +76,19 @@ export async function confirmEgress(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
log(` ${machine} reaches the internet over its uplink (${UPSTREAM} answered ${code})`);
|
log(` ${machine} reaches the internet over its uplink (${UPSTREAM} answered ${code})`);
|
||||||
|
|
||||||
|
// **Now that the path is proven, stop one dropped packet from costing a whole run.**
|
||||||
|
//
|
||||||
|
// The check above deliberately uses the uplink alone, because proving that path is the point
|
||||||
|
// of it. What follows is a different concern: a bed runs for hours after this, pulling images
|
||||||
|
// on four machines at once, and the uplink's resolver is a single server under exactly that
|
||||||
|
// load. Three runs have died at a lookup timeout well after this check passed — not because
|
||||||
|
// the path was broken, but because one query went unanswered.
|
||||||
|
//
|
||||||
|
// So the uplink stays first and keeps being used; public resolvers are appended behind it, and
|
||||||
|
// the retry is tightened so a silent server costs seconds rather than the step. A broken uplink
|
||||||
|
// still fails the check above, before any of this.
|
||||||
|
await resilientResolver(name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,6 +130,23 @@ async function reaches(name: string, waitSeconds: number): Promise<string | null
|
|||||||
* machine with egress that is the uplink by construction, since every scenario range is routed
|
* machine with egress that is the uplink by construction, since every scenario range is routed
|
||||||
* explicitly and nothing else defaults.
|
* explicitly and nothing else defaults.
|
||||||
*/
|
*/
|
||||||
|
/**
|
||||||
|
* Keep the uplink as the resolver and give it company.
|
||||||
|
*
|
||||||
|
* Appended rather than replacing: the uplink is still asked first and still proves the modelled
|
||||||
|
* path. The fallbacks only answer when it does not, which under a four-machine image pull is a
|
||||||
|
* thing that happens and has ended three runs.
|
||||||
|
*/
|
||||||
|
async function resilientResolver(name: string): Promise<void> {
|
||||||
|
await incus([
|
||||||
|
"exec", name, "--", "sh", "-c",
|
||||||
|
`via=$(ip -4 route show default | awk '{print $3}' | head -n1); ` +
|
||||||
|
`{ [ -n "$via" ] && printf 'nameserver %s\\n' "$via"; ` +
|
||||||
|
`printf 'nameserver 1.1.1.1\\nnameserver 8.8.8.8\\n'; ` +
|
||||||
|
`printf 'options timeout:2 attempts:3\\n'; } > /etc/resolv.conf; true`,
|
||||||
|
], 30_000);
|
||||||
|
}
|
||||||
|
|
||||||
async function pointResolverAtTheUplink(name: string): Promise<void> {
|
async function pointResolverAtTheUplink(name: string): Promise<void> {
|
||||||
await incus([
|
await incus([
|
||||||
"exec", name, "--", "sh", "-c",
|
"exec", name, "--", "sh", "-c",
|
||||||
|
|||||||
Reference in New Issue
Block a user