From b053dc66bf9d7d0e1870e71e927363340ed70699 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 07:39:30 +0200 Subject: [PATCH] Give the base build several mirrors and retry a failed pacman step (issue 700) The stock Arch image names one mirror, and the base build failed twice in a morning when mirrors.kernel.org timed out and then answered 500. Write a fixed list of six mirrors before anything is installed, so pacman falls back per file, and retry a failed step three times with a growing pause, saying which mirror failed. --- src/lifecycle/base.ts | 111 ++++++++++++++++++++++++++++++++++---- test/base-mirrors.test.ts | 100 ++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+), 9 deletions(-) create mode 100644 test/base-mirrors.test.ts diff --git a/src/lifecycle/base.ts b/src/lifecycle/base.ts index a9ccdd7..15762c4 100644 --- a/src/lifecycle/base.ts +++ b/src/lifecycle/base.ts @@ -23,6 +23,88 @@ export const UPSTREAM_IMAGE = "images:archlinux/current"; const BUILDER = "mesh-lab-base-builder"; +/** + * **The mirrors the builder installs from, several and in order** (novox/hq issue 700). + * + * The stock image names one mirror, mirrors.kernel.org, and a base build failed twice on it in one morning: + * timeouts while installing the runtime, then a 500 on wireguard-tools. pacman tries the next server in the + * list for each file it could not fetch, so a list of several turns one mirror's bad minute into a slower + * download instead of a failed build. + * + * A fixed list rather than ranking at build time: ranking (rankmirrors, reflector) needs a package installed + * first, from the one mirror that is the problem, and gives a different list on every build. These are + * Arch's own geo-routed mirror first, then well-kept European mirrors near the lab's machine, then the + * stock image's own. Each answered for core.db on 2026-10-14. A mirror that goes away costs a fallback, + * not a build, and is replaced here. + */ +export const MIRRORS = [ + "https://geo.mirror.pkgbuild.com/$repo/os/$arch", + "https://mirror.netcologne.de/archlinux/$repo/os/$arch", + "https://ftp.fau.de/archlinux/$repo/os/$arch", + "https://archlinux.mirror.liteserver.nl/$repo/os/$arch", + "https://mirror.leaseweb.net/archlinux/$repo/os/$arch", + "https://mirrors.kernel.org/archlinux/$repo/os/$arch", +]; + +/** The mirrorlist the builder is given: every mirror, in order. */ +export function mirrorlist(): string { + return "# Written by the mesh-lab base build (novox/hq issue 700): several mirrors, tried in order.\n" + + MIRRORS.map((m) => `Server = ${m}`).join("\n") + "\n"; +} + +/** The mirrors a failed pacman step names: "from " in what it said, each once. */ +export function failedMirrors(said: string): string[] { + const hosts = [...said.matchAll(/\bfrom ([a-z0-9.-]+\.[a-z]{2,})\b/gi)].map((m) => m[1]!.toLowerCase()); + return [...new Set(hosts)]; +} + +/** How long to wait before each retry of a failed pacman step: three retries, backing off. */ +export const PACMAN_BACKOFF_MS = [10_000, 30_000, 90_000]; + +/** What a base build runs through, so a test can stand in for the hypervisor and the clock. */ +export interface BaseDeps { + incus: (args: string[], timeoutMs?: number) => Promise; + incusOk: (args: string[], timeoutMs?: number) => Promise; + succeeds: (args: string[], timeoutMs?: number) => Promise; + sleep: (ms: number) => Promise; +} + +const realDeps: BaseDeps = { + incus, incusOk, succeeds, sleep: (ms) => new Promise((r) => setTimeout(r, ms)), +}; + +/** + * pacmanIn runs one pacman step in the builder, and **runs it again after a pause when it fails**, up to + * three more times, saying each time which mirror failed and what it said. Installing is idempotent — what is + * installed stays installed — so a retry repeats only what did not happen. The last failure is thrown, naming + * every mirror that failed along the way. + */ +export async function pacmanIn(name: string, args: string[], log: (m: string) => void, deps: BaseDeps = realDeps): + Promise { + const failed: string[] = []; + for (let attempt = 0; ; attempt++) { + try { + await deps.incus(["exec", name, "--", "pacman", ...args], 600_000); + return; + } catch (err) { + // What pacman said, where incus kept it apart; otherwise the whole message. + const said = (err as { stderr?: string }).stderr?.trim() || (err as Error).message; + const mirrors = failedMirrors(said); + for (const m of mirrors) if (!failed.includes(m)) failed.push(m); + const where = mirrors.length > 0 ? `from ${mirrors.join(", ")}` : "naming no mirror"; + const first = said.split("\n").find((l) => /error/i.test(l))?.trim() ?? said.split("\n")[0]?.trim() ?? ""; + if (attempt >= PACMAN_BACKOFF_MS.length) { + throw new BaseImageError(`pacman ${args.join(" ")} failed ${attempt + 1} times; the mirrors that failed: ` + + `${failed.join(", ") || "none named"}. Last: ${said}`); + } + const wait = PACMAN_BACKOFF_MS[attempt]!; + log(` pacman ${args.join(" ")} failed ${where} (${first}); trying again in ${wait / 1000}s ` + + `(attempt ${attempt + 2} of ${PACMAN_BACKOFF_MS.length + 1})`); + await deps.sleep(wait); + } + } +} + export class BaseImageError extends Error { constructor(message: string) { super(message); @@ -38,7 +120,9 @@ export class BaseImageError extends Error { */ export async function buildBaseImage( log: (message: string) => void = () => {}, + deps: BaseDeps = realDeps, ): Promise<{ alias: string; runtime: string }> { + const { incus, incusOk, succeeds } = deps; await succeeds(["delete", "-f", BUILDER], 120_000); log(` launching ${BUILDER} from ${UPSTREAM_IMAGE}, with a network`); @@ -50,10 +134,19 @@ export async function buildBaseImage( ], 300_000); try { - await waitForAgent(BUILDER); + await waitForAgent(BUILDER, deps); + + // Several mirrors before anything is fetched (MIRRORS), and read back: a list that was not written is + // the one mirror again, and the failure would look like the one this prevents. + log(` giving pacman ${MIRRORS.length} mirrors`); + await incus(["exec", BUILDER, "--", "sh", "-c", `printf '%s' '${mirrorlist()}' > /etc/pacman.d/mirrorlist`], 60_000); + const written = await incusOk(["exec", BUILDER, "--", "cat", "/etc/pacman.d/mirrorlist"], 60_000); + if (written?.trim() !== mirrorlist().trim()) { + throw new BaseImageError(`the mirrorlist written into ${BUILDER} reads back as ${JSON.stringify(written)}`); + } log(" installing a container runtime"); - await incus(["exec", BUILDER, "--", "pacman", "-Sy", "--noconfirm", "docker"], 600_000); + await pacmanIn(BUILDER, ["-Sy", "--noconfirm", "docker"], log, deps); // And the tools for the private network, for the same reason as the runtime: a sealed // scenario cannot install them, so a lab that omits them cannot test connectivity at all — @@ -63,27 +156,27 @@ export async function buildBaseImage( // declaration; a lab that brought the interface up itself would be testing its own setup // rather than the mesh's. log(" installing the tools for the private network"); - await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "wireguard-tools"], 600_000); + await pacmanIn(BUILDER, ["-S", "--noconfirm", "wireguard-tools"], log, deps); // And git, for the same reason again: a machine that builds modules clones them, and a sealed // scenario cannot install it. On a real build machine the mesh installs it as a package like // anything else — the lab is the special case, because its machines reach no mirror. log(" installing git, so a machine can build modules"); - await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "git"], 600_000); + await pacmanIn(BUILDER, ["-S", "--noconfirm", "git"], log, deps); // And nftables, because the mesh computes a machine's filtering and delivers it as a file // that a service reflects — and neither the file nor the service can install what loads it. // Installed and NOT enabled: whether a machine filters is the mesh's decision, and a lab that // turned it on itself would be testing its own setup. log(" installing nftables, so a machine can enforce what the mesh computed"); - await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "nftables"], 600_000); + await pacmanIn(BUILDER, ["-S", "--noconfirm", "nftables"], log, deps); // And dnsmasq, because a service is named under the machine it runs on — postgres.novox.internal // — and only a resolver can answer a name the mesh was never told about. Installed and NOT // started: whether a machine resolves for the mesh is the mesh's decision, and a lab that // turned it on itself would be testing its own setup. log(" installing dnsmasq, so a machine can answer names under another machine"); - await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "dnsmasq"], 600_000); + await pacmanIn(BUILDER, ["-S", "--noconfirm", "dnsmasq"], log, deps); // Trust the documentation ranges as plain-HTTP registries. // @@ -216,10 +309,10 @@ export async function baseImageExists(): Promise { * Wait for the guest agent, because `launch` returning means the VM started, not that anything * inside it will answer. */ -async function waitForAgent(name: string): Promise { +async function waitForAgent(name: string, deps: BaseDeps = realDeps): Promise { for (let i = 0; i < 90; i++) { - if (await succeeds(["exec", name, "--", "true"], 10_000)) return; - await new Promise((r) => setTimeout(r, 2_000)); + if (await deps.succeeds(["exec", name, "--", "true"], 10_000)) return; + await deps.sleep(2_000); } throw new BaseImageError(`${name} started and its agent never answered.`); } diff --git a/test/base-mirrors.test.ts b/test/base-mirrors.test.ts new file mode 100644 index 0000000..ba42078 --- /dev/null +++ b/test/base-mirrors.test.ts @@ -0,0 +1,100 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { IncusError } from "../src/incus/client.ts"; +import { + buildBaseImage, failedMirrors, MIRRORS, mirrorlist, PACMAN_BACKOFF_MS, pacmanIn, type BaseDeps, +} from "../src/lifecycle/base.ts"; +import { BASE_IMAGE_ALIAS } from "../src/lifecycle/place.ts"; + +/** + * Defends novox/hq issue 700: the base build installed from the one mirror the stock image names, + * mirrors.kernel.org, and failed twice in a morning when that mirror timed out and then answered 500. + */ + +const KERNEL_500 = "error: failed retrieving file 'wireguard-tools-1.0.20260223-1-x86_64.pkg.tar.zst' from " + + "mirrors.kernel.org : The requested URL returned error: 500\nwarning: failed to retrieve some files\n" + + "error: failed to commit transaction (failed to retrieve some files)"; + +/** A hypervisor for one base build: pacman fails `fails` times per step, then installs. */ +function aBuilder(fails: number, said = KERNEL_500) { + const ran: string[] = []; + const slept: number[] = []; + let mirrorlistWritten = ""; + const tries = new Map(); + const deps: BaseDeps = { + incus: async (args) => { + ran.push(args.join(" ")); + const at = args.indexOf("pacman"); + if (at >= 0) { + const step = args.slice(at).join(" "); + const n = (tries.get(step) ?? 0) + 1; + tries.set(step, n); + if (n <= fails) throw new IncusError(args, said, 1); + } + const printf = args.join(" ").match(/printf '%s' '([\s\S]*)' > \/etc\/pacman.d\/mirrorlist/); + if (printf) mirrorlistWritten = printf[1]!; + return { stdout: "", stderr: "" }; + }, + incusOk: async (args) => { + const c = args.join(" "); + if (c.endsWith("cat /etc/pacman.d/mirrorlist")) return mirrorlistWritten; + if (c.includes("{{.ServerVersion}}")) return "29.8.2"; + if (c.includes("InsecureRegistryCIDRs")) return "[127.0.0.0/8 192.0.2.0/24]"; + if (c.startsWith("image list")) return BASE_IMAGE_ALIAS; + return "tool 1.0"; + }, + succeeds: async () => true, + sleep: async (ms) => { slept.push(ms); }, + }; + return { deps, ran, slept }; +} + +test("the builder is given several mirrors, the stock one last, before anything is installed", async () => { + const hosts = MIRRORS.map((m) => new URL(m.replace("$repo", "core").replace("$arch", "x86_64")).host); + assert.ok(new Set(hosts).size >= 4, `too few mirrors: ${hosts.join(", ")}`); + assert.equal(hosts.at(-1), "mirrors.kernel.org"); + assert.ok(!mirrorlist().includes("'"), "the mirrorlist is written inside single quotes and holds one"); + assert.equal(mirrorlist().split("\n").filter((l) => l.startsWith("Server = ")).length, MIRRORS.length); + + const b = aBuilder(0); + await buildBaseImage(() => {}, b.deps); + const wrote = b.ran.findIndex((c) => c.includes("/etc/pacman.d/mirrorlist")); + const firstPacman = b.ran.findIndex((c) => c.includes(" pacman ")); + assert.ok(wrote >= 0 && wrote < firstPacman, "pacman ran before the mirrorlist was written"); +}); + +test("a pacman step that fails is tried again after a pause, saying which mirror failed", async () => { + const said: string[] = []; + const b = aBuilder(2); + await pacmanIn("builder", ["-S", "--noconfirm", "wireguard-tools"], (m) => said.push(m), b.deps); + assert.equal(b.ran.length, 3); + assert.deepEqual(b.slept, PACMAN_BACKOFF_MS.slice(0, 2)); + assert.ok(PACMAN_BACKOFF_MS[1]! > PACMAN_BACKOFF_MS[0]!, "no backoff"); + assert.equal(said.length, 2); + assert.match(said[0]!, /failed from mirrors\.kernel\.org \(error: failed retrieving file 'wireguard-tools/); + assert.match(said[0]!, /trying again in 10s \(attempt 2 of 4\)/); +}); + +test("a step that keeps failing stops the build, naming every mirror that failed", async () => { + const b = aBuilder(99, "error: too many errors from mirrors.kernel.org\nerror: failed retrieving file 'x' from ftp.fau.de : 500"); + await assert.rejects( + pacmanIn("builder", ["-Sy", "--noconfirm", "docker"], () => {}, b.deps), + /failed 4 times; the mirrors that failed: mirrors\.kernel\.org, ftp\.fau\.de/, + ); + assert.equal(b.ran.length, PACMAN_BACKOFF_MS.length + 1); +}); + +test("a whole base build rides out a mirror's bad minute on every package", async () => { + const b = aBuilder(1); + const built = await buildBaseImage(() => {}, b.deps); + assert.equal(built.alias, BASE_IMAGE_ALIAS); + const pacman = b.ran.filter((c) => c.includes(" pacman ")); + assert.equal(pacman.length, 10, pacman.join("\n")); +}); + +test("the mirrors a failure names are read from what pacman said, each once", () => { + assert.deepEqual(failedMirrors(KERNEL_500), ["mirrors.kernel.org"]); + assert.deepEqual(failedMirrors("Connection timed out after 10000 milliseconds\nerror: too many errors from mirrors.kernel.org"), + ["mirrors.kernel.org"]); + assert.deepEqual(failedMirrors("error: target not found: dockr"), []); +}); -- 2.54.0