Give the base build several mirrors and retry a failed pacman step (issue 700)

The stock Arch image names one mirror, and the base build failed twice in
a morning when mirrors.kernel.org timed out and then answered 500. Write a
fixed list of six mirrors before anything is installed, so pacman falls
back per file, and retry a failed step three times with a growing pause,
saying which mirror failed.

src/lifecycle/base.ts

@@ -23,6 +23,88 @@ export const UPSTREAM_IMAGE = "images:archlinux/current";
 
 const BUILDER = "mesh-lab-base-builder";
 
+/**
+ * **The mirrors the builder installs from, several and in order** (novox/hq issue 700).
+ *
+ * The stock image names one mirror, mirrors.kernel.org, and a base build failed twice on it in one morning:
+ * timeouts while installing the runtime, then a 500 on wireguard-tools. pacman tries the next server in the
+ * list for each file it could not fetch, so a list of several turns one mirror's bad minute into a slower
+ * download instead of a failed build.
+ *
+ * A fixed list rather than ranking at build time: ranking (rankmirrors, reflector) needs a package installed
+ * first, from the one mirror that is the problem, and gives a different list on every build. These are
+ * Arch's own geo-routed mirror first, then well-kept European mirrors near the lab's machine, then the
+ * stock image's own. Each answered for core.db on 2026-10-14. A mirror that goes away costs a fallback,
+ * not a build, and is replaced here.
+ */
+export const MIRRORS = [
+  "https://geo.mirror.pkgbuild.com/$repo/os/$arch",
+  "https://mirror.netcologne.de/archlinux/$repo/os/$arch",
+  "https://ftp.fau.de/archlinux/$repo/os/$arch",
+  "https://archlinux.mirror.liteserver.nl/$repo/os/$arch",
+  "https://mirror.leaseweb.net/archlinux/$repo/os/$arch",
+  "https://mirrors.kernel.org/archlinux/$repo/os/$arch",
+];
+
+/** The mirrorlist the builder is given: every mirror, in order. */
+export function mirrorlist(): string {
+  return "# Written by the mesh-lab base build (novox/hq issue 700): several mirrors, tried in order.\n" +
+    MIRRORS.map((m) => `Server = ${m}`).join("\n") + "\n";
+}
+
+/** The mirrors a failed pacman step names: "from <host>" in what it said, each once. */
+export function failedMirrors(said: string): string[] {
+  const hosts = [...said.matchAll(/\bfrom ([a-z0-9.-]+\.[a-z]{2,})\b/gi)].map((m) => m[1]!.toLowerCase());
+  return [...new Set(hosts)];
+}
+
+/** How long to wait before each retry of a failed pacman step: three retries, backing off. */
+export const PACMAN_BACKOFF_MS = [10_000, 30_000, 90_000];
+
+/** What a base build runs through, so a test can stand in for the hypervisor and the clock. */
+export interface BaseDeps {
+  incus: (args: string[], timeoutMs?: number) => Promise<unknown>;
+  incusOk: (args: string[], timeoutMs?: number) => Promise<string | null>;
+  succeeds: (args: string[], timeoutMs?: number) => Promise<boolean>;
+  sleep: (ms: number) => Promise<void>;
+}
+
+const realDeps: BaseDeps = {
+  incus, incusOk, succeeds, sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+};
+
+/**
+ * pacmanIn runs one pacman step in the builder, and **runs it again after a pause when it fails**, up to
+ * three more times, saying each time which mirror failed and what it said. Installing is idempotent — what is
+ * installed stays installed — so a retry repeats only what did not happen. The last failure is thrown, naming
+ * every mirror that failed along the way.
+ */
+export async function pacmanIn(name: string, args: string[], log: (m: string) => void, deps: BaseDeps = realDeps):
+  Promise<void> {
+  const failed: string[] = [];
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await deps.incus(["exec", name, "--", "pacman", ...args], 600_000);
+      return;
+    } catch (err) {
+      // What pacman said, where incus kept it apart; otherwise the whole message.
+      const said = (err as { stderr?: string }).stderr?.trim() || (err as Error).message;
+      const mirrors = failedMirrors(said);
+      for (const m of mirrors) if (!failed.includes(m)) failed.push(m);
+      const where = mirrors.length > 0 ? `from ${mirrors.join(", ")}` : "naming no mirror";
+      const first = said.split("\n").find((l) => /error/i.test(l))?.trim() ?? said.split("\n")[0]?.trim() ?? "";
+      if (attempt >= PACMAN_BACKOFF_MS.length) {
+        throw new BaseImageError(`pacman ${args.join(" ")} failed ${attempt + 1} times; the mirrors that failed: ` +
+          `${failed.join(", ") || "none named"}. Last: ${said}`);
+      }
+      const wait = PACMAN_BACKOFF_MS[attempt]!;
+      log(`  pacman ${args.join(" ")} failed ${where} (${first}); trying again in ${wait / 1000}s ` +
+        `(attempt ${attempt + 2} of ${PACMAN_BACKOFF_MS.length + 1})`);
+      await deps.sleep(wait);
+    }
+  }
+}
+
 export class BaseImageError extends Error {
   constructor(message: string) {
     super(message);
@@ -38,7 +120,9 @@ export class BaseImageError extends Error {
  */
 export async function buildBaseImage(
   log: (message: string) => void = () => {},
+  deps: BaseDeps = realDeps,
 ): Promise<{ alias: string; runtime: string }> {
+  const { incus, incusOk, succeeds } = deps;
   await succeeds(["delete", "-f", BUILDER], 120_000);
 
   log(`  launching ${BUILDER} from ${UPSTREAM_IMAGE}, with a network`);
@@ -50,10 +134,19 @@ export async function buildBaseImage(
   ], 300_000);
 
   try {
-    await waitForAgent(BUILDER);
+    await waitForAgent(BUILDER, deps);
+
+    // Several mirrors before anything is fetched (MIRRORS), and read back: a list that was not written is
+    // the one mirror again, and the failure would look like the one this prevents.
+    log(`  giving pacman ${MIRRORS.length} mirrors`);
+    await incus(["exec", BUILDER, "--", "sh", "-c", `printf '%s' '${mirrorlist()}' > /etc/pacman.d/mirrorlist`], 60_000);
+    const written = await incusOk(["exec", BUILDER, "--", "cat", "/etc/pacman.d/mirrorlist"], 60_000);
+    if (written?.trim() !== mirrorlist().trim()) {
+      throw new BaseImageError(`the mirrorlist written into ${BUILDER} reads back as ${JSON.stringify(written)}`);
+    }
 
     log("  installing a container runtime");
-    await incus(["exec", BUILDER, "--", "pacman", "-Sy", "--noconfirm", "docker"], 600_000);
+    await pacmanIn(BUILDER, ["-Sy", "--noconfirm", "docker"], log, deps);
 
     // And the tools for the private network, for the same reason as the runtime: a sealed
     // scenario cannot install them, so a lab that omits them cannot test connectivity at all —
@@ -63,27 +156,27 @@ export async function buildBaseImage(
     // declaration; a lab that brought the interface up itself would be testing its own setup
     // rather than the mesh's.
     log("  installing the tools for the private network");
-    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "wireguard-tools"], 600_000);
+    await pacmanIn(BUILDER, ["-S", "--noconfirm", "wireguard-tools"], log, deps);
 
     // And git, for the same reason again: a machine that builds modules clones them, and a sealed
     // scenario cannot install it. On a real build machine the mesh installs it as a package like
     // anything else — the lab is the special case, because its machines reach no mirror.
     log("  installing git, so a machine can build modules");
-    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "git"], 600_000);
+    await pacmanIn(BUILDER, ["-S", "--noconfirm", "git"], log, deps);
 
     // And nftables, because the mesh computes a machine's filtering and delivers it as a file
     // that a service reflects — and neither the file nor the service can install what loads it.
     // Installed and NOT enabled: whether a machine filters is the mesh's decision, and a lab that
     // turned it on itself would be testing its own setup.
     log("  installing nftables, so a machine can enforce what the mesh computed");
-    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "nftables"], 600_000);
+    await pacmanIn(BUILDER, ["-S", "--noconfirm", "nftables"], log, deps);
 
     // And dnsmasq, because a service is named under the machine it runs on — postgres.novox.internal
     // — and only a resolver can answer a name the mesh was never told about. Installed and NOT
     // started: whether a machine resolves for the mesh is the mesh's decision, and a lab that
     // turned it on itself would be testing its own setup.
     log("  installing dnsmasq, so a machine can answer names under another machine");
-    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "dnsmasq"], 600_000);
+    await pacmanIn(BUILDER, ["-S", "--noconfirm", "dnsmasq"], log, deps);
 
     // Trust the documentation ranges as plain-HTTP registries.
     //
@@ -216,10 +309,10 @@ export async function baseImageExists(): Promise<boolean> {
  * Wait for the guest agent, because `launch` returning means the VM started, not that anything
  * inside it will answer.
  */
-async function waitForAgent(name: string): Promise<void> {
+async function waitForAgent(name: string, deps: BaseDeps = realDeps): Promise<void> {
   for (let i = 0; i < 90; i++) {
-    if (await succeeds(["exec", name, "--", "true"], 10_000)) return;
-    await new Promise((r) => setTimeout(r, 2_000));
+    if (await deps.succeeds(["exec", name, "--", "true"], 10_000)) return;
+    await deps.sleep(2_000);
   }
   throw new BaseImageError(`${name} started and its agent never answered.`);
 }