Give the base build several mirrors and retry a failed pacman step (issue 700)

The stock Arch image names one mirror, and the base build failed twice in
a morning when mirrors.kernel.org timed out and then answered 500. Write a
fixed list of six mirrors before anything is installed, so pacman falls
back per file, and retry a failed step three times with a growing pause,
saying which mirror failed.

src/lifecycle/base.ts

@@ -23,6 +23,88 @@ export const UPSTREAM_IMAGE = "images:archlinux/current";
 
 const BUILDER = "mesh-lab-base-builder";
 
+/**
+ * **The mirrors the builder installs from, several and in order** (novox/hq issue 700).
+ *
+ * The stock image names one mirror, mirrors.kernel.org, and a base build failed twice on it in one morning:
+ * timeouts while installing the runtime, then a 500 on wireguard-tools. pacman tries the next server in the
+ * list for each file it could not fetch, so a list of several turns one mirror's bad minute into a slower
+ * download instead of a failed build.
+ *
+ * A fixed list rather than ranking at build time: ranking (rankmirrors, reflector) needs a package installed
+ * first, from the one mirror that is the problem, and gives a different list on every build. These are
+ * Arch's own geo-routed mirror first, then well-kept European mirrors near the lab's machine, then the
+ * stock image's own. Each answered for core.db on 2026-10-14. A mirror that goes away costs a fallback,
+ * not a build, and is replaced here.
+ */
+export const MIRRORS = [
+  "https://geo.mirror.pkgbuild.com/$repo/os/$arch",
+  "https://mirror.netcologne.de/archlinux/$repo/os/$arch",
+  "https://ftp.fau.de/archlinux/$repo/os/$arch",
+  "https://archlinux.mirror.liteserver.nl/$repo/os/$arch",
+  "https://mirror.leaseweb.net/archlinux/$repo/os/$arch",
+  "https://mirrors.kernel.org/archlinux/$repo/os/$arch",
+];
+
+/** The mirrorlist the builder is given: every mirror, in order. */
+export function mirrorlist(): string {
+  return "# Written by the mesh-lab base build (novox/hq issue 700): several mirrors, tried in order.\n" +
+    MIRRORS.map((m) => `Server = ${m}`).join("\n") + "\n";
+}
+
+/** The mirrors a failed pacman step names: "from <host>" in what it said, each once. */
+export function failedMirrors(said: string): string[] {
+  const hosts = [...said.matchAll(/\bfrom ([a-z0-9.-]+\.[a-z]{2,})\b/gi)].map((m) => m[1]!.toLowerCase());
+  return [...new Set(hosts)];
+}
+
+/** How long to wait before each retry of a failed pacman step: three retries, backing off. */
+export const PACMAN_BACKOFF_MS = [10_000, 30_000, 90_000];
+
+/** What a base build runs through, so a test can stand in for the hypervisor and the clock. */
+export interface BaseDeps {
+  incus: (args: string[], timeoutMs?: number) => Promise<unknown>;
+  incusOk: (args: string[], timeoutMs?: number) => Promise<string | null>;
+  succeeds: (args: string[], timeoutMs?: number) => Promise<boolean>;
+  sleep: (ms: number) => Promise<void>;
+}
+
+const realDeps: BaseDeps = {
+  incus, incusOk, succeeds, sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+};
+
+/**
+ * pacmanIn runs one pacman step in the builder, and **runs it again after a pause when it fails**, up to
+ * three more times, saying each time which mirror failed and what it said. Installing is idempotent — what is
+ * installed stays installed — so a retry repeats only what did not happen. The last failure is thrown, naming
+ * every mirror that failed along the way.
+ */
+export async function pacmanIn(name: string, args: string[], log: (m: string) => void, deps: BaseDeps = realDeps):
+  Promise<void> {
+  const failed: string[] = [];
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await deps.incus(["exec", name, "--", "pacman", ...args], 600_000);
+      return;
+    } catch (err) {
+      // What pacman said, where incus kept it apart; otherwise the whole message.
+      const said = (err as { stderr?: string }).stderr?.trim() || (err as Error).message;
+      const mirrors = failedMirrors(said);
+      for (const m of mirrors) if (!failed.includes(m)) failed.push(m);
+      const where = mirrors.length > 0 ? `from ${mirrors.join(", ")}` : "naming no mirror";
+      const first = said.split("\n").find((l) => /error/i.test(l))?.trim() ?? said.split("\n")[0]?.trim() ?? "";
+      if (attempt >= PACMAN_BACKOFF_MS.length) {
+        throw new BaseImageError(`pacman ${args.join(" ")} failed ${attempt + 1} times; the mirrors that failed: ` +
+          `${failed.join(", ") || "none named"}. Last: ${said}`);
+      }
+      const wait = PACMAN_BACKOFF_MS[attempt]!;
+      log(`  pacman ${args.join(" ")} failed ${where} (${first}); trying again in ${wait / 1000}s ` +
+        `(attempt ${attempt + 2} of ${PACMAN_BACKOFF_MS.length + 1})`);
+      await deps.sleep(wait);
+    }
+  }
+}
+
 export class BaseImageError extends Error {
   constructor(message: string) {
     super(message);
@@ -38,7 +120,9 @@ export class BaseImageError extends Error {
  */
 export async function buildBaseImage(
   log: (message: string) => void = () => {},
+  deps: BaseDeps = realDeps,
 ): Promise<{ alias: string; runtime: string }> {
+  const { incus, incusOk, succeeds } = deps;
   await succeeds(["delete", "-f", BUILDER], 120_000);
 
   log(`  launching ${BUILDER} from ${UPSTREAM_IMAGE}, with a network`);
@@ -50,10 +134,19 @@ export async function buildBaseImage(
   ], 300_000);
 
   try {
-    await waitForAgent(BUILDER);
+    await waitForAgent(BUILDER, deps);
+
+    // Several mirrors before anything is fetched (MIRRORS), and read back: a list that was not written is
+    // the one mirror again, and the failure would look like the one this prevents.
+    log(`  giving pacman ${MIRRORS.length} mirrors`);
+    await incus(["exec", BUILDER, "--", "sh", "-c", `printf '%s' '${mirrorlist()}' > /etc/pacman.d/mirrorlist`], 60_000);
+    const written = await incusOk(["exec", BUILDER, "--", "cat", "/etc/pacman.d/mirrorlist"], 60_000);
+    if (written?.trim() !== mirrorlist().trim()) {
+      throw new BaseImageError(`the mirrorlist written into ${BUILDER} reads back as ${JSON.stringify(written)}`);
+    }
 
     log("  installing a container runtime");
-    await incus(["exec", BUILDER, "--", "pacman", "-Sy", "--noconfirm", "docker"], 600_000);
+    await pacmanIn(BUILDER, ["-Sy", "--noconfirm", "docker"], log, deps);
 
     // And the tools for the private network, for the same reason as the runtime: a sealed
     // scenario cannot install them, so a lab that omits them cannot test connectivity at all —
@@ -63,27 +156,27 @@ export async function buildBaseImage(
     // declaration; a lab that brought the interface up itself would be testing its own setup
     // rather than the mesh's.
     log("  installing the tools for the private network");
-    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "wireguard-tools"], 600_000);
+    await pacmanIn(BUILDER, ["-S", "--noconfirm", "wireguard-tools"], log, deps);
 
     // And git, for the same reason again: a machine that builds modules clones them, and a sealed
     // scenario cannot install it. On a real build machine the mesh installs it as a package like
     // anything else — the lab is the special case, because its machines reach no mirror.
     log("  installing git, so a machine can build modules");
-    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "git"], 600_000);
+    await pacmanIn(BUILDER, ["-S", "--noconfirm", "git"], log, deps);
 
     // And nftables, because the mesh computes a machine's filtering and delivers it as a file
     // that a service reflects — and neither the file nor the service can install what loads it.
     // Installed and NOT enabled: whether a machine filters is the mesh's decision, and a lab that
     // turned it on itself would be testing its own setup.
     log("  installing nftables, so a machine can enforce what the mesh computed");
-    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "nftables"], 600_000);
+    await pacmanIn(BUILDER, ["-S", "--noconfirm", "nftables"], log, deps);
 
     // And dnsmasq, because a service is named under the machine it runs on — postgres.novox.internal
     // — and only a resolver can answer a name the mesh was never told about. Installed and NOT
     // started: whether a machine resolves for the mesh is the mesh's decision, and a lab that
     // turned it on itself would be testing its own setup.
     log("  installing dnsmasq, so a machine can answer names under another machine");
-    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "dnsmasq"], 600_000);
+    await pacmanIn(BUILDER, ["-S", "--noconfirm", "dnsmasq"], log, deps);
 
     // Trust the documentation ranges as plain-HTTP registries.
     //
@@ -216,10 +309,10 @@ export async function baseImageExists(): Promise<boolean> {
  * Wait for the guest agent, because `launch` returning means the VM started, not that anything
  * inside it will answer.
  */
-async function waitForAgent(name: string): Promise<void> {
+async function waitForAgent(name: string, deps: BaseDeps = realDeps): Promise<void> {
   for (let i = 0; i < 90; i++) {
-    if (await succeeds(["exec", name, "--", "true"], 10_000)) return;
+    if (await deps.succeeds(["exec", name, "--", "true"], 10_000)) return;
-    await new Promise((r) => setTimeout(r, 2_000));
+    await deps.sleep(2_000);
   }
   throw new BaseImageError(`${name} started and its agent never answered.`);
 }

test/base-mirrors.test.ts

@@ -0,0 +1,100 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { IncusError } from "../src/incus/client.ts";
+import {
+  buildBaseImage, failedMirrors, MIRRORS, mirrorlist, PACMAN_BACKOFF_MS, pacmanIn, type BaseDeps,
+} from "../src/lifecycle/base.ts";
+import { BASE_IMAGE_ALIAS } from "../src/lifecycle/place.ts";
+
+/**
+ * Defends novox/hq issue 700: the base build installed from the one mirror the stock image names,
+ * mirrors.kernel.org, and failed twice in a morning when that mirror timed out and then answered 500.
+ */
+
+const KERNEL_500 = "error: failed retrieving file 'wireguard-tools-1.0.20260223-1-x86_64.pkg.tar.zst' from " +
+  "mirrors.kernel.org : The requested URL returned error: 500\nwarning: failed to retrieve some files\n" +
+  "error: failed to commit transaction (failed to retrieve some files)";
+
+/** A hypervisor for one base build: pacman fails `fails` times per step, then installs. */
+function aBuilder(fails: number, said = KERNEL_500) {
+  const ran: string[] = [];
+  const slept: number[] = [];
+  let mirrorlistWritten = "";
+  const tries = new Map<string, number>();
+  const deps: BaseDeps = {
+    incus: async (args) => {
+      ran.push(args.join(" "));
+      const at = args.indexOf("pacman");
+      if (at >= 0) {
+        const step = args.slice(at).join(" ");
+        const n = (tries.get(step) ?? 0) + 1;
+        tries.set(step, n);
+        if (n <= fails) throw new IncusError(args, said, 1);
+      }
+      const printf = args.join(" ").match(/printf '%s' '([\s\S]*)' > \/etc\/pacman.d\/mirrorlist/);
+      if (printf) mirrorlistWritten = printf[1]!;
+      return { stdout: "", stderr: "" };
+    },
+    incusOk: async (args) => {
+      const c = args.join(" ");
+      if (c.endsWith("cat /etc/pacman.d/mirrorlist")) return mirrorlistWritten;
+      if (c.includes("{{.ServerVersion}}")) return "29.8.2";
+      if (c.includes("InsecureRegistryCIDRs")) return "[127.0.0.0/8 192.0.2.0/24]";
+      if (c.startsWith("image list")) return BASE_IMAGE_ALIAS;
+      return "tool 1.0";
+    },
+    succeeds: async () => true,
+    sleep: async (ms) => { slept.push(ms); },
+  };
+  return { deps, ran, slept };
+}
+
+test("the builder is given several mirrors, the stock one last, before anything is installed", async () => {
+  const hosts = MIRRORS.map((m) => new URL(m.replace("$repo", "core").replace("$arch", "x86_64")).host);
+  assert.ok(new Set(hosts).size >= 4, `too few mirrors: ${hosts.join(", ")}`);
+  assert.equal(hosts.at(-1), "mirrors.kernel.org");
+  assert.ok(!mirrorlist().includes("'"), "the mirrorlist is written inside single quotes and holds one");
+  assert.equal(mirrorlist().split("\n").filter((l) => l.startsWith("Server = ")).length, MIRRORS.length);
+
+  const b = aBuilder(0);
+  await buildBaseImage(() => {}, b.deps);
+  const wrote = b.ran.findIndex((c) => c.includes("/etc/pacman.d/mirrorlist"));
+  const firstPacman = b.ran.findIndex((c) => c.includes(" pacman "));
+  assert.ok(wrote >= 0 && wrote < firstPacman, "pacman ran before the mirrorlist was written");
+});
+
+test("a pacman step that fails is tried again after a pause, saying which mirror failed", async () => {
+  const said: string[] = [];
+  const b = aBuilder(2);
+  await pacmanIn("builder", ["-S", "--noconfirm", "wireguard-tools"], (m) => said.push(m), b.deps);
+  assert.equal(b.ran.length, 3);
+  assert.deepEqual(b.slept, PACMAN_BACKOFF_MS.slice(0, 2));
+  assert.ok(PACMAN_BACKOFF_MS[1]! > PACMAN_BACKOFF_MS[0]!, "no backoff");
+  assert.equal(said.length, 2);
+  assert.match(said[0]!, /failed from mirrors\.kernel\.org \(error: failed retrieving file 'wireguard-tools/);
+  assert.match(said[0]!, /trying again in 10s \(attempt 2 of 4\)/);
+});
+
+test("a step that keeps failing stops the build, naming every mirror that failed", async () => {
+  const b = aBuilder(99, "error: too many errors from mirrors.kernel.org\nerror: failed retrieving file 'x' from ftp.fau.de : 500");
+  await assert.rejects(
+    pacmanIn("builder", ["-Sy", "--noconfirm", "docker"], () => {}, b.deps),
+    /failed 4 times; the mirrors that failed: mirrors\.kernel\.org, ftp\.fau\.de/,
+  );
+  assert.equal(b.ran.length, PACMAN_BACKOFF_MS.length + 1);
+});
+
+test("a whole base build rides out a mirror's bad minute on every package", async () => {
+  const b = aBuilder(1);
+  const built = await buildBaseImage(() => {}, b.deps);
+  assert.equal(built.alias, BASE_IMAGE_ALIAS);
+  const pacman = b.ran.filter((c) => c.includes(" pacman "));
+  assert.equal(pacman.length, 10, pacman.join("\n"));
+});
+
+test("the mirrors a failure names are read from what pacman said, each once", () => {
+  assert.deepEqual(failedMirrors(KERNEL_500), ["mirrors.kernel.org"]);
+  assert.deepEqual(failedMirrors("Connection timed out after 10000 milliseconds\nerror: too many errors from mirrors.kernel.org"),
+    ["mirrors.kernel.org"]);
+  assert.deepEqual(failedMirrors("error: target not found: dockr"), []);
+});