mesh-lab/test/base-mirrors.test.ts

import { test } from "node:test";
import assert from "node:assert/strict";
import { IncusError } from "../src/incus/client.ts";
import {
  buildBaseImage, failedMirrors, MIRRORS, mirrorlist, PACMAN_BACKOFF_MS, pacmanIn, type BaseDeps,
} from "../src/lifecycle/base.ts";
import { BASE_IMAGE_ALIAS } from "../src/lifecycle/place.ts";

/**
 * Defends novox/hq issue 700: the base build installed from the one mirror the stock image names,
 * mirrors.kernel.org, and failed twice in a morning when that mirror timed out and then answered 500.
 */

const KERNEL_500 = "error: failed retrieving file 'wireguard-tools-1.0.20260223-1-x86_64.pkg.tar.zst' from " +
  "mirrors.kernel.org : The requested URL returned error: 500\nwarning: failed to retrieve some files\n" +
  "error: failed to commit transaction (failed to retrieve some files)";

/** A hypervisor for one base build: pacman fails `fails` times per step, then installs. */
function aBuilder(fails: number, said = KERNEL_500) {
  const ran: string[] = [];
  const slept: number[] = [];
  let mirrorlistWritten = "";
  const tries = new Map<string, number>();
  const deps: BaseDeps = {
    incus: async (args) => {
      ran.push(args.join(" "));
      const at = args.indexOf("pacman");
      if (at >= 0) {
        const step = args.slice(at).join(" ");
        const n = (tries.get(step) ?? 0) + 1;
        tries.set(step, n);
        if (n <= fails) throw new IncusError(args, said, 1);
      }
      const printf = args.join(" ").match(/printf '%s' '([\s\S]*)' > \/etc\/pacman.d\/mirrorlist/);
      if (printf) mirrorlistWritten = printf[1]!;
      return { stdout: "", stderr: "" };
    },
    incusOk: async (args) => {
      const c = args.join(" ");
      if (c.endsWith("cat /etc/pacman.d/mirrorlist")) return mirrorlistWritten;
      if (c.includes("{{.ServerVersion}}")) return "29.8.2";
      if (c.includes("InsecureRegistryCIDRs")) return "[127.0.0.0/8 192.0.2.0/24]";
      if (c.startsWith("image list")) return BASE_IMAGE_ALIAS;
      return "tool 1.0";
    },
    succeeds: async () => true,
    sleep: async (ms) => { slept.push(ms); },
  };
  return { deps, ran, slept };
}

test("the builder is given several mirrors, the stock one last, before anything is installed", async () => {
  const hosts = MIRRORS.map((m) => new URL(m.replace("$repo", "core").replace("$arch", "x86_64")).host);
  assert.ok(new Set(hosts).size >= 4, `too few mirrors: ${hosts.join(", ")}`);
  assert.equal(hosts.at(-1), "mirrors.kernel.org");
  assert.ok(!mirrorlist().includes("'"), "the mirrorlist is written inside single quotes and holds one");
  assert.equal(mirrorlist().split("\n").filter((l) => l.startsWith("Server = ")).length, MIRRORS.length);

  const b = aBuilder(0);
  await buildBaseImage(() => {}, b.deps);
  const wrote = b.ran.findIndex((c) => c.includes("/etc/pacman.d/mirrorlist"));
  const firstPacman = b.ran.findIndex((c) => c.includes(" pacman "));
  assert.ok(wrote >= 0 && wrote < firstPacman, "pacman ran before the mirrorlist was written");
});

test("a pacman step that fails is tried again after a pause, saying which mirror failed", async () => {
  const said: string[] = [];
  const b = aBuilder(2);
  await pacmanIn("builder", ["-S", "--noconfirm", "wireguard-tools"], (m) => said.push(m), b.deps);
  assert.equal(b.ran.length, 3);
  assert.deepEqual(b.slept, PACMAN_BACKOFF_MS.slice(0, 2));
  assert.ok(PACMAN_BACKOFF_MS[1]! > PACMAN_BACKOFF_MS[0]!, "no backoff");
  assert.equal(said.length, 2);
  assert.match(said[0]!, /failed from mirrors\.kernel\.org \(error: failed retrieving file 'wireguard-tools/);
  assert.match(said[0]!, /trying again in 10s \(attempt 2 of 4\)/);
});

test("a step that keeps failing stops the build, naming every mirror that failed", async () => {
  const b = aBuilder(99, "error: too many errors from mirrors.kernel.org\nerror: failed retrieving file 'x' from ftp.fau.de : 500");
  await assert.rejects(
    pacmanIn("builder", ["-Sy", "--noconfirm", "docker"], () => {}, b.deps),
    /failed 4 times; the mirrors that failed: mirrors\.kernel\.org, ftp\.fau\.de/,
  );
  assert.equal(b.ran.length, PACMAN_BACKOFF_MS.length + 1);
});

test("a whole base build rides out a mirror's bad minute on every package", async () => {
  const b = aBuilder(1);
  const built = await buildBaseImage(() => {}, b.deps);
  assert.equal(built.alias, BASE_IMAGE_ALIAS);
  const pacman = b.ran.filter((c) => c.includes(" pacman "));
  assert.equal(pacman.length, 10, pacman.join("\n"));
});

test("the mirrors a failure names are read from what pacman said, each once", () => {
  assert.deepEqual(failedMirrors(KERNEL_500), ["mirrors.kernel.org"]);
  assert.deepEqual(failedMirrors("Connection timed out after 10000 milliseconds\nerror: too many errors from mirrors.kernel.org"),
    ["mirrors.kernel.org"]);
  assert.deepEqual(failedMirrors("error: target not found: dockr"), []);
});