mesh-lab/src/lifecycle/egress.ts

/**
 * Confirming a machine that says it can reach the outside actually can.
 *
 * **This is what the registry-reachability check became.** The old one proved that every machine
 * could fetch a manifest from the registry the lab raised inside the scenario — a real check of a
 * fake path, since no production mesh has such a registry. What a machine actually does is pull
 * from the internet, and that is now the thing worth proving before a raise says it is finished.
 *
 * The failure it exists to stop is the same one, in the same shape: `raise` returns, the caller
 * applies a substrate, the first pull fails, no node enrols, and the instance is left a bare
 * shell — with the cause several steps back and looking like a mesh fault rather than a lab one.
 *
 * Two things are checked, in this order, because they fail differently and the difference is the
 * whole diagnosis:
 *
 *   - **A name resolves.** Without this the machine has a route and no way to use it, and every
 *     pull dies inside the runtime saying it cannot look up a host.
 *   - **The path carries.** A request to the registry every image ultimately comes from, over the
 *     uplink, through whatever gateway sits in front of this machine. Any HTTP answer counts: what
 *     is in question is the path, not whether Docker Hub likes us.
 */

import type { Scenario } from "../declaration/types.ts";
import { incus, incusOk } from "../incus/client.ts";

export class EgressError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "EgressError";
  }
}

/** The host every image is fetched through, in the end. Asked for, never pulled from, here. */
const UPSTREAM = "registry-1.docker.io";

/**
 * Confirm every machine declaring `egress` can resolve and reach the outside.
 *
 * Run after the routes and the firewalls, because that is the path a pull will take: a home node's
 * default is the uplink, its route to the rest of the scenario is through its gateway, and its own
 * filtering is in place. Checking earlier would prove something no pull relies on.
 */
export async function confirmEgress(
  scenario: Scenario,
  machineNames: Map<string, string>,
  log: (message: string) => void = () => {},
  waitSeconds = 120,
): Promise<void> {
  for (const [machine, spec] of Object.entries(scenario.machines)) {
    if (!spec.egress || spec.at === "detached") continue;
    const name = machineNames.get(machine);
    if (!name) continue;

    if (!(await resolves(name, waitSeconds))) {
      // One repair, then a verdict. The uplink is the lab's own network and its DHCP server is
      // also its resolver, so the machine has been told the answer and may simply have nowhere
      // to write it — an image without systemd-resolved leaves `UseDNS=yes` inert.
      await pointResolverAtTheUplink(name);
      if (!(await resolves(name, 30))) {
        throw new EgressError(
          `${machine} declares egress and cannot resolve ${UPSTREAM}.\n` +
            `  It has a route out and no way to use it, so every image pulled from the internet ` +
            `would fail inside the runtime as a lookup error.\n` +
            `  The uplink's DHCP server is also its resolver; this machine has not taken it.`,
        );
      }
    }

    const code = await reaches(name, waitSeconds);
    if (!code) {
      throw new EgressError(
        `${machine} declares egress, resolves names, and cannot reach ${UPSTREAM}.\n` +
          `  This is the PATH: its default route, the uplink, or the host's own forwarding. ` +
          `Every third-party image this machine needs is pulled from the internet, so anything ` +
          `applied to it would stop at the first container.`,
      );
    }
    log(`  ${machine} reaches the internet over its uplink (${UPSTREAM} answered ${code})`);
  }
}

async function resolves(name: string, waitSeconds: number): Promise<boolean> {
  const deadline = Date.now() + waitSeconds * 1_000;
  while (Date.now() < deadline) {
    const said = await incusOk(
      ["exec", name, "--", "sh", "-c", `getent hosts ${UPSTREAM} >/dev/null && echo yes`], 30_000,
    );
    if (said?.trim() === "yes") return true;
    await new Promise((r) => setTimeout(r, 3_000));
  }
  return false;
}

/**
 * Any HTTP status at all, which is what "the path carries" means.
 *
 * Not 200: an unauthenticated `/v2/` is answered 401 by design, and a check demanding 200 would
 * fail on a machine whose network is perfect.
 */
async function reaches(name: string, waitSeconds: number): Promise<string | null> {
  const deadline = Date.now() + waitSeconds * 1_000;
  while (Date.now() < deadline) {
    const said = (await incusOk(
      ["exec", name, "--", "sh", "-c",
       `curl -s -o /dev/null -w '%{http_code}' --max-time 15 https://${UPSTREAM}/v2/`], 40_000,
    ))?.trim();
    if (said && /^[1-5][0-9]{2}$/.test(said)) return said;
    await new Promise((r) => setTimeout(r, 5_000));
  }
  return null;
}

/**
 * Write a resolver of last resort: the uplink's own gateway, which serves DHCP and DNS both.
 *
 * Deliberately the machine's default next hop rather than a name looked up somewhere — for a
 * machine with egress that is the uplink by construction, since every scenario range is routed
 * explicitly and nothing else defaults.
 */
async function pointResolverAtTheUplink(name: string): Promise<void> {
  await incus([
    "exec", name, "--", "sh", "-c",
    `via=$(ip -4 route show default | awk '{print $3}' | head -n1); ` +
      `[ -n "$via" ] && printf 'nameserver %s\\n' "$via" > /etc/resolv.conf; true`,
  ], 30_000);
}