diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index c26ead3..afe54e3 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -24,7 +24,7 @@ import { planRouters, raiseRouters, raiseTransit } from "./router.ts"; import { applyHostFirewalls } from "./firewall.ts"; import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts"; import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts"; -import { discardStock, raiseRegistry, stockRegistry } from "./registry.ts"; +import { confirmRegistryServes, discardStock, raiseRegistry, stockRegistry } from "./registry.ts"; import { log as record } from "../log.ts"; /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */ @@ -340,6 +340,21 @@ export async function raise( enter("applying host firewalls"); await applyHostFirewalls(scenario, byMachine, log); + // **Only now can "the registry is serving" be said truthfully.** Raising it proved the + // registry answers on its own machine; a machine pulls across a segment, and the home nodes + // pull through a gateway whose route and firewall were applied in the two steps above. So the + // path is checked here, where it is finally the one a pull will take — and before `placing`, + // which is minutes of work that a machine unable to fetch an image cannot use. + // + // The alternative is what happened: `raise` returned, the caller applied a substrate whose + // every image is pinned to this registry, the first pull failed, no node enrolled, and the + // instance was left a bare shell. A raise that reports success owes the next step the fact it + // depends on. + if (registry) { + enter("confirming the registry serves the machines"); + await confirmRegistryServes(registry, [...byMachine.values()], stock, log); + } + // Last, and only once the underlay is real. Placing before the machines can reach each // other would test the host against a network the scenario does not describe. enter("placing"); diff --git a/src/lifecycle/registry.ts b/src/lifecycle/registry.ts index 682c47b..6a6be7c 100644 --- a/src/lifecycle/registry.ts +++ b/src/lifecycle/registry.ts @@ -412,6 +412,72 @@ export async function raiseRegistry( return { machine: name, segment: segment.name, address, pinned }; } +/** + * Confirm the registry serves the MACHINES, not just itself. + * + * **`raiseRegistry` proves the wrong thing, and the difference cost a whole raise.** It curls + * `localhost:5000` from inside the registry's own machine — which says the registry process is up + * and holds the blobs, and says nothing at all about the path every other machine actually uses: + * across a segment, and for the home nodes through a NAT gateway whose route is applied two steps + * LATER. So `raise` could return "serving" with the anchor unable to reach the registry at all, the + * substrate apply's first pull would fail, no node could enrol, and the instance was left a bare + * shell — VMs and a registry and nothing else. + * + * A raise is not finished while that is still possible. This is the check that makes "the registry + * is serving" mean what the next step needs it to mean: from each machine, on the address it will + * pin, over the network it will use, once its route and its firewall are in place. + * + * **What it proves and what it does not.** It asks for `/v2/` and then for one stocked manifest BY + * DIGEST — the same two requests a pull begins with, from the same place. It does not fetch layers: + * every digest was already read back inside the registry machine, so what is in question here is + * the path, not the content, and a probe per machine keeps the check to seconds rather than the + * many minutes a full pull of a hundred images would take. + */ +export async function confirmRegistryServes( + registry: RaisedRegistry, + machines: string[], + stock: Stock, + log: (message: string) => void = () => {}, + waitSeconds = 180, +): Promise { + const probe = stock.images[0]; + if (!probe) return; + const base = `http://${registry.address}:${REGISTRY_PORT}`; + const manifest = + `-H "Accept: application/vnd.docker.distribution.manifest.v2+json" ` + + `${base}/v2/${probe.repository}/manifests/${probe.digest}`; + + for (const machine of machines) { + const deadline = Date.now() + waitSeconds * 1_000; + let last = ""; + let served = false; + while (!served && Date.now() < deadline) { + // One shell, two requests: the machine can reach the registry AND the registry answers for + // an image by the digest a declaration pins. Either alone passes on a registry serving + // nothing, which is the failure this whole function exists to stop reporting as success. + const said = await incusOk(["exec", machine, "--", "sh", "-c", + `printf '%s %s' ` + + `"$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 ${base}/v2/)" ` + + `"$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 ${manifest})"`, + ], 40_000); + last = said?.trim() ?? ""; + served = last === "200 200"; + if (!served) await new Promise((r) => setTimeout(r, 3_000)); + } + if (!served) { + throw new RegistryError( + `${machine} cannot pull from the registry at ${registry.address}:${REGISTRY_PORT} after ` + + `${waitSeconds}s (it got "${last || "nothing"}" for /v2/ and for ` + + `${probe.repository}@${probe.digest}).\n` + + ` The registry answers on its own machine, so this is the PATH: this machine's route, ` + + `its gateway, or its firewall. Every image this scenario declares is unreachable from ` + + `here, so anything applied to it would fail on its first pull.`, + ); + } + log(` ${machine} can pull from ${registry.address}:${REGISTRY_PORT}`); + } +} + async function waitForAgent(name: string): Promise { for (let i = 0; i < 90; i++) { if (await succeeds(["exec", name, "--", "true"], 10_000)) return;