src/lifecycle/raise.ts

@@ -24,7 +24,7 @@ import { planRouters, raiseRouters, raiseTransit } from "./router.ts";
 import { applyHostFirewalls } from "./firewall.ts";
 import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts";
 import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts";
-import { discardStock, raiseRegistry, stockRegistry } from "./registry.ts";
+import { confirmRegistryServes, discardStock, raiseRegistry, stockRegistry } from "./registry.ts";
 import { log as record } from "../log.ts";
 
 /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */
@@ -340,6 +340,21 @@ export async function raise(
     enter("applying host firewalls");
     await applyHostFirewalls(scenario, byMachine, log);
 
+    // **Only now can "the registry is serving" be said truthfully.** Raising it proved the
+    // registry answers on its own machine; a machine pulls across a segment, and the home nodes
+    // pull through a gateway whose route and firewall were applied in the two steps above. So the
+    // path is checked here, where it is finally the one a pull will take — and before `placing`,
+    // which is minutes of work that a machine unable to fetch an image cannot use.
+    //
+    // The alternative is what happened: `raise` returned, the caller applied a substrate whose
+    // every image is pinned to this registry, the first pull failed, no node enrolled, and the
+    // instance was left a bare shell. A raise that reports success owes the next step the fact it
+    // depends on.
+    if (registry) {
+      enter("confirming the registry serves the machines");
+      await confirmRegistryServes(registry, [...byMachine.values()], stock, log);
+    }
+
     // Last, and only once the underlay is real. Placing before the machines can reach each
     // other would test the host against a network the scenario does not describe.
     enter("placing");

src/lifecycle/registry.ts

@@ -412,6 +412,72 @@ export async function raiseRegistry(
   return { machine: name, segment: segment.name, address, pinned };
 }
 
+/**
+ * Confirm the registry serves the MACHINES, not just itself.
+ *
+ * **`raiseRegistry` proves the wrong thing, and the difference cost a whole raise.** It curls
+ * `localhost:5000` from inside the registry's own machine — which says the registry process is up
+ * and holds the blobs, and says nothing at all about the path every other machine actually uses:
+ * across a segment, and for the home nodes through a NAT gateway whose route is applied two steps
+ * LATER. So `raise` could return "serving" with the anchor unable to reach the registry at all, the
+ * substrate apply's first pull would fail, no node could enrol, and the instance was left a bare
+ * shell — VMs and a registry and nothing else.
+ *
+ * A raise is not finished while that is still possible. This is the check that makes "the registry
+ * is serving" mean what the next step needs it to mean: from each machine, on the address it will
+ * pin, over the network it will use, once its route and its firewall are in place.
+ *
+ * **What it proves and what it does not.** It asks for `/v2/` and then for one stocked manifest BY
+ * DIGEST — the same two requests a pull begins with, from the same place. It does not fetch layers:
+ * every digest was already read back inside the registry machine, so what is in question here is
+ * the path, not the content, and a probe per machine keeps the check to seconds rather than the
+ * many minutes a full pull of a hundred images would take.
+ */
+export async function confirmRegistryServes(
+  registry: RaisedRegistry,
+  machines: string[],
+  stock: Stock,
+  log: (message: string) => void = () => {},
+  waitSeconds = 180,
+): Promise<void> {
+  const probe = stock.images[0];
+  if (!probe) return;
+  const base = `http://${registry.address}:${REGISTRY_PORT}`;
+  const manifest =
+    `-H "Accept: application/vnd.docker.distribution.manifest.v2+json" ` +
+    `${base}/v2/${probe.repository}/manifests/${probe.digest}`;
+
+  for (const machine of machines) {
+    const deadline = Date.now() + waitSeconds * 1_000;
+    let last = "";
+    let served = false;
+    while (!served && Date.now() < deadline) {
+      // One shell, two requests: the machine can reach the registry AND the registry answers for
+      // an image by the digest a declaration pins. Either alone passes on a registry serving
+      // nothing, which is the failure this whole function exists to stop reporting as success.
+      const said = await incusOk(["exec", machine, "--", "sh", "-c",
+        `printf '%s %s' ` +
+        `"$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 ${base}/v2/)" ` +
+        `"$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 ${manifest})"`,
+      ], 40_000);
+      last = said?.trim() ?? "";
+      served = last === "200 200";
+      if (!served) await new Promise((r) => setTimeout(r, 3_000));
+    }
+    if (!served) {
+      throw new RegistryError(
+        `${machine} cannot pull from the registry at ${registry.address}:${REGISTRY_PORT} after ` +
+          `${waitSeconds}s (it got "${last || "nothing"}" for /v2/ and for ` +
+          `${probe.repository}@${probe.digest}).\n` +
+          `  The registry answers on its own machine, so this is the PATH: this machine's route, ` +
+          `its gateway, or its firewall. Every image this scenario declares is unreachable from ` +
+          `here, so anything applied to it would fail on its first pull.`,
+      );
+    }
+    log(`  ${machine} can pull from ${registry.address}:${REGISTRY_PORT}`);
+  }
+}
+
 async function waitForAgent(name: string): Promise<void> {
   for (let i = 0; i < 90; i++) {
     if (await succeeds(["exec", name, "--", "true"], 10_000)) return;