README.md

@@ -144,6 +144,18 @@ export MESH_LAB_BUNDLE=<mesh-host>/examples/substrate-first-node.lock
 export MESH_LAB_MODULES=<mesh-control>/examples/modules
 export MESH_LAB_BUILDER=<mesh-control>/build/mesh-builder   # build/, which is git-ignored
 
+# The installer. `suite` builds it with mesh-host's `make bootstrap`, which embeds a `docker save`
+# of the control-plane image — so it is built AFTER that image, in the same run, or it carries a
+# stale one sealed inside a binary where nothing would ever notice. The whole-mesh bed raises its
+# anchor by RUNNING this, rather than by applying a substrate bundle itself (novox/hq ADR 0067).
+export MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
+export MESH_LAB_CONTROL_IMAGE=mesh-control:development   # optional; what it carries
+
+# A checkout of the mesh's catalogue. The installer reads the registry's and the control plane's
+# manifests from a copy of it ON THE MACHINE, because at genesis there is no forge, no build
+# machine and — until the registry is up — nothing serving anything.
+export MESH_LAB_CATALOG=<mesh-catalog>/modules
+
 # Built with `go build -o <path> ./examples/<name>` in mesh-control.
 export MESH_LAB_PROVISIONER=<somewhere>/postgres-provisioner
 export MESH_LAB_OBJECTSTORE_PROVISIONER=<somewhere>/objectstore-provisioner

scenarios/whole-mesh-full.yml

@@ -42,6 +42,15 @@
 # network on the container port, so the host side is free to move). Reported as a topology finding.
 #
 #   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
+#   MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap   MESH_LAB_CATALOG=.../mesh-catalog/modules
+#
+# GENESIS AND JOINING ARE TWO DIFFERENT ACTS, and this bed distinguishes them. novox is brought
+# into existence by `mesh-bootstrap` — the same program a bare machine runs — and is afterwards a
+# working mesh of one, with a registry and a control plane that is an ordinary module pinned to an
+# image that registry serves. ace, shanks and g14 then JOIN it: host binary, token, enrol, run. No
+# bootstrap, no substrate, no registry. novox is never enrolled twice, because the installer
+# already did it.
+#
 # The images: are the UNION of the novox set (feat/novox-conversions @ 431310f: the slug + roundcube
 # fixes, so only-office/de-spiegel/amqp-email-forwarder now resolve) and the ace media/home set, and
 # every one of them must already be BUILT on the workstation — nothing can fetch them.
@@ -76,12 +85,18 @@ machines:
     memory: 24GiB
     cpus: 8
     disk: 130GiB
-    # The substrate's control plane, the proxy, and a runtime per module novox is assigned.
+    # The proxy, and a runtime per module novox is assigned. step-ca, photos, invoicing, novox.be,
-    # step-ca, photos, invoicing, novox.be, only-office, de-spiegel, amqp-email-forwarder, registry,
+    # only-office, de-spiegel, amqp-email-forwarder, registry, firewall and fail2ban are not here
-    # firewall and fail2ban are not here because they carry no runtime image of their own — what
+    # because they carry no runtime image of their own — what they run is third-party or is the
-    # they run is third-party or is the node itself.
+    # node itself.
+    #
+    # **mesh-control is NOT here, and its absence is the point** (novox/hq ADR 0067). The anchor is
+    # brought into existence by the installer, and the installer carries the control plane's image
+    # inside itself — that is the whole reason a machine that can reach no registry can still raise
+    # a mesh. Handing it over from the workstation as well would mean the bed never found out
+    # whether the installer really carries it: the load would say "already held" and the fiction
+    # would be invisible, which is exactly the class of thing the lab's own registry used to hide.
     images:
-      - mesh-control:development
       - mesh-route-proxy:development
       - mesh-runtime-postgres:development
       - mesh-runtime-redis:development
@@ -164,7 +179,6 @@ machines:
 # a machine holds it because it was handed it. Everything else the modules run comes from the
 # internet and is not named here at all.
 images:
-  - mesh-control:development
   # --- per-module runtimes (union) ---
   - mesh-runtime-postgres:development
   - mesh-runtime-redis:development

src/lifecycle/operate.ts

@@ -61,11 +61,45 @@ export async function exec(
    */
   timeoutMs = 120_000,
 ): Promise<{ stdout: string; stderr: string }> {
+  const name = await instanceNameOf(instanceId, machine);
+  return incus(["exec", name, "--", ...command], timeoutMs);
+}
+
+/**
+ * What the hypervisor calls one of a scenario's machines.
+ *
+ * Asked of the daemon by the metadata each instance carries, and only derived from the naming
+ * rule when it answers nothing — the same order `exec` has always used. It is exported because
+ * the placement stage takes this name rather than the pair, and a caller that wants to put
+ * something on one machine after the raise (the installer, a catalogue checkout) would otherwise
+ * have to reimplement the lookup and get the fallback wrong.
+ */
+export async function instanceNameOf(instanceId: string, machine: string): Promise<string> {
   const found = (await taggedInstances()).find(
     (i) => i.instanceId === instanceId && i.machine === machine,
   );
-  const name = found?.name ?? machineName(instanceId, machine);
+  return found?.name ?? machineName(instanceId, machine);
-  return incus(["exec", name, "--", ...command], timeoutMs);
+}
+
+/**
+ * Put a file from this workstation inside a machine.
+ *
+ * The long default timeout is not generosity: what goes through here is an installer carrying a
+ * container image, which is tens of megabytes, and a push that is merely slow must not look like
+ * a push that is stuck.
+ */
+export async function push(
+  instanceId: string,
+  machine: string,
+  local: string,
+  remote: string,
+  mode?: string,
+  timeoutMs = 900_000,
+): Promise<void> {
+  const name = await instanceNameOf(instanceId, machine);
+  const args = ["file", "push", local, `${name}${remote}`];
+  if (mode) args.push("--mode", mode);
+  await incus(args, timeoutMs);
 }
 
 /**

src/lifecycle/place.ts

@@ -42,6 +42,17 @@ export function isPlaceable(artifact: string): boolean {
 /** Where the host binary lives on a machine once placed. */
 export const HOST_PATH = "/usr/local/bin/mesh-host";
 
+/**
+ * Where the installer lives on a machine once placed.
+ *
+ * **Beside the host, because it is the same tier and the same delivery** (novox/hq ADR 0067):
+ * bootstrapping is done by hand and it changes a machine, which is what tier 0 is — but `mesh-host`
+ * says of itself that it connects to nothing and listens on nothing, and an installer that loads
+ * images and interrogates a control plane cannot be folded into it without making that sentence
+ * false. Two programs, one shelf.
+ */
+export const BOOTSTRAP_PATH = "/usr/local/bin/mesh-bootstrap";
+
 export interface Placement {
   machine: string;
   artifacts: string[];
@@ -78,6 +89,14 @@ export function hostBinaryPath(): string | null {
   return process.env["MESH_LAB_HOST_BINARY"] ?? null;
 }
 
+/**
+ * The installer to place, from the environment. Same rule as the host binary: an explicit path,
+ * and nothing is guessed.
+ */
+export function bootstrapBinaryPath(): string | null {
+  return process.env["MESH_LAB_BOOTSTRAP_BINARY"] ?? null;
+}
+
 export class PlacementError extends Error {
   readonly machine: string;
 
@@ -146,6 +165,41 @@ export async function placeHost(
   return { machine, profile, version };
 }
 
+/**
+ * Put the installer on a machine and ask it what it is.
+ *
+ * The same shape as {@link placeHost} and for the same reason: the version is read back from the
+ * running binary, because a file arriving is not a program working (novox/hq ADR 0018). The
+ * installer is the larger of the two by an order of magnitude — it carries a saved container image
+ * — so a copy that half-arrived is a real possibility rather than a theoretical one.
+ *
+ * It is placed on ONE machine, not all of them. Only the anchor is bootstrapped; every other
+ * machine joins a mesh that already exists, with the host binary and a token and nothing else.
+ */
+export async function placeBootstrap(
+  instanceName: string,
+  machine: string,
+  binary: string,
+  log: (message: string) => void = () => {},
+): Promise<string> {
+  await incus(["file", "push", binary, `${instanceName}${BOOTSTRAP_PATH}`, "--mode", "0755"],
+    900_000);
+
+  const version = (await incusOk(
+    ["exec", instanceName, "--", BOOTSTRAP_PATH, "version"], 60_000,
+  ))?.trim();
+  if (!version) {
+    throw new PlacementError(
+      machine,
+      `the installer was copied to ${machine} and does not run there. It carries a saved ` +
+        `container image and is twenty megabytes or so, which is exactly the size at which a ` +
+        `truncated copy stops being theoretical.`,
+    );
+  }
+  log(`  placed the installer on ${machine} (${version})`);
+  return version;
+}
+
 /** Place everything a scenario declares. */
 export async function applyPlacements(
   scenario: Scenario,

src/rebuild.ts

@@ -2,7 +2,8 @@
  * Rebuild what the lab runs, from source, before it runs.
  *
  * **A stale artifact reporting success against old rules is the fault this project keeps writing
- * down** (novox/hq 04-ISSUES/005). The lab consumes three artifacts from two repositories, and they
+ * down** (novox/hq 04-ISSUES/005). The lab consumes a handful of artifacts from two repositories,
+ * and they
  * were rebuilt by hand, one at a time, from memory. A rename in the control plane's catalogue needs
  * both the control-plane image *and* the builder binary, because both parse manifests; rebuilding
  * one left a binary eleven hours old refusing a field the mesh had just renamed, and cost a full
@@ -73,9 +74,40 @@ export function planned(env: NodeJS.ProcessEnv = process.env): Build[] {
       });
     }
   }
+
+  // **The installer, carrying the control plane's image.**
+  //
+  // Last, and that is an ordering rather than a preference: `make bootstrap` embeds the output of
+  // `docker save <image>`, so the image has to have been built by the step above or the installer
+  // carries whatever was lying around — the eleven-hour-old artifact again, this time inside a
+  // binary where nothing would ever notice.
+  //
+  // It is built here at all because the bed now bootstraps THROUGH it (novox/hq ADR 0067): the
+  // anchor is brought into existence by running the same program a bare machine runs, rather than
+  // by the bed applying a substrate bundle by hand and calling that an install. An installer that
+  // was stale would be a bed proving something about last week's procedure.
+  const installer = env["MESH_LAB_BOOTSTRAP_BINARY"];
+  if (installer && where["mesh-host"]) {
+    builds.push({
+      what: "installer",
+      in: where["mesh-host"],
+      argv: ["make", "bootstrap", `IMAGE=${controlPlaneImage(env)}`, `BOOTSTRAP_OUT=${installer}`],
+    });
+  }
   return builds;
 }
 
+/**
+ * The control-plane image the installer carries.
+ *
+ * `mesh-control:development` is what mesh-control's `make image` tags, and what the scenarios name
+ * — one tag, said in one place. It is overridable because a release installer carries a release
+ * image, and nothing about that is the lab's business.
+ */
+export function controlPlaneImage(env: NodeJS.ProcessEnv = process.env): string {
+  return env["MESH_LAB_CONTROL_IMAGE"] ?? "mesh-control:development";
+}
+
 /** rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. */
 export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] {
   const built: string[] = [];

test/integration/whole-mesh-full.test.ts

@@ -12,6 +12,25 @@
  * There is NO separate anchor: novox IS the anchor. The substrate runs on novox, and novox also
  * enrols as a node and receives its own service set — the substrate host and a service node at once.
  *
+ * TWO ACTS, AND THE BED NOW DISTINGUISHES THEM (novox/hq ADR 0067).
+ *
+ *   GENESIS — novox is brought into existence by `mesh-bootstrap`, the installer, run on the
+ *     machine exactly as a person would run it on a bare one: preflight, load the carried
+ *     control-plane image, write the bundle, apply, verify, enrol itself, install the registry
+ *     module, push the control-plane image into it, reinstall the control plane as an ordinary
+ *     module pinned to the digest that push produced, retire the temporary one. Afterwards novox
+ *     is a WORKING MESH OF ONE, and this bed asserts exactly that before going any further.
+ *
+ *   JOINING — ace, shanks and g14 then join a mesh that already exists: host binary, token,
+ *     `enrol`, run the agent. No bootstrap, no substrate, no registry. novox is NOT enrolled again.
+ *
+ * The bed used to do neither. It applied the substrate bundle itself and looped enrolment over all
+ * four machines as one continuous operation — which got the order right by accident and modelled
+ * the wrong shape, and is why ADR 0067's own acceptance check ("the bed bootstraps through the
+ * installer rather than around it") went unmet. Genesis GATES joining: if it stops, the bed says
+ * which of the installer's ten steps it stopped at and goes no further, because a second machine
+ * joining a mesh that is not ready is a different failure and must not be mistaken for this one.
+ *
  * THE THING THIS BED EXISTS TO PROVE (the flat beds never could): does the WireGuard overlay tunnel
  * FORM across the access point? A home node (ace/shanks/g14) dials novox's PUBLIC hub endpoint
  * 192.0.2.20:51820/udp OUT through the household gateway's masquerade; the handshake has to complete
@@ -32,21 +51,26 @@
  * behaves like every other: raise in before(), destroy in after().
  *
  *   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
+ *   MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap   MESH_LAB_CATALOG=.../mesh-catalog/modules
  */
 
 import { test, before, after } from "node:test";
 import assert from "node:assert/strict";
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
 import { loadScenario } from "../../src/declaration/parse.ts";
 import { raise } from "../../src/lifecycle/raise.ts";
-import { destroy, exec } from "../../src/lifecycle/operate.ts";
+import { destroy, exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts";
-import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
+import {
+  bootstrapBinaryPath, hostBinaryPath, placeBootstrap, BOOTSTRAP_PATH, HOST_PATH,
+} from "../../src/lifecycle/place.ts";
 import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
 import type { HeldImage } from "../../src/pinning.ts";
 
 const capability = await labIsUsable();
 const binary = hostBinaryPath();
+const installer = bootstrapBinaryPath();
 const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
 const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
 
@@ -56,15 +80,51 @@ const skip = !capability.usable
     ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
     : !bundle || !existsSync(bundle)
       ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)"
-      : false;
+      : !installer || !existsSync(installer)
+        ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap (mesh-host `make " +
+          "bootstrap IMAGE=mesh-control:development`). The anchor is raised BY the installer now, " +
+          "so a run without one would be testing the procedure this bed exists to stop testing"
+        : false;
 
 const SCENARIO = "whole-mesh-full";
 /** novox hosts the substrate and the control plane; it is where `mesh` commands run. */
 const CONTROL = "novox";
-/** Every node that enrols. novox is on hosting; the rest are behind the home gateway. */
+/** Every node in the mesh. novox is on hosting; the rest are behind the home gateway. */
 const NODES = ["novox", "ace", "shanks", "g14"];
+/**
+ * The machines that JOIN. novox is not one of them, and that is the distinction this bed was
+ * restructured to make: novox is brought into existence by the installer, which enrols it as part
+ * of genesis. Enrolling it again here would be a second identity the mesh does not know.
+ */
 const HOME_NODES = ["ace", "shanks", "g14"];
 
+/** A checkout of the mesh's catalogue, on the anchor, for the installer to read manifests from. */
+const CATALOGUE_ON_MACHINE = "/opt/mesh-catalog";
+
+/**
+ * The manifests `mesh-bootstrap` reads out of that checkout — and only those.
+ *
+ * Named here rather than pushing the whole repository because the whole repository is a hundred
+ * megabytes of `node_modules` and the installer opens exactly two files: mesh-host's
+ * `internal/bootstrap` RegistryModule and ControlPlaneModule. If it ever opens a third, this list
+ * is where the bed finds out, by the installer saying which manifest it could not read.
+ */
+const CATALOGUE_MODULES = ["registry", "mesh-control"];
+
+/**
+ * Where this mesh keeps its own images, as the anchor reaches it.
+ *
+ * **Loopback, and that is a finding rather than a shortcut.** The mesh's registry is plain HTTP on
+ * purpose — it is reached over the mesh's own network, which is already the encrypted and
+ * authenticated thing — and a container runtime refuses a plain-HTTP registry at any address
+ * EXCEPT a loopback one unless it has been told to allow it. So genesis can push to 127.0.0.1:5000
+ * with no configuration, and the reference the control-plane module is then pinned to is one only
+ * the anchor can pull. On this bed that is enough, because only the anchor runs the control plane.
+ * A mesh where a second machine had to pull it would need the runtimes told about the registry
+ * first, and nothing in the design says who does that.
+ */
+const MESH_REGISTRY = "127.0.0.1:5000";
+
 /**
  * ADR 0066 — the domain each public-facing node composes its routed names under.
  *
@@ -127,6 +187,11 @@ const NOVOX: Mod[] = [
   { name: "amqp-email-forwarder", containers: ["amqp-email-forwarder"] },
   { name: "portainer", containers: ["portainer", "mesh-portainer"] },
   { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] },
+  // Already installed, assigned and running — genesis needed it to publish the control plane's
+  // image. Left in the plan on purpose: registering the same manifest and assigning it again is
+  // what an operator's `module add` + `assign` would do on a mesh that already has it, and a
+  // module the installer put there had better survive being asked for a second time. It also keeps
+  // mesh-registry in the convergence report, where a reader expects to see it.
   { name: "registry", containers: ["mesh-registry"] },
   {
     name: "mailu",
@@ -404,6 +469,227 @@ async function overlayAddr(node: string): Promise<string> {
   return out.split("\n").map((l) => l.trim()).find(Boolean) ?? "";
 }
 
+// ==================================================================================================
+// PHASE 1 — GENESIS. novox is brought into existence by the installer.
+// ==================================================================================================
+
+/** What genesis did, or where it stopped. */
+interface GenesisResult {
+  ok: boolean;
+  /** `step 7 of 10, registry` — the installer's own words, so the bed reports the cause. */
+  step: string;
+  why: string;
+  report: string[];
+}
+
+/** The step a failed `mesh-bootstrap` names, as it prints it, or "" if it named none. */
+function stepIn(said: string): string {
+  return said.match(/^mesh-bootstrap: (step \d+ of \d+, [a-z-]+):/m)?.[1] ?? "";
+}
+
+/**
+ * Put on the anchor what the installer needs to read, and run it.
+ *
+ * **This is the whole of what changed, and it is not a refactor.** The bed used to apply the
+ * substrate bundle itself, by hand, and then enrol four machines in one loop. It got the order
+ * right by accident and it modelled the wrong shape: an install procedure that exists only as a
+ * test fixture is exercised by whoever writes tests and never by whoever installs, which is why
+ * every bootstrap fault this year was found late (novox/hq ADR 0067). The anchor is now raised by
+ * running the same program a bare machine runs, and the bed only reads the result.
+ *
+ * It is run up to three times. Not to paper over a failure — every attempt's failing step is
+ * printed — but because `mesh-bootstrap` is idempotent by design and says so, and because the one
+ * thing here that fails for a reason which goes away by itself is a pull: the store, the broker and
+ * the registry come from the internet, through a household gateway's masquerade, and Docker Hub
+ * rate-limiting an anonymous pull is not this mesh's fault. A re-run is the retry, and it is the
+ * retry the installer's own documentation names.
+ */
+async function genesis(images: HeldImage[]): Promise<GenesisResult> {
+  const report: string[] = ["================ GENESIS: novox becomes a mesh of one ================"];
+  const stop = (step: string, why: string): GenesisResult => {
+    report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`);
+    return { ok: false, step, why, report };
+  };
+
+  // The installer, beside the host binary. Everything else on this machine was placed by `raise`;
+  // this one is placed here because only the anchor is bootstrapped.
+  const name = await instanceNameOf(instanceId, CONTROL);
+  const version = await placeBootstrap(name, CONTROL, installer as string,
+    (m) => console.log(`genesis:${m}`));
+  report.push(`  installer         ${version} at ${BOOTSTRAP_PATH}`);
+
+  // The catalogue. `mesh-bootstrap --catalog` reads manifests from a CHECKOUT on the machine,
+  // because at this moment the mesh has no forge, no build machine and — until step 7 finishes —
+  // no registry. A manifest is a file, and somebody has to have put it there.
+  await must(CONTROL, `mkdir -p ${CATALOGUE_MODULES.map((m) => `${CATALOGUE_ON_MACHINE}/modules/${m}`).join(" ")}`);
+  for (const module of CATALOGUE_MODULES) {
+    const from = resolve(catalogDir, module, "module.json");
+    assert.ok(existsSync(from), `the catalogue has no ${module}/module.json at ${from}`);
+    await push(instanceId, CONTROL, from, `${CATALOGUE_ON_MACHINE}/modules/${module}/module.json`);
+  }
+  report.push(`  catalogue         ${CATALOGUE_MODULES.join(", ")} at ${CATALOGUE_ON_MACHINE}`);
+
+  // The substrate TEMPLATE — not the bundle. The installer produces the bundle from it: it replaces
+  // the control plane's image with the id of the image it carries, renames that container
+  // `temp-mesh-control`, and writes the result where a person can read it.
+  //
+  // Two substitutions still happen here, and both belong to the bed rather than to the installer.
+  // The example names three images at a registry the lab no longer raises: the store and the broker
+  // become the upstream references mesh-catalog pins (harness), and the machine pulls them over its
+  // uplink like any first node. The third, mesh-control, is deliberately LEFT naming that dead
+  // registry — the installer overwrites it, and leaving it proves that it does.
+  //
+  // And the broker's advertised address. The template hardcodes 192.0.2.10:5671, the old
+  // separate-anchor address; a token carries MESH_BROKER_ADDRESS verbatim as the endpoint an
+  // enrolling node dials, so with the substrate on novox it must be novox's own public address or
+  // every node would enrol against a dead one. The installer refuses to guess this and says so
+  // loudly, which is right — it does not know what this machine is called from outside.
+  const template = bundleFor(images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671");
+  const local = join(tmpdir(), `mesh-lab-substrate-${process.pid}.lock`);
+  writeFileSync(local, template);
+  await push(instanceId, CONTROL, local, "/tmp/substrate-template.lock");
+
+  const command = [
+    BOOTSTRAP_PATH,
+    `--bundle /tmp/substrate-template.lock`,
+    `--catalog ${CATALOGUE_ON_MACHINE}`,
+    `--node ${CONTROL}`,
+    `--registry ${MESH_REGISTRY}`,
+    `--host ${HOST_PATH}`,
+    // The lab has no unit to supervise the host with, and the installer refuses to invent one — a
+    // unit file is a packaging decision. This is the arrangement it offers instead, and it is loud
+    // about what it is: a host started this way does not survive a reboot.
+    `--host-in-background`,
+  ].join(" ");
+
+  let said = "";
+  let step = "";
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    const ran = await on(CONTROL, command, 2_400_000);
+    said = ran.out;
+    console.log(`\n---- mesh-bootstrap on ${CONTROL} (attempt ${attempt}) ----\n${said}`);
+    if (ran.ok) {
+      step = "";
+      break;
+    }
+    step = stepIn(said);
+    if (attempt < 3) {
+      console.log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; ` +
+        `re-running in 30s — every step it already did will say so`);
+      await new Promise((r) => setTimeout(r, 30_000));
+    }
+  }
+  if (step) {
+    return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n"));
+  }
+
+  // ------------------------------------------------------------------------------------------
+  // And now the only thing that matters: is novox a WORKING MESH OF ONE? Asked of the machine,
+  // never inferred from the installer exiting zero (novox/hq ADR 0018).
+  // ------------------------------------------------------------------------------------------
+
+  // 1. The control plane answers. Asked of the PERMANENT container by name — `status` opens all
+  //    three stores, so a reply proves the connections it was given are the substrate's own.
+  const answered = await on(CONTROL, `docker exec mesh-control /mesh-control status`, 60_000);
+  report.push(`  control plane     ${answered.ok ? answered.out.split("\n")[0] : "NO ANSWER"}`);
+  if (!answered.ok) return stop("after step 10", `mesh-control does not answer:\n${answered.out}`);
+
+  // 2. The registry replies on /v2/ — the registry API's own "yes, I am one and I am ready". A
+  //    container that is up is not a registry that serves.
+  const v2 = await on(CONTROL,
+    `curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://${MESH_REGISTRY}/v2/`);
+  const v2Code = v2.out.trim();
+  report.push(`  registry /v2/     ${v2Code || "no answer"}`);
+  if (v2Code !== "200") return stop("after step 10", `the mesh's own registry answered ${v2Code || "nothing"}`);
+
+  // 3. THE PIVOT COMPLETED. ADR 0067 states this check in as many words: after installing, the
+  //    running control plane's image is pinned by a digest THE MESH'S OWN REGISTRY ASSIGNED — not
+  //    by an image id. If it is still an image id, the substrate's container is what is running,
+  //    nothing was published, and this mesh can never roll out its own upgrades.
+  const pinnedTo = (await on(CONTROL, `docker inspect --format '{{.Config.Image}}' mesh-control`))
+    .out.trim();
+  report.push(`  pinned to         ${pinnedTo || "(nothing)"}`);
+  if (/^sha256:[0-9a-f]{64}$/.test(pinnedTo)) {
+    return stop("after step 10",
+      `mesh-control is running from ${pinnedTo}, which is an IMAGE ID — the digest of the image's ` +
+      `own configuration, which no registry ever served. The pivot did not happen: what is ` +
+      `running is the image the installer carried, not one this mesh published, so this mesh ` +
+      `cannot upgrade itself (novox/hq ADR 0067, "the pivot completed").`);
+  }
+  if (!new RegExp(`^${MESH_REGISTRY.replaceAll(".", "\\.")}/mesh-control@sha256:[0-9a-f]{64}$`)
+    .test(pinnedTo)) {
+    return stop("after step 10",
+      `mesh-control is running from ${pinnedTo || "nothing this bed could read"}, which is not a ` +
+      `digest assigned by ${MESH_REGISTRY}.`);
+  }
+
+  // 3b. And the registry really serves it, asked of the registry rather than of the container. A
+  //     reference is a claim; a tag list is the registry agreeing.
+  const tags = await on(CONTROL,
+    `curl -s --max-time 10 http://${MESH_REGISTRY}/v2/mesh-control/tags/list`);
+  report.push(`  registry holds    ${tags.out.trim() || "nothing"}`);
+  if (!tags.out.includes("genesis")) {
+    return stop("after step 10",
+      `${MESH_REGISTRY} does not serve mesh-control, so the digest the container is pinned to ` +
+      `names an image nothing can pull: ${tags.out.trim()}`);
+  }
+
+  // 4. The temporary control plane is GONE. Two control planes is the half-finished state, and the
+  //    name is the audit: a machine running mesh-control and not temp-mesh-control has pivoted.
+  const temp = await on(CONTROL, `docker inspect --format '{{.State.Status}}' temp-mesh-control`);
+  report.push(`  temp-mesh-control ${temp.ok ? `STILL HERE (${temp.out.trim()})` : "gone"}`);
+  if (temp.ok) {
+    return stop("after step 10",
+      `temp-mesh-control is still ${temp.out.trim()}. Two control planes are consuming this ` +
+      `mesh's broker queues; neither is wrong and the pivot is not finished.`);
+  }
+
+  // 5. And the mesh has heard from its one node. Everything the join phase does next depends on it.
+  const nodes = await on(CONTROL, `docker exec mesh-control /mesh-control node list`);
+  report.push(`  node list         ${nodes.out.trim().split("\n").join(" | ")}`);
+  const line = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${CONTROL} `));
+  if (!line || !/^\S+\s+here\b/.test(line)) {
+    return stop("after step 10",
+      `the mesh has not heard from ${CONTROL}: ${line ?? "it has no record of it at all"}`);
+  }
+
+  report.push(`\nVERDICT: ${CONTROL} is a working mesh of one, bootstrapped through the installer.`);
+  return { ok: true, step: "", why: "", report };
+}
+
+// ==================================================================================================
+// PHASE 2 — JOINING. Everything else is a machine joining a mesh that already exists.
+// ==================================================================================================
+
+/**
+ * ace, shanks and g14 join. Host binary plus a token — no bootstrap, no substrate, no registry.
+ *
+ * **novox is not in this loop.** It was enrolled by the installer, as part of becoming a mesh, and
+ * enrolling it again would present the mesh with a second identity for a node it already knows —
+ * which `mesh-host enrol` refuses, and rightly.
+ *
+ * The home nodes reach novox's public 192.0.2.20:5671 by dialling OUT through the household
+ * gateway, so the enrol itself is the first proof that outbound home→public works.
+ */
+async function joinTheMesh(): Promise<void> {
+  // ADR 0066: said as soon as the record exists, because everything routed is composed from it. A
+  // node that faces the outside has one; the workstations do not, and are given none. The anchor's
+  // is set here rather than in genesis because it is a fact about the mesh, not part of raising
+  // one — and the installer has an opinion about neither.
+  const anchorDomain = PUBLIC_DOMAIN[CONTROL];
+  if (anchorDomain) await mesh(`node public-domain ${CONTROL} ${anchorDomain}`);
+
+  for (const machine of HOME_NODES) {
+    await mesh(`node add ${machine}`);
+    const domain = PUBLIC_DOMAIN[machine];
+    if (domain) await mesh(`node public-domain ${machine} ${domain}`);
+    const token = tokenFrom(await mesh(`token issue --node ${machine}`));
+    const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
+    assert.match(said, new RegExp(`enrolled as ${machine}`), said);
+    await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
+  }
+}
+
 before(async () => {
   if (skip) return;
   assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
@@ -416,67 +702,35 @@ before(async () => {
   held = raised.images;
   console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
 
-  // novox raises the substrate from its bundle. The store and the broker keep upstream references
+  // ---- PHASE 1, and it GATES phase 2 ---------------------------------------------------------
-  // and novox PULLS them, over its uplink, the way any first node does; mesh-control exists in no
-  // registry, so it becomes the ID novox holds it under — the whole of what changed here.
   //
-  // The rest is the collapse: the substrate rides novox, not a separate anchor. The bundle hardcodes the
+  // Caught rather than allowed to propagate, so that a failure BEFORE the installer ran — a
-  // broker's advertised address as 192.0.2.10:5671 (the OLD separate-anchor address) — and a token
+  // manifest that is not where the bed thought, a binary that would not copy — is reported in the
-  // carries MESH_BROKER_ADDRESS verbatim as the endpoint an enrolling node dials. With the substrate
+  // same shape as one the installer itself named, instead of as a bare stack trace from a helper.
-  // on novox that endpoint must be novox's own public address, or every node (novox included) would
+  let genesisResult: GenesisResult;
-  // enrol against a dead address. The broker serves its cert on all interfaces and the token pins by
+  try {
-  // fingerprint, not hostname, so only the address needs correcting.
+    genesisResult = await genesis(raised.images);
-  const bundleText = bundleFor(raised.images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671");
+  } catch (err) {
-  await must(CONTROL, `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleText}\nMESHBUNDLE`);
+    genesisResult = {
+      ok: false,
+      step: "getting the anchor ready to be bootstrapped — the installer never ran",
+      why: (err as Error).message,
+      report: ["================ GENESIS: novox becomes a mesh of one ================"],
+    };
+  }
+  console.log(genesisResult.report.join("\n"));
+  assert.ok(genesisResult.ok,
+    `GENESIS FAILED — ${genesisResult.step || "no step named"}.\n\n${genesisResult.why}\n\n` +
+    `No other machine was asked to join. A second machine joining a mesh that is not ready is a ` +
+    `different failure with a different cause, and running it now would bury this one under it.\n\n` +
+    genesisResult.report.join("\n"));
 
-  // **Belt as well as braces on fetching.** `raise` refuses to return until every machine with
+  // ---- PHASE 2 --------------------------------------------------------------------------------
-  // egress has resolved a name and reached the internet, so the first attempt should be the only
+  await joinTheMesh();
-  // one. This retry is here because of what the failure looked like when there was no such
-  // guarantee: the apply died on a pull, `before` threw, and the instance was left a bare shell —
-  // VMs, no substrate, no enrolment, nothing to read.
-  //
-  // And a pull is now genuinely the one step that can fail for a reason which goes away by itself:
-  // the store and the broker come from the internet, through a household gateway's masquerade, and
-  // a registry elsewhere having a bad minute is not this mesh's fault. That is the trade this bed
-  // accepts — it is no longer hermetic, because a real node is not either, and the faults it was
-  // hiding were exactly the ones that only appear when a machine has to fetch for itself.
-  {
-    let applied = false;
-    let said = "";
-    for (let attempt = 1; attempt <= 3 && !applied; attempt++) {
-      const tried = await on(CONTROL, `${HOST_PATH} apply /tmp/substrate.lock`, 900_000);
-      applied = tried.ok;
-      said = tried.out;
-      if (!applied && attempt < 3) {
-        console.log(`substrate apply attempt ${attempt} failed; retrying in 30s:\n${said.split("\n").slice(-8).join("\n")}`);
-        await new Promise((r) => setTimeout(r, 30_000));
-      }
-    }
-    assert.ok(applied, `the substrate did not apply on novox after three attempts:\n${said}`);
-  }
-  const up = await must(CONTROL, `docker ps --format '{{.Names}}'`);
-  for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) {
-    assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`);
-  }
-
-  // Every node joins the one mesh and runs a host. The home nodes reach novox's public 192.0.2.20:5671
-  // by dialling OUT through the household gateway — the enrol itself is the first proof that outbound
-  // home→public works. novox enrols too: substrate host and service node at once.
-  for (const machine of NODES) {
-    await mesh(`node add ${machine}`);
-    // ADR 0066: said as soon as the record exists, because everything routed is composed from it.
-    // A node that faces the outside has one; the workstations do not, and are given none.
-    const domain = PUBLIC_DOMAIN[machine];
-    if (domain) await mesh(`node public-domain ${machine} ${domain}`);
-    const token = tokenFrom(await mesh(`token issue --node ${machine}`));
-    const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
-    assert.match(said, new RegExp(`enrolled as ${machine}`), said);
-    await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
-  }
 
   // The operator provides ace's media library (ADR 0051 accesses confirm the paths, create nothing).
   await must("ace", `mkdir -p ${MEDIA_DIRS.join(" ")}`);
-}, { timeout: 3_600_000 });
+}, { timeout: 5_400_000 });
 
 after(async () => {
   if (KEEP) {

test/rebuild.test.ts

@@ -1,7 +1,8 @@
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { planned } from "../src/rebuild.ts";
+import { planned, controlPlaneImage } from "../src/rebuild.ts";
 import { repositories } from "../src/repos.ts";
+import { loadScenario } from "../src/declaration/parse.ts";
 
 // The control plane's image and the builder are one step, not two.
 //
@@ -39,6 +40,52 @@ test("every image the lab runs is rebuilt, not only the control plane's", () =>
   }
 });
 
+// The installer is built, and it is built AFTER the image it carries.
+//
+// `make bootstrap` embeds the output of `docker save <image>`, so an installer built before the
+// control plane's image is one carrying whatever was lying around — 04-ISSUES/005 again, this time
+// sealed inside a binary where nothing would ever notice. The bed raises its anchor by running this
+// program (novox/hq ADR 0067), so a stale one is a bed proving something about last week.
+test("the installer is built, carrying the image built in the same run", () => {
+  const builds = planned({
+    MESH_LAB_HOST_BINARY: "/repo/host/mesh-host",
+    MESH_LAB_MODULES: "/repo/control/examples/modules",
+    MESH_LAB_BOOTSTRAP_BINARY: "/repo/host/mesh-bootstrap",
+  });
+  const what = builds.map((b) => b.what);
+  assert.ok(what.includes("installer"), "the installer is never built, so the bed carries a stale one");
+  assert.ok(
+    what.indexOf("images") < what.indexOf("installer"),
+    `the installer is built before the image it embeds: ${what.join(", ")}`,
+  );
+
+  const installer = builds.find((b) => b.what === "installer")!;
+  assert.equal(installer.in, "/repo/host");
+  assert.ok(installer.argv.includes(`IMAGE=${controlPlaneImage({})}`), installer.argv.join(" "));
+  assert.ok(installer.argv.includes("BOOTSTRAP_OUT=/repo/host/mesh-bootstrap"), installer.argv.join(" "));
+});
+
+// The anchor must NOT be handed the control plane's image.
+//
+// The installer carries it, which is the whole reason a machine that can reach no registry can
+// raise a mesh (novox/hq ADR 0067). Hand it over from the workstation as well and the installer's
+// load says "already held", the carrying is never exercised, and the bed goes green on a fiction —
+// the same class of thing the lab's own registry used to hide. Asserted on the file rather than
+// remembered, because a list of images is exactly the kind of thing somebody tops up.
+test("the whole-mesh bed hands its anchor no control-plane image", () => {
+  const scenario = loadScenario("scenarios/whole-mesh-full.yml");
+  const named = [
+    ...(scenario.images ?? []),
+    ...Object.values(scenario.machines).flatMap((m) => m.images ?? []),
+  ];
+  assert.deepEqual(
+    named.filter((i) => i.startsWith("mesh-control")),
+    [],
+    "the anchor is handed mesh-control, so genesis would never find out whether the installer " +
+      "really carries it",
+  );
+});
+
 // A repository this run was not pointed at is not built, and not claimed.
 test("only what this run was pointed at is built", () => {
   assert.deepEqual(planned({}), []);