scenarios/fresh-mesh.yml

@@ -0,0 +1,106 @@
+# FOUR FRESH MACHINES AND THE INSTALLER. Nothing is handed to them.
+#
+# This is `whole-mesh-full`'s topology with `genesis-single`'s honesty. The four-machine bed loads
+# thirty-four of the mesh's own images onto its machines from the workstation, because it does not
+# build them — they are produced by hand beside the bed and copied in. That is a shape no real
+# installation has, and it is the same class of fiction the lab already removed once: it used to
+# raise a registry inside the scenario, and a bootstrap that only worked against it went green here
+# and would have failed on any real machine.
+#
+# So this bed hands over NOTHING. There is no `images:` list. Every machine gets a container
+# runtime and the host binary, which are prerequisites of the machine rather than parts of the
+# mesh, and after that the mesh is on its own:
+#
+#   - the substrate, the registry and the builder's own dependencies are PULLED from the internet,
+#     which is where a bare machine gets them;
+#   - the control plane is BUILT, by the builder the installer carries, from a repository and a
+#     commit it is told to use;
+#   - every module after that is BUILT by the mesh's own builder and published into the mesh's own
+#     registry, and a machine that runs one PULLS it from there.
+#
+# That last clause is the thing no bed has ever checked, and it is why this one has four machines
+# rather than one. Genesis puts the builder, the registry and everything they make on ONE machine.
+# A second machine running a mesh-built module has to fetch it from a registry that asks who it is,
+# and nothing yet gives a joined node an account for it. The bed asks anyway, in its own test, so
+# the gap is a named failure rather than an absence.
+#
+#   hosting (public, routable)         home (private, behind the access point)
+#     novox  192.0.2.20  ── anchor       ace     10.99.1.10  home server
+#              substrate, registry,      shanks  10.99.1.20  workstation
+#              builder, control plane    g14     10.99.1.30  workstation
+#
+# EGRESS IS NOT OPTIONAL HERE. With nothing loaded, a sealed machine stops at the installer's first
+# pull. Every machine has a way out, and it is a SECOND path: each still reaches the rest of the
+# scenario through its declared gateway, and the uplink carries only what leaves the scenario —
+# the public images, and the forge the control plane is cloned from.
+#
+#   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap
+#   MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
+#   MESH_LAB_CATALOG=.../mesh-catalog/modules
+#   MESH_LAB_SOURCE=<forge url>   MESH_LAB_SOURCE_REF=<commit>
+scenario: fresh-mesh
+
+segments:
+  # The routable segment. novox lives here; its public address is the broker endpoint every token
+  # carries and the overlay hub the home nodes dial.
+  hosting:
+    kind: public
+    cidr: [192.0.2.0/24]
+
+  # The household segment behind an ordinary home router: masquerades v4 outbound, forwards
+  # inbound, expires idle mappings after two minutes.
+  home:
+    kind: private
+    cidr: [10.99.1.0/24]
+    gateway:
+      to: hosting
+      address: [192.0.2.50]
+      nat: [v4]
+      forwardable: true
+      mapping_ttl: 120s
+
+machines:
+  # The anchor. Raised by the installer into a mesh of one, and then asked to build.
+  #
+  # Sized for what it actually does here: the store, the broker, the registry, TWO control planes
+  # during the pivot, the builder, and a build workspace holding a Node toolchain image and an npm
+  # cache. It is NOT sized for the whole novox service set, because this bed does not run one — it
+  # proves the machinery that would produce it.
+  novox:
+    at: { segment: hosting, address: [192.0.2.20] }
+    egress: true
+    inbound: allow
+    memory: 12GiB
+    cpus: 6
+    disk: 60GiB
+
+  # Three machines that JOIN. Host binary and a token, nothing else — no bootstrap, no substrate,
+  # no registry. They are deliberately small: what they are here to prove is that a joined machine
+  # can be given a module the mesh built, which is a question about credentials and not about load.
+  ace:
+    at: { segment: home, address: [10.99.1.10] }
+    egress: true
+    inbound: allow
+    memory: 4GiB
+    cpus: 2
+    disk: 25GiB
+  shanks:
+    at: { segment: home, address: [10.99.1.20] }
+    egress: true
+    inbound: allow
+    memory: 3GiB
+    cpus: 2
+    disk: 20GiB
+  g14:
+    at: { segment: home, address: [10.99.1.30] }
+    egress: true
+    inbound: allow
+    memory: 3GiB
+    cpus: 2
+    disk: 20GiB
+
+# **No `images:` key, and that is the whole point of this file.** Anything a machine holds here, it
+# pulled or the mesh built. See the header.
+
+place:
+  all: [host, runtime]

src/lifecycle/egress.ts

@@ -89,9 +89,40 @@ export async function confirmEgress(
     // the retry is tightened so a silent server costs seconds rather than the step. A broken uplink
     // still fails the check above, before any of this.
     await resilientResolver(name);
+
+    // **And then prove it is STEADY, because one success proved nothing.**
+    //
+    // The check above is satisfied by a single answer, and that is how a machine resolving one
+    // query in three passed it and then killed two installs twenty minutes later. What a run needs
+    // is not "a name resolved once" but "names resolve reliably", and those differ by exactly the
+    // failure that has cost the most time here. Consecutive, because alternating success and
+    // timeout is the observed shape — a total count would pass on the same machine.
+    const steady = await steadilyResolves(name, 5);
+    if (steady < 5) {
+      throw new EgressError(
+        `${machine} resolves ${UPSTREAM} only ${steady} time(s) in five consecutive tries.\n` +
+          `  It has egress and an unreliable resolver, which does not fail here — it fails later, ` +
+          `inside a pull or a clone, as "could not resolve host" with the cause long out of view.\n` +
+          `  The uplink's own resolver is the usual culprit and is deliberately last in the list; ` +
+          `a machine still failing this has something wrong upstream of the lab.`,
+      );
+    }
+    log(`  ${machine} resolves steadily (5/5)`);
   }
 }
 
+/** How many of `tries` consecutive lookups answered. Stops at the first failure. */
+async function steadilyResolves(name: string, tries: number): Promise<number> {
+  for (let i = 0; i < tries; i++) {
+    const said = await incusOk(
+      ["exec", name, "--", "sh", "-c",
+       `timeout 8 getent hosts ${UPSTREAM} >/dev/null && echo yes`], 20_000,
+    );
+    if (said?.trim() !== "yes") return i;
+  }
+  return tries;
+}
+
 async function resolves(name: string, waitSeconds: number): Promise<boolean> {
   const deadline = Date.now() + waitSeconds * 1_000;
   while (Date.now() < deadline) {
@@ -140,11 +171,22 @@ async function reaches(name: string, waitSeconds: number): Promise<string | null
  * The shape of the problem is visible in `resolvectl status`: the machine has sensible global
  * fallbacks, and the *link* carrying the default route has exactly one server — the uplink gateway.
  * resolved will not reach for a global fallback while the link it is using has a server of its own,
- * so one unanswered packet is one failed lookup. Under four machines pulling images at once that
- * happens, and it has ended three runs long after the egress check passed.
+ * so one unanswered packet is one failed lookup.
  *
- * The uplink stays first, so the modelled path is still the one used and still proven by the check
- * above. The others answer only when it does not.
+ * **The uplink goes last, and this is a preference, not a fix.** The uplink's resolver is a single
+ * dnsmasq with no unique knowledge any scenario machine needs — machines here address each other by
+ * address, and the mesh writes its own names into /etc/hosts — so there is no reason for anything
+ * to wait on it. It is kept, last, so DHCP-supplied names still answer.
+ *
+ * **It is written down as a preference because it was once mistaken for the cure.** Lookups were
+ * timing out two times in three; the uplink was first in the list; reordering it made five in five;
+ * the conclusion drew itself and was wrong. The machines were sharing ONE DHCP lease (see
+ * `distinguishMachines` in raise.ts), so which machine could resolve anything depended on which had
+ * last won an ARP race — and reordering resolvers on a machine that has just won looks exactly like
+ * a fix. Nothing below this line will save a bed whose machines share an address.
+ *
+ * The caches are flushed afterwards, because resolved remembers the failures it collected while
+ * the bad server was in front.
  */
 async function resilientResolver(name: string): Promise<void> {
   await incus([
@@ -152,7 +194,8 @@ async function resilientResolver(name: string): Promise<void> {
     `link=$(ip -4 route show default | awk '{print $5}' | head -n1); ` +
       `via=$(ip -4 route show default | awk '{print $3}' | head -n1); ` +
       `if [ -n "$link" ] && command -v resolvectl >/dev/null 2>&1; then ` +
-      `resolvectl dns "$link" $via 1.1.1.1 8.8.8.8 >/dev/null 2>&1 || true; fi; true`,
+      `resolvectl dns "$link" 1.1.1.1 8.8.8.8 9.9.9.9 $via >/dev/null 2>&1 || true; ` +
+      `resolvectl flush-caches >/dev/null 2>&1 || true; fi; true`,
   ], 30_000);
 }
 

src/lifecycle/raise.ts

@@ -307,9 +307,18 @@ export async function raise(
     enter("waiting for machines to become usable");
     await waitUntilAllUsable(created, readyTimeout, log);
 
+    // **Before the uplink lease is asked for, make sure each machine asks as itself.**
+    enter("giving each machine its own identity");
+    await distinguishMachines(created, log);
+
     enter("applying declared addresses");
     await applyAddresses(scenario, instanceId, byMachine, log);
 
+    // The positive control for the step above: if two machines share an uplink address, say so
+    // HERE, where it is one obvious sentence, rather than letting it surface an hour later as
+    // intermittent name resolution on some machines and not others.
+    await noTwoMachinesShareAnAddress(scenario, byMachine, log);
+
     // Transit first: a gateway's default route points at it, so it has to exist.
     enter("wiring the public networks together");
     const transit = await raiseTransit(scenario, instanceId, log);
@@ -356,3 +365,85 @@ export async function raise(
     throw new RaiseError(instanceId, step, cause);
   }
 }
+
+/**
+ * Give every machine a machine-id of its own, before any of them asks for an uplink lease.
+ *
+ * **Every machine in a bed is a clone of one base image, so they all boot with the SAME
+ * `/etc/machine-id`.** systemd's DHCP client derives its client identifier from that file, and the
+ * uplink's dnsmasq keys leases on the client identifier rather than on the MAC — so four machines
+ * with four distinct MACs were all handed *the same address*, and the host kept one ARP entry for
+ * it. Whichever machine last answered an ARP request got everybody's replies.
+ *
+ * This is the fault behind every "the lab's DNS is flaky" run. It does not present as an address
+ * conflict; it presents as name resolution that works two times in three, as pulls that succeed on
+ * a retry, and as one machine out of four being fine — because that machine happened to be holding
+ * the address. It survived an earlier diagnosis that blamed resolver ordering, because reordering
+ * resolvers on a machine that has just won the ARP race does appear to fix it.
+ *
+ * **Ordering is the whole of it, and the first version got it wrong in the other direction.** That
+ * version also restarted networkd and waited for a new lease, which is what you would do to repair
+ * a machine already holding a shared address. Here there is nothing to repair yet: the uplink is
+ * still down at this point and `applyAddresses` is what asks for the lease. So this writes the
+ * identity and stops, and the request that follows is made as somebody.
+ *
+ * Machines without `systemd-machine-id-setup` are left alone; the step is advisory.
+ */
+async function distinguishMachines(names: string[], log: (m: string) => void): Promise<void> {
+  for (const name of names) {
+    const said = await incusOk([
+      "exec", name, "--", "sh", "-c",
+      // Regenerated rather than written: `systemd-machine-id-setup` owns the format, and a
+      // hand-made value that is not 32 hex characters is rejected by systemd at next boot.
+      `command -v systemd-machine-id-setup >/dev/null 2>&1 || { echo unchanged; exit 0; }; ` +
+        `rm -f /etc/machine-id && systemd-machine-id-setup >/dev/null 2>&1; ` +
+        `cat /etc/machine-id`,
+    ], 60_000);
+    log(`  ${name} is ${said?.trim().slice(0, 12) || "unchanged"}`);
+  }
+}
+
+/**
+ * Refuse to go on if two machines took the same uplink address.
+ *
+ * A check rather than a comment, because the failure it guards is invisible where it happens and
+ * unrecognisable where it surfaces. Every machine that declares egress is asked what address it
+ * holds; two the same is a stop, named as what it is.
+ */
+async function noTwoMachinesShareAnAddress(
+  scenario: Scenario,
+  byMachine: Map<string, string>,
+  log: (m: string) => void,
+): Promise<void> {
+  const held = new Map<string, string[]>();
+  for (const [machine, spec] of Object.entries(scenario.machines)) {
+    if (!spec.egress || spec.at === "detached") continue;
+    const name = byMachine.get(machine);
+    if (!name) continue;
+    const said = (await incusOk([
+      "exec", name, "--", "sh", "-c",
+      // The `src` of the default route, NOT its last field — the first version of this took
+      // `$NF` and compared four machines' route METRIC, which is identical by construction and
+      // made the guard fire on every bed. A check that cannot be wrong is worth less than one
+      // that is read carefully once.
+      `ip -4 -o route show default 2>/dev/null | ` +
+        `awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}' | head -n1`,
+    ], 30_000))?.trim();
+    if (!said) continue;
+    held.set(said, [...(held.get(said) ?? []), machine]);
+  }
+  const shared = [...held.entries()].filter(([, who]) => who.length > 1);
+  if (shared.length === 0) {
+    if (held.size > 0) log(`  every machine with egress took an address of its own`);
+    return;
+  }
+  throw new Error(
+    `two machines took the SAME uplink address: ` +
+      shared.map(([a, who]) => `${a} held by ${who.join(" and ")}`).join("; ") + `.\n` +
+      `  They are clones of one image, so they present one DHCP client identity unless each is ` +
+      `given its own machine-id before the lease is asked for.\n` +
+      `  This does not fail as an address conflict. It fails later, as name resolution that works ` +
+      `about two times in three and as pulls that succeed on a retry, because the host holds one ` +
+      `ARP entry and whichever machine answered last receives the replies.`,
+  );
+}

test/integration/fresh-mesh.test.ts

@@ -0,0 +1,355 @@
+/**
+ * FOUR FRESH MACHINES, AND NOTHING HANDED TO THEM.
+ *
+ * The four-machine bed (`whole-mesh-full`) proves the mesh converges. It does so by loading
+ * thirty-four of the mesh's own images onto its machines from the workstation, because it does not
+ * build them — something beside the bed built them and copied them in. No real installation looks
+ * like that, and the lab has been burned by exactly this shape before: it used to raise a registry
+ * inside the scenario, and a bootstrap that only worked against that registry went green here and
+ * would have failed on any bare machine.
+ *
+ * This bed hands over nothing. The scenario has no `images:` list at all. What the machines get is
+ * a container runtime and the host binary — prerequisites of a machine, not parts of a mesh — and
+ * from there:
+ *
+ *   1. novox is raised into a mesh of one by the installer, which BUILDS the control plane.
+ *   2. ace, shanks and g14 JOIN it, across a household NAT, with a token and nothing else.
+ *   3. The mesh builds the shared base from source, with its own builder.
+ *   4. The mesh builds a real module standing on that base.
+ *   5. The anchor runs it, pinned to a digest the mesh's own registry assigned.
+ *   6. A JOINED machine runs it — which means pulling from a registry that asks who it is.
+ *
+ * Steps 1 and 2 are proven elsewhere and are here because the later ones need them. **Steps 3
+ * through 6 are what this bed exists for**, and 6 is the one nothing has ever checked: genesis
+ * puts the builder, the registry and everything they produce on ONE machine, so every earlier
+ * proof of a mesh-built module running is a proof about the machine that built it. A second
+ * machine has to fetch, and fetching needs an account nothing yet grants (novox/hq issue 042).
+ *
+ * Each step is recorded separately rather than allowed to throw, so a gap at 6 reports as a gap at
+ * 6 instead of erasing the evidence for 3, 4 and 5.
+ *
+ *   MESH_LAB_INCUS='sudo -n incus'
+ *   MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
+ *   MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap
+ *   MESH_LAB_BUNDLE=.../mesh-host/examples/substrate-first-node.lock
+ *   MESH_LAB_CATALOG=.../mesh-catalog/modules
+ *   MESH_LAB_SOURCE=<forge>/mesh-control.git   MESH_LAB_SOURCE_REF=<commit>
+ *   MESH_LAB_KEEP=1   to leave it standing afterwards
+ */
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
+import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
+import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts";
+import { genesis, type GenesisResult } from "./genesis.ts";
+
+const SCENARIO = "fresh-mesh";
+const CONTROL = "novox";
+const HOME_NODES = ["ace", "shanks", "g14"];
+/** The joined machine asked to run a mesh-built module. Any of the three would do. */
+const SECOND = "ace";
+
+/** The anchor's public address — what every other machine dials, and what its own token must name. */
+const ANCHOR = "192.0.2.20";
+/** Where this mesh's registry answers, on the anchor's public address so a joined node can reach it. */
+const REGISTRY = `${ANCHOR}:5000`;
+
+/**
+ * The module built on top of the base, and the base it stands on.
+ *
+ * `amqp-ping` is deliberately small and deliberately REAL: its own TypeScript, compiled by the
+ * shared toolchain, running on the shared runtime, talking to the broker. A module whose artifact
+ * is a mirrored public image would pass every assertion below while skipping the whole of what is
+ * under test (novox/hq SELF-UPGRADE-PLAN, rule 1).
+ */
+const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" };
+const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping" };
+
+const capability = await labIsUsable();
+const binary = hostBinaryPath();
+const installer = bootstrapBinaryPath();
+const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
+const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
+const source = process.env["MESH_LAB_SOURCE"] ?? "";
+const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? "";
+const KEEP = !!process.env["MESH_LAB_KEEP"];
+const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "fresh-mesh-live" : undefined);
+
+/**
+ * Where a repository other than the control plane's lives.
+ *
+ * Derived from `MESH_LAB_SOURCE` by swapping the last path segment, because every one of these
+ * repositories sits beside the others under the same owner on the same forge. Overridable, so a
+ * forge that is arranged differently does not need this bed edited.
+ */
+function forgeUrl(repo: string): string {
+  const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`];
+  if (override) return override;
+  return source.replace(/[^/]+\.git$/, `${repo}.git`);
+}
+/** A branch is acceptable for an ordinary build; only genesis insists on a commit (ADR 0071). */
+const buildRef = process.env["MESH_LAB_BUILD_REF"] ?? "main";
+
+/**
+ * The shared base's manifest, on this workstation.
+ *
+ * The base is a repository with a manifest at its root (novox/hq ADR 0069), so unlike the
+ * catalogue's modules it is not under `MESH_LAB_CATALOG`. Derived from that path on the convention
+ * that the checkouts sit beside each other, and overridable for a layout where they do not.
+ */
+const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ??
+  resolve(catalogDir, "..", "..", BASE.repo, "module.json");
+
+const skip =
+  !capability.usable ? capability.why :
+  !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
+  !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
+  !source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" :
+  !sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" :
+  !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate template" :
+  !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
+  false;
+
+let instanceId = "";
+let raised: GenesisResult;
+
+// ---- talking to the machines ------------------------------------------------------------------
+
+function quote(s: string): string {
+  return `'${s.replaceAll("'", `'\\''`)}'`;
+}
+async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
+  const { stdout } = await exec(instanceId, machine, [
+    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
+  ], timeoutMs);
+  const marker = stdout.lastIndexOf("__exit=");
+  if (marker < 0) return { out: stdout, ok: false };
+  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
+}
+async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
+  const { out, ok } = await on(machine, command, timeoutMs);
+  if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
+  return out;
+}
+async function mesh(command: string, timeoutMs?: number): Promise<string> {
+  return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
+}
+function tokenFrom(said: string): string {
+  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
+  assert.ok(found, `no token in:\n${said}`);
+  return found;
+}
+
+// ---- steps, recorded rather than thrown --------------------------------------------------------
+
+interface Step { ok: boolean; why: string; said: string }
+const steps = new Map<string, Step>();
+const order: string[] = [];
+
+/** Run a step, remember what it said, and never throw. A step whose predecessor failed is skipped. */
+async function step(name: string, after_: string | null, fn: () => Promise<string>): Promise<void> {
+  order.push(name);
+  if (after_ && !steps.get(after_)?.ok) {
+    steps.set(name, { ok: false, why: `not attempted — "${after_}" did not succeed`, said: "" });
+    console.log(`SKIPPED  ${name}`);
+    return;
+  }
+  console.log(`\n======== ${name} ========`);
+  try {
+    const said = await fn();
+    steps.set(name, { ok: true, why: "", said });
+    console.log(`OK       ${name}`);
+  } catch (err) {
+    const why = (err as Error).message;
+    steps.set(name, { ok: false, why, said: "" });
+    console.log(`FAILED   ${name}\n${why.split("\n").slice(0, 25).join("\n")}`);
+  }
+}
+
+function report(name: string): string {
+  const s = steps.get(name);
+  if (!s) return `${name}: never ran`;
+  const lines = order.map((n) => {
+    const it = steps.get(n);
+    return `  ${it?.ok ? "PASS" : "FAIL"}  ${n}`;
+  });
+  return `${s.why}\n\nWhere this bed got to:\n${lines.join("\n")}`;
+}
+
+before(async () => {
+  if (skip) return;
+
+  const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
+    onProgress: (m) => console.log(`raise: ${m}`),
+    ...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
+  });
+  instanceId = bed.instanceId;
+  console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
+  console.log(`NOTHING WAS LOADED: this scenario names no images. Every image on every machine ` +
+    `below was pulled from the internet or built by the mesh.`);
+
+  // ---- 1. GENESIS -----------------------------------------------------------------------------
+  //
+  // The shared description, the same one `genesis-single` calls. The bundle is the TEMPLATE with
+  // nothing held: no image is pre-resolved, because none is here to resolve to.
+  await step("novox becomes a mesh of one, raised by the installer", null, async () => {
+    try {
+      raised = await genesis({
+        instanceId,
+        node: CONTROL,
+        installer: installer as string,
+        catalogDir,
+        // The broker's advertised address, corrected.
+        //
+        // The template hardcodes 192.0.2.10:5671 — the address of the anchor in the single-machine
+        // bed. A token carries this verbatim as the endpoint an enrolling node dials, so on a mesh
+        // whose anchor is somewhere else every node, including this one, would enrol against an
+        // address nothing answers on. The installer refuses to guess it and says so, which is
+        // right: it does not know what this machine is called from outside.
+        bundleTemplate: substrateBundle(bundle, []).replaceAll("192.0.2.10:5671", `${ANCHOR}:5671`),
+        registry: REGISTRY,
+        source,
+        sourceRef,
+        log: (m) => console.log(m),
+      });
+    } catch (err) {
+      throw new Error(`the installer never ran: ${(err as Error).message}`);
+    }
+    if (!raised.ok) throw new Error(`${raised.step || "no step named"}: ${raised.why}\n\n${raised.report.join("\n")}`);
+    return raised.report.join("\n");
+  });
+
+  // ---- 2. JOINING -----------------------------------------------------------------------------
+  //
+  // Host binary and a token. novox is NOT in this loop — the installer enrolled it, and enrolling
+  // it again would offer the mesh a second identity for a node it already knows.
+  await step("three machines join it across the household gateway",
+    "novox becomes a mesh of one, raised by the installer", async () => {
+    const said: string[] = [];
+    for (const machine of HOME_NODES) {
+      await mesh(`node add ${machine}`);
+      const token = tokenFrom(await mesh(`token issue --node ${machine}`));
+      const out = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
+      assert.match(out, new RegExp(`enrolled as ${machine}`), out);
+      await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
+      said.push(`  ${machine} enrolled and running`);
+    }
+    const nodes = await mesh("node list");
+    said.push(nodes.trim());
+    return said.join("\n");
+  });
+
+  // ---- 3. THE MESH BUILDS THE SHARED BASE -----------------------------------------------------
+  //
+  // The toolchain and runtime every module with code of its own stands on. It is a module, and it
+  // is built like one — cloned from the forge by the builder installing put here, compiled on the
+  // machine, published into the mesh's own registry.
+  await step("the mesh builds the shared base from source",
+    "three machines join it across the household gateway", async () => {
+    assert.ok(existsSync(baseManifest),
+      `no manifest for the shared base at ${baseManifest} — set MESH_LAB_BASE_MANIFEST`);
+    await push(instanceId, CONTROL, baseManifest, `/tmp/${BASE.module}.json`);
+    // Registered from the manifest the builder will also read, so what the mesh holds and what it
+    // builds are the same description of the same module.
+    await mesh(`module add /tmp/${BASE.module}.json`).catch(() => {});
+    const built = await mesh(
+      `build ${forgeUrl(BASE.repo)} --ref ${buildRef} --wait 1200s`, 1_500_000);
+    assert.doesNotMatch(built, /failed/i, built);
+    return built;
+  });
+
+  // ---- 4. AND A MODULE STANDING ON IT ---------------------------------------------------------
+  await step("the mesh builds a module standing on that base",
+    "the mesh builds the shared base from source", async () => {
+    const manifest = resolve(catalogDir, MODULE.module, "module.json");
+    assert.ok(existsSync(manifest), `no manifest at ${manifest}`);
+    await push(instanceId, CONTROL, manifest, `/tmp/${MODULE.module}.json`);
+    await mesh(`module add /tmp/${MODULE.module}.json`);
+    const built = await mesh(
+      `build ${forgeUrl(MODULE.repo)} --path ${MODULE.path} --ref ${buildRef} --wait 1200s`,
+      1_500_000);
+    assert.doesNotMatch(built, /failed/i, built);
+    // The point of the whole step: what came out is named by a digest this mesh's registry
+    // assigned, not by a placeholder and not by a tag.
+    const builds = await mesh(`builds ${MODULE.module}`);
+    assert.match(builds, /sha256:[0-9a-f]{12}/,
+      `the build recorded no digest — the module is not pinned to anything this registry serves:\n${builds}`);
+    return `${built}\n${builds}`;
+  });
+
+  // ---- 5. THE ANCHOR RUNS IT ------------------------------------------------------------------
+  //
+  // The machine that built it. This is the case every earlier proof covered, and it is here as the
+  // control for step 6: if this fails, step 6's failure says nothing about fetching.
+  await step("the anchor runs the module the mesh built",
+    "the mesh builds a module standing on that base", async () => {
+    await mesh(`module issue ${MODULE.module} --node ${CONTROL}`).catch(() => {});
+    await mesh(`assign ${CONTROL} ${MODULE.module}`);
+    await mesh(`push ${CONTROL}`, 600_000);
+    for (let i = 0; i < 40; i++) {
+      const ps = (await on(CONTROL, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out;
+      if (/amqp-ping/.test(ps) && /Up /.test(ps.split("\n").find((l) => l.includes("amqp-ping")) ?? "")) {
+        return ps;
+      }
+      await new Promise((r) => setTimeout(r, 5_000));
+    }
+    throw new Error(`amqp-ping never came up on ${CONTROL}:\n` +
+      (await on(CONTROL, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out);
+  });
+
+  // ---- 6. AND A MACHINE THAT DID NOT BUILD IT -------------------------------------------------
+  //
+  // **The thing nothing has ever checked.** ace did not build this image and has never seen it. To
+  // run it, it must fetch it from the mesh's registry — and a joined node has no account there.
+  // The mesh grants a consumer a credential for a database; it does not yet do so for the store
+  // its own images live in (novox/hq issue 042).
+  //
+  // Asked anyway, and asked LAST, so that when it fails the five steps above still stand as
+  // evidence of what does work.
+  await step(`a joined machine runs the module the mesh built`,
+    "the anchor runs the module the mesh built", async () => {
+    await mesh(`module issue ${MODULE.module} --node ${SECOND}`).catch(() => {});
+    await mesh(`assign ${SECOND} ${MODULE.module}`);
+    await mesh(`push ${SECOND}`, 600_000);
+    for (let i = 0; i < 40; i++) {
+      const ps = (await on(SECOND, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out;
+      const line = ps.split("\n").find((l) => l.includes("amqp-ping"));
+      if (line && /Up /.test(line)) return ps;
+      await new Promise((r) => setTimeout(r, 5_000));
+    }
+    const state = (await on(SECOND, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
+    const log = (await on(SECOND, `tail -40 /var/log/mesh-host.log`)).out;
+    throw new Error(
+      `amqp-ping never came up on ${SECOND} — the machine that did NOT build it.\n\n` +
+      `containers:\n${state}\n\nwhat the host said:\n${log}`);
+  });
+
+  console.log(`\n================ WHAT THIS MESH DID FOR ITSELF ================`);
+  for (const n of order) console.log(`  ${steps.get(n)?.ok ? "PASS" : "FAIL"}  ${n}`);
+}, { timeout: 7_200_000 });
+
+after(async () => {
+  if (KEEP) {
+    console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
+    return;
+  }
+  if (instanceId) await destroy(instanceId);
+  await destroyAll(`${SCENARIO}-`);
+}, { timeout: 900_000 });
+
+for (const name of [
+  "novox becomes a mesh of one, raised by the installer",
+  "three machines join it across the household gateway",
+  "the mesh builds the shared base from source",
+  "the mesh builds a module standing on that base",
+  "the anchor runs the module the mesh built",
+  "a joined machine runs the module the mesh built",
+]) {
+  test(name, { skip, timeout: 60_000 }, () => {
+    assert.ok(steps.get(name)?.ok, report(name));
+  });
+}