/**
 * FOUR FRESH MACHINES, AND NOTHING HANDED TO THEM.
 *
 * The four-machine bed (`whole-mesh-full`) proves the mesh converges. It does so by loading
 * thirty-four of the mesh's own images onto its machines from the workstation, because it does not
 * build them — something beside the bed built them and copied them in. No real installation looks
 * like that, and the lab has been burned by exactly this shape before: it used to raise a registry
 * inside the scenario, and a bootstrap that only worked against that registry went green here and
 * would have failed on any bare machine.
 *
 * This bed hands over nothing. The scenario has no `images:` list at all. What the machines get is
 * a container runtime and the host binary — prerequisites of a machine, not parts of a mesh — and
 * from there:
 *
 *   1. novox is raised into a mesh of one by the installer, which BUILDS the control plane.
 *   2. ace, shanks and g14 JOIN it, across a household NAT, with a token and nothing else.
 *   3. The mesh builds the shared base from source, with its own builder.
 *   4. The mesh builds a real module standing on that base.
 *   5. The anchor runs it, pinned to a digest the mesh's own registry assigned.
 *   6. A JOINED machine runs it — which means pulling from a registry that asks who it is.
 *
 * Steps 1 and 2 are proven elsewhere and are here because the later ones need them. **Steps 3
 * through 6 are what this bed exists for**, and 6 is the one nothing has ever checked: genesis
 * puts the builder, the registry and everything they produce on ONE machine, so every earlier
 * proof of a mesh-built module running is a proof about the machine that built it. A second
 * machine has to fetch, and fetching needs an account nothing yet grants (novox/hq issue 042).
 *
 * Each step is recorded separately rather than allowed to throw, so a gap at 6 reports as a gap at
 * 6 instead of erasing the evidence for 3, 4 and 5.
 *
 *   MESH_LAB_INCUS='sudo -n incus'
 *   MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
 *   MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap
 *   MESH_LAB_BUNDLE=.../mesh-host/examples/substrate-first-node.lock
 *   MESH_LAB_CATALOG=.../mesh-catalog/modules
 *   MESH_LAB_SOURCE=<forge>/mesh-control.git   MESH_LAB_SOURCE_REF=<commit>
 *   MESH_LAB_KEEP=1   to leave it standing afterwards
 */
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync } from "node:fs";
import { resolve } from "node:path";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts";
import { genesis, type GenesisResult } from "./genesis.ts";
import { incus } from "../../src/incus/client.ts";
import { instanceNameOf } from "../../src/lifecycle/operate.ts";
import { waitUntilAllUsable } from "../../src/lifecycle/ready.ts";

const SCENARIO = "one-node-mesh";
const CONTROL = "anchor";

/** The anchor's public address — what every other machine dials, and what its own token must name. */
const ANCHOR = "192.0.2.10";
/** Where this mesh's registry answers, on the anchor's public address so a joined node can reach it. */
const REGISTRY = `${ANCHOR}:5000`;

/**
 * The module built on top of the base, and the base it stands on.
 *
 * `amqp-ping` is deliberately small and deliberately REAL: its own TypeScript, compiled by the
 * shared toolchain, running on the shared runtime, talking to the broker. A module whose artifact
 * is a mirrored public image would pass every assertion below while skipping the whole of what is
 * under test (novox/hq SELF-UPGRADE-PLAN, rule 1).
 */
const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" };
/**
 * What `amqp-ping` requires, and what the substrate does not supply.
 *
 * The installer raises a broker, but as a bundle resource — plumbing, not a module the mesh has a
 * record of, so it provides nothing to anything. A module asking for `amqp` is asking for a
 * provider in the graph, and this is it. No build: its image is upstream.
 */
const PROVIDER = { module: "lavinmq", repo: "mesh-catalog", path: "modules/lavinmq", container: "mesh-lavinmq" };
/**
 * The store, and the catalogue that cannot exist without it.
 *
 * Both were missing from this test entirely, and nothing complained, because nothing asked. A mesh
 * with no catalogue holds no module graph — it cannot say what it has, what a module is made of,
 * what a change reaches, or what must be rebuilt. It ran anyway, which is the point: "the mesh is
 * up" was being read off genesis finishing.
 */
const STORE = { module: "postgres", repo: "mesh-catalog", path: "modules/postgres", container: "mesh-postgres" };
const CATALOGUE = { module: "mesh-catalog", repo: "mesh-catalog", path: "modules/mesh-catalog", container: "mesh-catalog" };
/** The control plane, rebuilt from its own repository — the step that ends the installer's tenure. */
const CONTROL_PLANE = { module: "mesh-control", repo: "mesh-control", path: "", container: "mesh-control" };

/**
 * What this mesh must hold when it is finished, and what must be RUNNING on the machine.
 *
 * Written down as a list rather than checked one step at a time, because "is this a mesh" is a
 * question about the set. The three the build loop cannot produce for itself — the control plane,
 * the registry and the builder — are here too: they are carried in, and a mesh missing any of them
 * is not one.
 */
const MUST_HOLD = ["mesh-control", "registry", "builder", "mesh-tools", "postgres",
  "mesh-catalog", "lavinmq", "amqp-ping"];
const MUST_RUN = ["mesh-control", "mesh-registry", "mesh-broker", "mesh-store",
  "mesh-postgres", "mesh-catalog", "mesh-lavinmq", "amqp-ping"];
/** Named once, because the step title is also how later steps say what they waited on. */
const NEEDS = "the mesh runs a broker for that module to talk to";
const GENESIS = "a bare machine becomes a mesh of one, raised by the installer";
const BASE_BUILT = "the mesh builds the shared base from source";
const STORE_RUNS = "the mesh builds and runs a store of its own";
const CATALOGUE_RUNS = "the mesh builds and runs its own catalogue";
const CONTROL_REBUILT = "the mesh rebuilds its own control plane from source";
const MODULE_BUILT = "the mesh builds a module standing on that base";
const ANCHOR_RUNS = "the anchor runs the module the mesh built";
const DESCRIBES = "the mesh can describe itself, and what it says is true";
const NETWORK = "the machine's networking is what the modules asked for";
const FOLLOWS = "a change to a module's source reaches the machine on its own";
const SURVIVES = "the mesh comes back after the machine reboots";
const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping" };

const capability = await labIsUsable();
const binary = hostBinaryPath();
const installer = bootstrapBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
const source = process.env["MESH_LAB_SOURCE"] ?? "";
const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? "";
const KEEP = !!process.env["MESH_LAB_KEEP"];
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "fresh-mesh-live" : undefined);

/**
 * Where a repository other than the control plane's lives.
 *
 * Derived from `MESH_LAB_SOURCE` by swapping the last path segment, because every one of these
 * repositories sits beside the others under the same owner on the same forge. Overridable, so a
 * forge that is arranged differently does not need this bed edited.
 */
function forgeUrl(repo: string): string {
  const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`];
  if (override) return override;
  return source.replace(/[^/]+\.git$/, `${repo}.git`);
}
/**
 * What to build, per repository.
 *
 * A branch is acceptable for an ordinary build; only genesis insists on a commit (ADR 0071). Per
 * repository rather than one value for all of them, because a change under test usually lives in
 * one repository and the rest should be built from what everyone else has — building them all from
 * a feature branch would prove that branch against itself.
 *
 *   MESH_LAB_BUILD_REF                 the default for every repository
 *   MESH_LAB_BUILD_REF_MESH_CATALOG    ...overridden for one
 */
function refFor(repo: string): string {
  const override = process.env[`MESH_LAB_BUILD_REF_${repo.toUpperCase().replaceAll("-", "_")}`];
  return override ?? process.env["MESH_LAB_BUILD_REF"] ?? "main";
}

/**
 * The shared base's manifest, on this workstation.
 *
 * The base is a repository with a manifest at its root (novox/hq ADR 0069), so unlike the
 * catalogue's modules it is not under `MESH_LAB_CATALOG`. Derived from that path on the convention
 * that the checkouts sit beside each other, and overridable for a layout where they do not.
 */
const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ??
  resolve(catalogDir, "..", "..", BASE.repo, "module.json");

const skip =
  !capability.usable ? capability.why :
  !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
  !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
  !source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" :
  !sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" :
  !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate template" :
  !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
  false;

let instanceId = "";
let raised: GenesisResult;

// ---- talking to the machines ------------------------------------------------------------------

function quote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
  const { stdout } = await exec(instanceId, machine, [
    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
  ], timeoutMs);
  const marker = stdout.lastIndexOf("__exit=");
  if (marker < 0) return { out: stdout, ok: false };
  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
  const { out, ok } = await on(machine, command, timeoutMs);
  if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
  return out;
}
async function mesh(command: string, timeoutMs?: number): Promise<string> {
  return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}

/**
 * Stop the machine and start it again, the way a power cut or a kernel upgrade would.
 *
 * There is no restart in the lab's own vocabulary, which is its own small finding: nothing had ever
 * needed one, because nothing had ever asked whether a mesh comes back.
 */
async function restartMachine(machine: string): Promise<void> {
  const name = await instanceNameOf(instanceId, machine);
  await incus(["restart", name], 180_000);
  await waitUntilAllUsable([name], 300, (m) => console.log(`  restart: ${m}`));
}

/** A module the mesh has to make for itself: where its source is, and what it runs when it works. */
interface CoreModule { module: string; repo: string; path: string; container: string }

/**
 * Build a module from source, install it, and wait for it to actually run.
 *
 * The whole sequence a module goes through, in one place because the mesh has to do it for several
 * before it is a mesh at all: register the manifest, build it from its repository and path, issue
 * the broker account its runtime needs, assign it, send it, and then ask the machine whether the
 * container is up.
 *
 * **The account is not optional and is not swallowed.** A module's runtime is a tool host: it
 * connects to the mesh's broker before doing anything. Without an account the mesh still fills the
 * secret the module declares it owns — with a generated value — so the container starts, fails to
 * parse a password as a credential document, and loops on a JSON syntax error mentioning no
 * missing account. That cost a step that reported PASS.
 */
async function bringUp(m: CoreModule): Promise<string> {
  await registerModule(m.module, resolve(catalogDir, m.module, "module.json"));
  const built = await mesh(
    `build ${forgeUrl(m.repo)} --path ${m.path} --ref ${refFor(m.repo)} --wait 1200s`, 1_500_000);
  assert.doesNotMatch(built, /failed/i, built);
  await mesh(`module issue ${m.module} --node ${CONTROL}`);
  await mesh(`assign ${CONTROL} ${m.module}`);
  await mesh(`push ${CONTROL}`, 600_000);
  return `${built}\n${await waitForContainer(CONTROL, m.container)}`;
}

/**
 * Wait for one container, BY NAME, to be running.
 *
 * **Named exactly, because substring matching passed a step that had failed.** Waiting for "a
 * container whose name contains lavinmq" was satisfied by the broker — `lavinmq`, up and healthy —
 * while the thing actually under test, the module's own runtime `mesh-lavinmq`, was crash-looping
 * beside it. The step went green and the fault was found by reading `docker ps` by hand.
 */
async function waitForContainer(node: string, container: string, seconds = 200): Promise<string> {
  const deadline = Date.now() + seconds * 1_000;
  let last = "";
  while (Date.now() < deadline) {
    const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
    last = ps;
    const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container);
    if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps;
    await new Promise((r) => setTimeout(r, 5_000));
  }
  const logs = (await on(node, `docker logs --tail 15 ${container} 2>&1`)).out;
  throw new Error(
    `${container} is not running on ${node}.\n\ncontainers:\n${last}\n\nwhat it said:\n${logs}`);
}

/**
 * Register a module from a manifest on this workstation.
 *
 * **The control plane runs in a container, so a file on the machine is not a file it can open.**
 * Pushing the manifest to the machine and naming that path got `no such file or directory` from
 * inside mesh-control, which is correct and was briefly mistaken for a missing manifest. It is
 * copied the last step of the way with `docker cp`.
 *
 * **Into the root, not into /tmp.** The control plane's image is a minimal one and has no `/tmp`
 * to copy into — `docker cp` says so in those words. `/` is the one directory every image has.
 */
async function registerModule(module: string, manifest: string): Promise<string> {
  assert.ok(existsSync(manifest), `no manifest for ${module} at ${manifest}`);
  const onMachine = `/tmp/${module}.json`;
  const inContainer = `/${module}.json`;
  await push(instanceId, CONTROL, manifest, onMachine);
  await must(CONTROL, `docker cp ${onMachine} mesh-control:${inContainer}`);
  return mesh(`module add ${inContainer}`);
}
function tokenFrom(said: string): string {
  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
  assert.ok(found, `no token in:\n${said}`);
  return found;
}

// ---- steps, recorded rather than thrown --------------------------------------------------------

interface Step { ok: boolean; why: string; said: string }
const steps = new Map<string, Step>();
const order: string[] = [];

/** Run a step, remember what it said, and never throw. A step whose predecessor failed is skipped. */
async function step(name: string, after_: string | null, fn: () => Promise<string>): Promise<void> {
  order.push(name);
  if (after_ && !steps.get(after_)?.ok) {
    steps.set(name, { ok: false, why: `not attempted — "${after_}" did not succeed`, said: "" });
    console.log(`SKIPPED  ${name}`);
    return;
  }
  console.log(`\n======== ${name} ========`);
  try {
    const said = await fn();
    steps.set(name, { ok: true, why: "", said });
    console.log(`OK       ${name}`);
  } catch (err) {
    const why = (err as Error).message;
    steps.set(name, { ok: false, why, said: "" });
    console.log(`FAILED   ${name}\n${why.split("\n").slice(0, 25).join("\n")}`);
  }
}

function report(name: string): string {
  const s = steps.get(name);
  if (!s) return `${name}: never ran`;
  const lines = order.map((n) => {
    const it = steps.get(n);
    return `  ${it?.ok ? "PASS" : "FAIL"}  ${n}`;
  });
  return `${s.why}\n\nWhere this bed got to:\n${lines.join("\n")}`;
}

before(async () => {
  if (skip) return;

  const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
    onProgress: (m) => console.log(`raise: ${m}`),
    ...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
  });
  instanceId = bed.instanceId;
  console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
  console.log(`NOTHING WAS LOADED: this scenario names no images. Every image on every machine ` +
    `below was pulled from the internet or built by the mesh.`);

  // ---- 1. GENESIS -----------------------------------------------------------------------------
  //
  // The shared description, the same one `genesis-single` calls. The bundle is the TEMPLATE with
  // nothing held: no image is pre-resolved, because none is here to resolve to.
  await step("a bare machine becomes a mesh of one, raised by the installer", null, async () => {
    try {
      raised = await genesis({
        instanceId,
        node: CONTROL,
        installer: installer as string,
        catalogDir,
        // The broker's advertised address, corrected.
        //
        // The template hardcodes 192.0.2.10:5671 — the address of the anchor in the single-machine
        // bed. A token carries this verbatim as the endpoint an enrolling node dials, so on a mesh
        // whose anchor is somewhere else every node, including this one, would enrol against an
        // address nothing answers on. The installer refuses to guess it and says so, which is
        // right: it does not know what this machine is called from outside.
        bundleTemplate: substrateBundle(bundle, []).replaceAll("192.0.2.10:5671", `${ANCHOR}:5671`),
        registry: REGISTRY,
        source,
        sourceRef,
        log: (m) => console.log(m),
      });
    } catch (err) {
      throw new Error(`the installer never ran: ${(err as Error).message}`);
    }
    if (!raised.ok) throw new Error(`${raised.step || "no step named"}: ${raised.why}\n\n${raised.report.join("\n")}`);
    return raised.report.join("\n");
  });

  // ---- 2..6. THE MESH BECOMES ONE --------------------------------------------------------------
  //
  // **Joining used to be here, second, and that was the wrong order.** Three machines were enrolled
  // into a mesh that could not yet produce a single module, and the step was reported as though
  // something had been shown. They do join — reliably — but joining a mesh that can build nothing
  // proves only that enrolment works, which was never the doubtful part.
  //
  // `17-raising-a-mesh` says it plainly: genesis ends with a mesh of one that RUNS, and that is not
  // the same as a mesh that WORKS; what remains after the core modules are built is "adding
  // machines, and deciding what they run". So everything a mesh needs to be a mesh happens first,
  // and machines arrive at the end.

  // The toolchain and runtime every module with code of its own stands on. It is a module, and it
  // is built like one — cloned from the forge by the builder installing put here, compiled on the
  // machine, published into the mesh's own registry.
  await step(BASE_BUILT, GENESIS, async () => {
    // Registered from the manifest the builder will also read, so what the mesh holds and what it
    // builds are the same description of the same module.
    //
    // **Not swallowed.** This call used to end in `.catch(() => {})`, on the reasoning that the
    // base might already be known. It hid a real failure — the manifest was being named at a path
    // inside a container that had never seen it — and the step passed anyway, because a base with
    // nothing to stand on builds whether or not the mesh has a record of it.
    await registerModule(BASE.module, baseManifest);
    const built = await mesh(
      `build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000);
    assert.doesNotMatch(built, /failed/i, built);
    return built;
  });

  // A store of its own. **Not the substrate's.** The installer raises a store for the control
  // plane to keep its own records in, the way it raises a broker — plumbing, not a module the mesh
  // has any record of, so it provides nothing to anything. A module that wants a database wants a
  // provider in the graph, and the catalogue below is the first thing to want one.
  await step(STORE_RUNS, BASE_BUILT, () => bringUp(STORE));

  // And the catalogue, which was missing from this test altogether.
  //
  // Without it the mesh holds no module graph: it cannot say what it has, what a module is made
  // of, what a change to one reaches, or what must be rebuilt. A mesh in that state still runs,
  // which is exactly how its absence went unnoticed — "the mesh is up" was being read off the
  // installer finishing rather than off the mesh being able to answer anything.
  await step(CATALOGUE_RUNS, STORE_RUNS, () => bringUp(CATALOGUE));

  // The control plane, rebuilt from its own repository and rolled out.
  //
  // The installer built it once, which is what got the mesh running. Building it again THROUGH THE
  // MODULE PATH — build, notice the version moved, roll it out — is a different claim: it is the
  // moment the mesh stops depending on the installer for anything, and the first time the thing
  // that performs an upgrade performs one on itself.
  await step(CONTROL_REBUILT, CATALOGUE_RUNS, async () => {
    const built = await mesh(
      `build ${forgeUrl(CONTROL_PLANE.repo)} --ref ${sourceRef} --wait 1200s`, 1_500_000);
    assert.doesNotMatch(built, /failed/i, built);
    const rolled = await mesh(`upgrade ${CONTROL_PLANE.module} roll-out`, 900_000);
    // Asked of the machine rather than believed from the command: the control plane that answers
    // afterwards is the one that has to be running for anything below this line to mean anything.
    await waitForContainer(CONTROL, CONTROL_PLANE.container);
    return `${built}\n${rolled}`;
  });

  // ---- 7..8. SOMETHING TO RUN ---------------------------------------------------------------
  await step(MODULE_BUILT, CONTROL_REBUILT, async () => {
    await registerModule(MODULE.module, resolve(catalogDir, MODULE.module, "module.json"));
    const built = await mesh(
      `build ${forgeUrl(MODULE.repo)} --path ${MODULE.path} --ref ${refFor(MODULE.repo)} --wait 1200s`,
      1_500_000);
    assert.doesNotMatch(built, /failed/i, built);
    // The point of the whole step: what came out is named by a digest this mesh's registry
    // assigned, not by a placeholder and not by a tag.
    const builds = await mesh(`builds ${MODULE.module}`);
    assert.match(builds, /sha256:[0-9a-f]{12}/,
      `the build recorded no digest — the module is not pinned to anything this registry serves:\n${builds}`);
    return `${built}\n${builds}`;
  });

  // `amqp-ping` requires the `amqp` provision, and the mesh refused to place it: "nothing provides
  // amqp, wanted by amqp-ping". That refusal is correct and is the reason this step exists rather
  // than the reason to pick an easier module.
  //
  // `lavinmq` is that module. It was first added here on the belief that it needed no building —
  // its broker is an upstream image — and the mesh refused it again: two of its three containers
  // named a placeholder digest, "which is never a real image". Also right. The broker is upstream,
  // but the module is not only the broker: it carries a run-once bootstrap that writes the broker's
  // configuration, a provisioner that grants each consumer its own vhost and user, tools and an
  // event consumer, all of it its own code.
  await step(NEEDS, MODULE_BUILT, () => bringUp(PROVIDER));

  // The machine that built it. This is the case every earlier proof covered, and it is here as the
  // control for the last step: if this fails, that one's failure says nothing about fetching.
  await step(ANCHOR_RUNS, NEEDS, async () => {
    await mesh(`module issue ${MODULE.module} --node ${CONTROL}`);
    await mesh(`assign ${CONTROL} ${MODULE.module}`);
    await mesh(`push ${CONTROL}`, 600_000);
    return waitForContainer(CONTROL, MODULE.module);
  });

  // ---- 9. IT CAN DESCRIBE ITSELF ---------------------------------------------------------------
  //
  // **Presence is not function, and this step exists because I kept confusing them.** A container
  // being up was taken as the catalogue working; a name containing "lavinmq" was taken as the
  // module running. The mesh holds a graph — nodes, what each is assigned, what capabilities each
  // has, which claims are occupied, what each module is configured with, which provisions exist and
  // who holds them — and none of it was ever asked a question.
  await step(DESCRIBES, ANCHOR_RUNS, async () => {
    const said: string[] = [];

    // The mesh's own verdict on itself. Nothing wrong, nothing waiting, nothing behind.
    const state = JSON.parse(await mesh("status --json")) as {
      wrong: { node: string; outcome: string; refused?: string }[];
      waiting: { node: string }[];
      reported: { node: string; outcome: string; current: boolean }[];
    };
    assert.equal(state.wrong.length, 0,
      `the mesh reports something wrong:\n${JSON.stringify(state.wrong, null, 2)}`);
    assert.equal(state.waiting.length, 0,
      `the mesh is waiting on a node:\n${JSON.stringify(state.waiting, null, 2)}`);
    const node = state.reported.find((r) => r.node === CONTROL);
    assert.ok(node, `the mesh does not report the only machine it has:\n${JSON.stringify(state)}`);
    assert.equal(node.outcome, "applied", `${CONTROL} did not apply what it was sent: ${node.outcome}`);
    assert.ok(node.current, `${CONTROL} is not running what the mesh would send it`);
    said.push(`  status            one node, applied, current, nothing wrong`);

    // Every module a mesh has to hold, including the three it cannot build for itself.
    const modules = await mesh("module list");
    for (const m of MUST_HOLD) {
      assert.match(modules, new RegExp(`^${m}\\b`, "m"),
        `the mesh holds no ${m}. A mesh without it is not finished:\n${modules}`);
    }
    said.push(`  module list       ${MUST_HOLD.length} modules, all present`);

    // What the machine would be sent, and what it names. **No placeholder may survive here** — a
    // digest of all zeroes is never a real image, and a declaration carrying one reaches a machine
    // that will try to fetch it.
    const plan = await mesh(`plan ${CONTROL} --json`);
    assert.doesNotMatch(plan, /sha256:0{64}/,
      `the machine's own plan names a placeholder digest, which is never a real image`);
    assert.match(plan, /sha256:[0-9a-f]{64}/, `the plan pins nothing by digest at all`);
    said.push(`  plan              every image pinned, no placeholders`);

    // And the catalogue, ASKED rather than observed. These five questions are what it exists for.
    const ask = async (tool: string, args = "{}") =>
      must(CONTROL, `docker exec mesh-catalog mesh-tools invoke mesh-catalog ${tool} ${quote(args)}`,
        120_000);

    const held = await ask("catalog_modules");
    for (const m of MUST_HOLD) {
      if (m === "registry" || m === "builder" || m === "mesh-control") continue; // carried, not built here
      assert.ok(held.includes(m), `the catalogue does not know about ${m}:\n${held}`);
    }
    assert.doesNotMatch(held, /sha256:0{64}/, `the catalogue holds a placeholder version`);
    said.push(`  catalog_modules   every built module, each with a version this mesh made`);

    const provides = await ask("catalog_provides", JSON.stringify({ provision: "amqp" }));
    assert.ok(provides.includes(PROVIDER.module),
      `the catalogue cannot say what provides amqp, which is the question it exists to answer:\n${provides}`);
    said.push(`  catalog_provides  amqp is answered by ${PROVIDER.module}`);

    const stale = await ask("catalog_stale");
    said.push(`  catalog_stale     ${stale.trim().slice(0, 120)}`);
    return said.join("\n");
  });

  // ---- 10. AND ITS NETWORKING IS WHAT WAS ASKED FOR ---------------------------------------------
  //
  // **Left out of this test entirely until it was pointed out**, which is hard to defend: the
  // firewall is generated from what modules declare they listen on, and a firewall that opens the
  // wrong set is either a service nobody can reach or a port nobody meant to publish. Neither shows
  // up as a failed container.
  await step(NETWORK, DESCRIBES, async () => {
    const said: string[] = [];

    const ruleset = (await on(CONTROL, `nft list table inet mesh 2>&1`)).out;
    assert.match(ruleset, /chain input/, `the mesh's own firewall table is not there:\n${ruleset}`);
    // Closed by default, or the rules below decide nothing.
    assert.match(ruleset, /policy drop/, `the firewall does not default to closed:\n${ruleset}`);
    // The floor: a machine that cannot be reached over ssh is a machine nobody can repair.
    assert.match(ruleset, /\b22\b/, `ssh is not allowed anywhere in the ruleset`);
    // What a module actually declared. The registry says it listens on 5000 for the mesh.
    assert.match(ruleset, /\b5000\b/,
      `the registry declares it listens on 5000 and nothing opened it:\n${ruleset}`);
    said.push(`  firewall          default closed, ssh open, declared ports open`);

    // The names the mesh writes for itself. A consumer reaching a provider by its `.internal`
    // address depends on this file, and on it reaching inside containers.
    const hosts = (await on(CONTROL, `cat /etc/hosts`)).out;
    assert.match(hosts, /\.internal/, `the mesh wrote no .internal names:\n${hosts}`);
    said.push(`  names             ${(hosts.match(/[a-z0-9-]+\.internal/g) ?? []).join(" ")}`);

    // The networks the declarations asked for, rather than whatever the runtime had lying around.
    const networks = (await on(CONTROL, `docker network ls --format '{{.Name}}'`)).out;
    for (const wanted of ["lavinmq", "amqp-ping"]) {
      assert.match(networks, new RegExp(`^${wanted}$`, "m"),
        `the ${wanted} module declares a network and none exists:\n${networks}`);
    }
    said.push(`  networks          module networks present`);
    return said.join("\n");
  });

  // ---- 11. A CHANGE REACHES THE MACHINE ON ITS OWN ----------------------------------------------
  //
  // The whole point of the mesh, and the capability the migration depends on: move a module's
  // source and the running copy follows, with nobody driving the steps. Everything above is
  // machinery; this is what the machinery is for.
  await step(FOLLOWS, NETWORK, async () => {
    const before = await mesh(`builds ${MODULE.module}`);
    const wasPinned = before.match(/sha256:[0-9a-f]{64}/)?.[0] ?? "";
    assert.ok(wasPinned, `nothing is pinned to rebuild from:\n${before}`);

    // The mesh is told its copy is older than the source. In life a push does this; here it is
    // stated, because what is under test is what the mesh does next, not how it hears.
    const head = (await must(CONTROL, `git ls-remote ${forgeUrl(MODULE.repo)} ` +
      `${refFor(MODULE.repo)} | cut -f1`, 120_000)).trim();
    assert.match(head, /^[0-9a-f]{40}$/, `could not read the source's head: ${head}`);
    await mesh(`module moved ${MODULE.module} ${head}`);

    const behind = await mesh(`status`);
    assert.match(behind, /behind|build --behind/,
      `the mesh does not report a module behind its source:\n${behind}`);

    await mesh(`build --behind --wait 1200s`, 1_500_000);
    const rolled = await mesh(`upgrade ${MODULE.module} roll-out`, 900_000);
    await waitForContainer(CONTROL, MODULE.module);

    const after = await mesh(`builds ${MODULE.module}`);
    assert.match(after, /sha256:[0-9a-f]{64}/, `nothing was pinned after the rebuild:\n${after}`);
    return `${behind}\n${rolled}\n${after}`;
  });

  // ---- 12. AND IT SURVIVES THE MACHINE STOPPING -------------------------------------------------
  //
  // **Never once tested.** A mesh that works until the machine reboots is a demonstration, not
  // something to move real services onto — and the installer is explicit that a host started the
  // way the lab starts it does not survive a reboot, which makes this the check that says whether
  // that matters.
  await step(SURVIVES, FOLLOWS, async () => {
    await restartMachine(CONTROL);
    const missing: string[] = [];
    for (const container of MUST_RUN) {
      try {
        await waitForContainer(CONTROL, container, 240);
      } catch {
        missing.push(container);
      }
    }
    const ps = (await on(CONTROL, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
    assert.equal(missing.length, 0,
      `after a reboot these are not running: ${missing.join(", ")}\n\ncontainers:\n${ps}`);
    return ps;
  });

  console.log(`\n================ WHAT THIS MESH DID FOR ITSELF ================`);
  for (const n of order) console.log(`  ${steps.get(n)?.ok ? "PASS" : "FAIL"}  ${n}`);
}, { timeout: 7_200_000 });

after(async () => {
  if (KEEP) {
    console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
    return;
  }
  if (instanceId) await destroy(instanceId);
  await destroyAll(`${SCENARIO}-`);
}, { timeout: 900_000 });

for (const name of [
  GENESIS,
  BASE_BUILT,
  STORE_RUNS,
  CATALOGUE_RUNS,
  CONTROL_REBUILT,
  MODULE_BUILT,
  NEEDS,
  ANCHOR_RUNS,
  DESCRIBES,
  NETWORK,
  FOLLOWS,
  SURVIVES,
]) {
  test(name, { skip, timeout: 60_000 }, () => {
    assert.ok(steps.get(name)?.ok, report(name));
  });
}