mesh-lab/test/integration/genesis.ts

/**
 * GENESIS — a machine with no mesh becomes a mesh of one, by running the installer.
 *
 * This is the mesh's own install procedure, exercised the way an operator runs it: the same
 * program a bare machine runs, given the same inputs, checked by asking the machine rather than
 * by trusting the installer's exit code (novox/hq ADR 0018, ADR 0067).
 *
 * It lives here, shared, for one reason. An install procedure that exists only inside one bed is
 * exercised by whoever writes that bed and never by whoever installs, and a second bed describing
 * it differently is the arrangement that already failed — a fixture invented a registry that
 * exists in no production and hid two separate faults for as long as it existed. There is one
 * description of genesis, and both the single-node bed and the four-node bed call it.
 */
import { existsSync, writeFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts";
import { placeBootstrap, BOOTSTRAP_PATH, HOST_PATH } from "../../src/lifecycle/place.ts";

/** What genesis did, or where it stopped. */
export interface GenesisResult {
  ok: boolean;
  /** `step 7 of 10, registry` — the installer's own words, so a bed reports the cause. */
  step: string;
  why: string;
  report: string[];
}

export interface GenesisOptions {
  instanceId: string;
  /** The machine being raised into a mesh of one. */
  node: string;
  /** Path to the built `mesh-bootstrap` binary on this workstation. */
  installer: string;
  /** A checkout of mesh-catalog's `modules/` on this workstation. */
  catalogDir: string;
  /**
   * The substrate TEMPLATE's content — not a bundle. The installer produces the bundle from it,
   * replacing the control plane's image with the id of the image it carries.
   */
  bundleTemplate: string;
  /** Manifests the installer reads. Only these two: it opens no others. */
  catalogueModules?: string[];
  catalogueOnMachine?: string;
  /** Where this mesh's own registry will answer. */
  registry?: string;
  /**
   * Where the control plane is built from, and the commit.
   *
   * The installer carries a builder rather than a finished control plane (novox/hq ADR 0073), so
   * it has to be told what to make. A commit rather than a branch, because what genesis clones is
   * the trust anchor for everything the mesh will ever run (ADR 0071).
   */
  source: string;
  sourceRef: string;
  /**
   * Phase two: where the shared base and the catalogue's modules are built from.
   *
   * The installer no longer stops at a mesh that runs — it builds the base, a store, the
   * catalogue, chooses the network and the filter, and asks a human where one must choose. This
   * bed has no human, so every choice arrives as a flag, and a required choice with no flag is
   * the installer refusing — which is the behaviour, not a lab problem.
   */
  toolsSource: string;
  catalogSource: string;
  /** What of the catalogue to build. A branch under test is the usual reason this is not main. */
  catalogRef?: string;
  log?: (m: string) => void;
}

/** The step a failed `mesh-bootstrap` names, as it prints it, or "" if it named none. */
export function stepIn(said: string): string {
  return said.match(/^mesh-bootstrap: (step \d+ of \d+, [a-z-]+):/m)?.[1] ?? "";
}

export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
  const node = o.node;
  const registry = o.registry ?? "127.0.0.1:5000";
  const modules = o.catalogueModules ?? ["distribution", "mesh-control", "builder"];
  const catalogueOnMachine = o.catalogueOnMachine ?? "/opt/mesh-catalog";
  const log = o.log ?? (() => {});

  const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`];
  const stop = (step: string, why: string): GenesisResult => {
    report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`);
    return { ok: false, step, why, report };
  };

  const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => {
    const { stdout } = await exec(o.instanceId, node, [
      "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
    ], timeoutMs);
    const marker = stdout.lastIndexOf("__exit=");
    if (marker < 0) return { out: stdout, ok: false };
    return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
  };
  const must = async (command: string, timeoutMs?: number): Promise<string> => {
    const { out, ok } = await on(command, timeoutMs);
    if (!ok) throw new Error(`${node}: ${command}\n${out}`);
    return out;
  };

  // The installer, beside the host binary. Everything else was placed by `raise`; this one is
  // placed here because only the first machine is bootstrapped.
  const name = await instanceNameOf(o.instanceId, node);
  const version = await placeBootstrap(name, node, o.installer, (m) => log(`genesis:${m}`));
  report.push(`  installer         ${version} at ${BOOTSTRAP_PATH}`);
  report.push(`  builds from       ${o.source} at ${o.sourceRef.slice(0, 8)}`);

  // The catalogue. `mesh-bootstrap --catalog` reads manifests from a CHECKOUT on the machine,
  // because at this moment the mesh has no forge, no build machine and — until the registry step
  // finishes — no registry. A manifest is a file, and somebody has to have put it there.
  //
  // **The WHOLE checkout, not the three genesis names.** `--catalog` is documented as a checkout
  // of the catalogue repository, and phase two reads more from it than genesis does — postgres,
  // mesh-catalog, the packet filter, whatever extras. Stocking only the bootstrap three left
  // phase two unable to read a manifest that was never put there (novox/hq installer step 14). The
  // production equivalent is an operator with a full checkout, or the installer cloning the repo;
  // the lab stands in for that by copying the whole tree once.
  const bundleTar = join(tmpdir(), `mesh-lab-catalogue-${process.pid}-${node}.tar`);
  execFileSync("tar", ["-cf", bundleTar, "-C", o.catalogDir, "."]);
  await must(`mkdir -p ${catalogueOnMachine}/modules`);
  await push(o.instanceId, node, bundleTar, "/tmp/catalogue.tar");
  await must(`tar -xf /tmp/catalogue.tar -C ${catalogueOnMachine}/modules`);
  // The three genesis itself needs must be present, or the pivot cannot even begin — checked here
  // rather than discovered at step 8, where the message is about a missing file.
  for (const module of modules) {
    const at = `${catalogueOnMachine}/modules/${module}/module.json`;
    if (!(await on(`test -f ${at}`)).ok) {
      return stop("preparing the catalogue", `the catalogue has no ${module}/module.json`);
    }
  }
  report.push(`  catalogue         full checkout at ${catalogueOnMachine} (${modules.join(", ")} + phase two)`);

  const local = join(tmpdir(), `mesh-lab-substrate-${process.pid}-${node}.lock`);
  writeFileSync(local, o.bundleTemplate);
  await push(o.instanceId, node, local, "/tmp/substrate-template.lock");

  const command = [
    BOOTSTRAP_PATH,
    `--source ${o.source}`,
    `--source-ref ${o.sourceRef}`,
    `--bundle /tmp/substrate-template.lock`,
    `--catalog ${catalogueOnMachine}`,
    `--node ${node}`,
    `--registry ${registry}`,
    `--tools-source ${o.toolsSource}`,
    `--catalog-source ${o.catalogSource}`,
    `--catalog-ref ${o.catalogRef ?? "main"}`,
    // The choices, answered the unattended way. Both have one option today, so these are
    // redundant on purpose: the day a second filter exists, this bed keeps working instead of
    // refusing, and choosing becomes a thing it visibly does.
    `--private-network wireguard`,
    `--packet-filter nftables`,
    `--host ${HOST_PATH}`,
    // The lab has no unit to supervise the host with, and the installer refuses to invent one — a
    // unit file is a packaging decision. A host started this way does not survive a reboot.
    `--host-in-background`,
  ].join(" ");

  // Run up to three times. Not to paper over a failure — every attempt's failing step is printed —
  // but because the installer is idempotent by design and says so, and because the one thing that
  // fails for a reason which goes away by itself is a pull: the store, broker and registry come
  // from the internet, and a rate-limited anonymous pull is not this mesh's fault.
  let said = "";
  let step = "";
  for (let attempt = 1; attempt <= 3; attempt++) {
    const ran = await on(command, 2_400_000);
    said = ran.out;
    log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`);
    if (ran.ok) { step = ""; break; }
    step = stepIn(said);
    if (attempt < 3) {
      log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`);
      await new Promise((r) => setTimeout(r, 30_000));
    }
  }
  if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n"));

  // ------------------------------------------------------------------------------------------
  // Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting
  // zero (novox/hq ADR 0018).
  // ------------------------------------------------------------------------------------------

  // 1. The control plane answers, asked of the PERMANENT container by name.
  const answered = await on(`docker exec mesh-control /mesh-control status`, 60_000);
  report.push(`  control plane     ${answered.ok ? answered.out.split("\n")[0] : "NO ANSWER"}`);
  if (!answered.ok) return stop("after the last step", `mesh-control does not answer:\n${answered.out}`);

  // 2. The registry replies on /v2/. A container that is up is not a registry that serves.
  const v2 = await on(`curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://${registry}/v2/`);
  const v2Code = v2.out.trim();
  report.push(`  registry /v2/     ${v2Code || "no answer"}`);
  if (v2Code !== "200") return stop("after the last step", `the mesh's own registry answered ${v2Code || "nothing"}`);

  // 3. THE PIVOT COMPLETED — the running control plane is pinned by a digest THIS MESH'S REGISTRY
  //    assigned, not by an image id (ADR 0067 states this check in as many words).
  const pinnedTo = (await on(`docker inspect --format '{{.Config.Image}}' mesh-control`)).out.trim();
  report.push(`  pinned to         ${pinnedTo || "(nothing)"}`);
  if (/^sha256:[0-9a-f]{64}$/.test(pinnedTo)) {
    return stop("after the last step",
      `mesh-control is running from ${pinnedTo}, which is an IMAGE ID — the digest of the image's ` +
      `own configuration, which no registry ever served. The pivot did not happen, so this mesh ` +
      `cannot upgrade itself (novox/hq ADR 0067, "the pivot completed").`);
  }
  if (!new RegExp(`^${registry.replaceAll(".", "\\.")}/mesh-control@sha256:[0-9a-f]{64}$`).test(pinnedTo)) {
    return stop("after the last step",
      `mesh-control is running from ${pinnedTo || "nothing this bed could read"}, which is not a ` +
      `digest assigned by ${registry}.`);
  }

  // 3a. THE CONTROL PLANE WAS BUILT, not carried.
  //
  // **The distinction the installer now exists to make** (novox/hq ADR 0073). A mesh running an
  // image it was handed cannot rebuild the thing that runs it, and looks identical from the
  // outside to one that can — same container, same digest, same registry. The difference is
  // whether a build happened, and the only place that is visible is the installer saying so.
  //
  // Checked against the commit this bed asked for, not merely that some build occurred: an
  // installer that quietly built something else would satisfy a weaker check and raise a mesh
  // nobody asked for.
  const wanted = o.sourceRef.slice(0, 8);
  if (!new RegExp(`built mesh-control from ${wanted}`).test(said)) {
    return stop("after the last step",
      `the installer never said it built mesh-control from ${wanted}. What runs may have been ` +
      `carried rather than made here, which is a mesh that cannot rebuild its own control plane. ` +
      `The installer said:\n${said.split("\n").filter((l) => /built|build/.test(l)).join("\n") || "(nothing about building)"}`);
  }
  report.push(`  built here        mesh-control from ${wanted}, by the carried builder`);

  // 3b. And the registry really serves it. A reference is a claim; a tag list is the registry agreeing.
  const tags = await on(`curl -s --max-time 10 http://${registry}/v2/mesh-control/tags/list`);
  report.push(`  registry holds    ${tags.out.trim() || "nothing"}`);
  if (!tags.out.includes("genesis")) {
    return stop("after the last step",
      `${registry} does not serve mesh-control, so the digest the container is pinned to names an ` +
      `image nothing can pull: ${tags.out.trim()}`);
  }

  // 4. The temporary control plane is GONE. The name is the audit.
  const temp = await on(`docker inspect --format '{{.State.Status}}' temp-mesh-control`);
  report.push(`  temp-mesh-control ${temp.ok ? `STILL HERE (${temp.out.trim()})` : "gone"}`);
  if (temp.ok) {
    return stop("after the last step",
      `temp-mesh-control is still ${temp.out.trim()}. Two control planes are consuming this mesh's ` +
      `broker queues; neither is wrong and the pivot is not finished.`);
  }

  // 5. And the mesh has heard from its one node.
  const nodes = await on(`docker exec mesh-control /mesh-control node list`);
  report.push(`  node list         ${nodes.out.trim().split("\n").join(" | ")}`);
  const line = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${node} `));
  if (!line || !/^\S+\s+here\b/.test(line)) {
    return stop("after the last step", `the mesh has not heard from ${node}: ${line ?? "it has no record of it at all"}`);
  }

  report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`);
  return { ok: true, step: "", why: "", report };
}