/** * GENESIS — a machine with no mesh becomes a mesh of one, by running the installer. * * This is the mesh's own install procedure, exercised the way an operator runs it: the same * program a bare machine runs, given the same inputs, checked by asking the machine rather than * by trusting the installer's exit code (novox/hq ADR 0018, ADR 0067). * * It lives here, shared, for one reason. An install procedure that exists only inside one bed is * exercised by whoever writes that bed and never by whoever installs, and a second bed describing * it differently is the arrangement that already failed — a fixture invented a registry that * exists in no production and hid two separate faults for as long as it existed. There is one * description of genesis, and both the single-node bed and the four-node bed call it. */ import { existsSync, writeFileSync } from "node:fs"; import { execFileSync } from "node:child_process"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts"; import { placeBootstrap, installHostService, BOOTSTRAP_PATH, HOST_PATH } from "../../src/lifecycle/place.ts"; import { dirname } from "node:path"; /** What genesis did, or where it stopped. */ export interface GenesisResult { ok: boolean; /** `step 7 of 10, registry` — the installer's own words, so a bed reports the cause. */ step: string; why: string; report: string[]; } export interface GenesisOptions { instanceId: string; /** The machine being raised into a mesh of one. */ node: string; /** Path to the built `mesh-bootstrap` binary on this workstation. */ installer: string; /** A checkout of mesh-catalog's `modules/` on this workstation. */ catalogDir: string; /** * The foundation TEMPLATE's content — not a bundle. The installer produces the bundle from it, * replacing the control plane's image with the id of the image it carries. */ bundleTemplate: string; /** Manifests the installer reads. Only these two: it opens no others. */ catalogueModules?: string[]; catalogueOnMachine?: string; /** Where this mesh's own registry will answer. */ registry?: string; /** * Where the control plane is built from, and the commit. * * The installer carries a builder rather than a finished control plane (novox/hq ADR 0073), so * it has to be told what to make. A commit rather than a branch, because what genesis clones is * the trust anchor for everything the mesh will ever run (ADR 0071). */ source: string; sourceRef: string; /** * Phase two: where the shared base and the catalogue's modules are built from. * * The installer no longer stops at a mesh that runs — it builds the base, a store, the * catalogue, chooses the network and the filter, and asks a human where one must choose. This * bed has no human, so every choice arrives as a flag, and a required choice with no flag is * the installer refusing — which is the behaviour, not a lab problem. */ toolsSource: string; catalogSource: string; /** Where the shared library is built from — published before the base resolves it (ADR 0076). */ sdkSource: string; sdkRef?: string; /** What of the base repo to build. Defaults to main; a feature branch under test overrides it. */ toolsRef?: string; /** The site the anchor is placed at on the private network. Must match how the operator/test * places it afterwards, or the first re-place changes the overlay and recreates the control plane. */ site?: string; /** Supervise the host as a systemd service (the real install path) instead of --host-in-background, * so it survives a reboot. Needs hostBinary to locate the packaging. */ hostService?: boolean; /** The built mesh-host binary on this workstation; its repo's packaging/ dir supplies the unit. */ hostBinary?: string; /** What of the catalogue to build. A branch under test is the usual reason this is not main. */ catalogRef?: string; log?: (m: string) => void; } /** The step a failed `mesh-bootstrap` names, as it prints it, or "" if it named none. */ export function stepIn(said: string): string { return said.match(/^mesh-bootstrap: (step \d+ of \d+, [a-z-]+):/m)?.[1] ?? ""; } export async function genesis(o: GenesisOptions): Promise { const node = o.node; const registry = o.registry ?? "127.0.0.1:5000"; const modules = o.catalogueModules ?? ["distribution", "mesh-controller", "builder"]; const catalogueOnMachine = o.catalogueOnMachine ?? "/opt/mesh-catalog"; const log = o.log ?? (() => {}); const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`]; const stop = (step: string, why: string): GenesisResult => { report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`); return { ok: false, step, why, report }; }; const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => { const { stdout } = await exec(o.instanceId, node, [ "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, ], timeoutMs); const marker = stdout.lastIndexOf("__exit="); if (marker < 0) return { out: stdout, ok: false }; return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; }; const must = async (command: string, timeoutMs?: number): Promise => { const { out, ok } = await on(command, timeoutMs); if (!ok) throw new Error(`${node}: ${command}\n${out}`); return out; }; // The installer, beside the host binary. Everything else was placed by `raise`; this one is // placed here because only the first machine is bootstrapped. const name = await instanceNameOf(o.instanceId, node); const version = await placeBootstrap(name, node, o.installer, (m) => log(`genesis:${m}`)); report.push(` installer ${version} at ${BOOTSTRAP_PATH}`); report.push(` builds from ${o.source} at ${o.sourceRef.slice(0, 8)}`); // The catalogue. `mesh-bootstrap --catalog` reads manifests from a CHECKOUT on the machine, // because at this moment the mesh has no forge, no build machine and — until the registry step // finishes — no registry. A manifest is a file, and somebody has to have put it there. // // **The WHOLE checkout, not the three genesis names.** `--catalog` is documented as a checkout // of the catalogue repository, and phase two reads more from it than genesis does — postgres, // mesh-catalog, the packet filter, whatever extras. Stocking only the bootstrap three left // phase two unable to read a manifest that was never put there (novox/hq installer step 14). The // production equivalent is an operator with a full checkout, or the installer cloning the repo; // the lab stands in for that by copying the whole tree once. const bundleTar = join(tmpdir(), `mesh-lab-catalogue-${process.pid}-${node}.tar`); execFileSync("tar", ["-cf", bundleTar, "-C", o.catalogDir, "."]); await must(`mkdir -p ${catalogueOnMachine}/modules`); await push(o.instanceId, node, bundleTar, "/tmp/catalogue.tar"); await must(`tar -xf /tmp/catalogue.tar -C ${catalogueOnMachine}/modules`); // The three genesis itself needs must be present, or the pivot cannot even begin — checked here // rather than discovered at step 8, where the message is about a missing file. for (const module of modules) { const at = `${catalogueOnMachine}/modules/${module}/module.json`; if (!(await on(`test -f ${at}`)).ok) { return stop("preparing the catalogue", `the catalogue has no ${module}/module.json`); } } report.push(` catalogue full checkout at ${catalogueOnMachine} (${modules.join(", ")} + phase two)`); const local = join(tmpdir(), `mesh-lab-foundation-${process.pid}-${node}.lock`); writeFileSync(local, o.bundleTemplate); await push(o.instanceId, node, local, "/tmp/foundation-template.lock"); // Supervise the host as a service (the real install path) when asked — the only way it survives a // reboot. Installs the shipped packaging in the machine, then lets --host-service start+enable it. if (o.hostService) { if (!o.hostBinary) { return stop("preparing the host service", "hostService needs hostBinary to find the packaging"); } await installHostService(name, node, join(dirname(o.hostBinary), "packaging"), (m) => log(`genesis:${m}`)); report.push(` host supervised by nox-mesh-host.service`); } const command = [ BOOTSTRAP_PATH, `--source ${o.source}`, `--source-ref ${o.sourceRef}`, `--bundle /tmp/foundation-template.lock`, `--catalog ${catalogueOnMachine}`, `--node ${node}`, `--registry ${registry}`, `--tools-source ${o.toolsSource}`, `--tools-ref ${o.toolsRef ?? "main"}`, `--catalog-source ${o.catalogSource}`, `--catalog-ref ${o.catalogRef ?? "main"}`, `--sdk-source ${o.sdkSource}`, `--sdk-ref ${o.sdkRef ?? "main"}`, // The choices, answered the unattended way. Both have one option today, so these are // redundant on purpose: the day a second filter exists, this bed keeps working instead of // refusing, and choosing becomes a thing it visibly does. `--site ${o.site ?? "main"}`, `--private-network wireguard`, `--packet-filter nftables`, `--host ${HOST_PATH}`, // A service when the packaging was installed above (survives a reboot); otherwise the // background process, which does not — the installer refuses to invent a unit either way. ...(o.hostService ? [] as string[] : [`--host-in-background`]), ].join(" "); // Run up to three times. Not to paper over a failure — every attempt's failing step is printed — // but because the installer is idempotent by design and says so, and because the one thing that // fails for a reason which goes away by itself is a pull: the store, broker and registry come // from the internet, and a rate-limited anonymous pull is not this mesh's fault. let said = ""; let step = ""; for (let attempt = 1; attempt <= 3; attempt++) { const ran = await on(command, 2_400_000); said = ran.out; log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`); if (ran.ok) { step = ""; break; } step = stepIn(said); if (attempt < 3) { log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`); await new Promise((r) => setTimeout(r, 30_000)); } } if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n")); // ------------------------------------------------------------------------------------------ // Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting // zero (novox/hq ADR 0018). // ------------------------------------------------------------------------------------------ // 1. The control plane answers, asked of the PERMANENT container by name. const answered = await on(`docker exec mesh-controller /mesh-controller status`, 60_000); report.push(` control plane ${answered.ok ? answered.out.split("\n")[0] : "NO ANSWER"}`); if (!answered.ok) return stop("after the last step", `mesh-controller does not answer:\n${answered.out}`); // 2. The registry replies on /v2/. A container that is up is not a registry that serves. const v2 = await on(`curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://${registry}/v2/`); const v2Code = v2.out.trim(); report.push(` registry /v2/ ${v2Code || "no answer"}`); if (v2Code !== "200") return stop("after the last step", `the mesh's own registry answered ${v2Code || "nothing"}`); // 3. THE PIVOT COMPLETED — the running control plane is pinned by a digest THIS MESH'S REGISTRY // assigned, not by an image id (ADR 0067 states this check in as many words). const pinnedTo = (await on(`docker inspect --format '{{.Config.Image}}' mesh-controller`)).out.trim(); report.push(` pinned to ${pinnedTo || "(nothing)"}`); if (/^sha256:[0-9a-f]{64}$/.test(pinnedTo)) { return stop("after the last step", `mesh-controller is running from ${pinnedTo}, which is an IMAGE ID — the digest of the image's ` + `own configuration, which no registry ever served. The pivot did not happen, so this mesh ` + `cannot upgrade itself (novox/hq ADR 0067, "the pivot completed").`); } if (!new RegExp(`^${registry.replaceAll(".", "\\.")}/mesh-controller@sha256:[0-9a-f]{64}$`).test(pinnedTo)) { return stop("after the last step", `mesh-controller is running from ${pinnedTo || "nothing this bed could read"}, which is not a ` + `digest assigned by ${registry}.`); } // 3a. THE CONTROL PLANE WAS BUILT, not carried. // // **The distinction the installer now exists to make** (novox/hq ADR 0073). A mesh running an // image it was handed cannot rebuild the thing that runs it, and looks identical from the // outside to one that can — same container, same digest, same registry. The difference is // whether a build happened, and the only place that is visible is the installer saying so. // // Checked against the commit this bed asked for, not merely that some build occurred: an // installer that quietly built something else would satisfy a weaker check and raise a mesh // nobody asked for. const wanted = o.sourceRef.slice(0, 8); if (!new RegExp(`built mesh-controller from ${wanted}`).test(said)) { return stop("after the last step", `the installer never said it built mesh-controller from ${wanted}. What runs may have been ` + `carried rather than made here, which is a mesh that cannot rebuild its own control plane. ` + `The installer said:\n${said.split("\n").filter((l) => /built|build/.test(l)).join("\n") || "(nothing about building)"}`); } report.push(` built here mesh-controller from ${wanted}, by the carried builder`); // 3b. And the registry really serves it. A reference is a claim; a tag list is the registry agreeing. const tags = await on(`curl -s --max-time 10 http://${registry}/v2/mesh-controller/tags/list`); report.push(` registry holds ${tags.out.trim() || "nothing"}`); if (!tags.out.includes("genesis")) { return stop("after the last step", `${registry} does not serve mesh-controller, so the digest the container is pinned to names an ` + `image nothing can pull: ${tags.out.trim()}`); } // 4. The temporary control plane is GONE. The name is the audit. const temp = await on(`docker inspect --format '{{.State.Status}}' temp-mesh-controller`); report.push(` temp-mesh-controller ${temp.ok ? `STILL HERE (${temp.out.trim()})` : "gone"}`); if (temp.ok) { return stop("after the last step", `temp-mesh-controller is still ${temp.out.trim()}. Two control planes are consuming this mesh's ` + `broker queues; neither is wrong and the pivot is not finished.`); } // 5. And the mesh has heard from its one node. const nodes = await on(`docker exec mesh-controller /mesh-controller node list`); report.push(` node list ${nodes.out.trim().split("\n").join(" | ")}`); const line = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${node} `)); if (!line || !/^\S+\s+here\b/.test(line)) { return stop("after the last step", `the mesh has not heard from ${node}: ${line ?? "it has no record of it at all"}`); } report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`); return { ok: true, step: "", why: "", report }; }