319 lines
17 KiB
TypeScript
319 lines
17 KiB
TypeScript
/**
|
|
* GENESIS — a machine with no mesh becomes a mesh of one, by running the installer.
|
|
*
|
|
* This is the mesh's own install procedure, exercised the way an operator runs it: the same
|
|
* program a bare machine runs, given the same inputs, checked by asking the machine rather than
|
|
* by trusting the installer's exit code (novox/hq ADR 0018, ADR 0067).
|
|
*
|
|
* It lives here, shared, for one reason. An install procedure that exists only inside one bed is
|
|
* exercised by whoever writes that bed and never by whoever installs, and a second bed describing
|
|
* it differently is the arrangement that already failed — a fixture invented a registry that
|
|
* exists in no production and hid two separate faults for as long as it existed. There is one
|
|
* description of genesis, and both the single-node bed and the four-node bed call it.
|
|
*/
|
|
import { existsSync, writeFileSync } from "node:fs";
|
|
import { execFileSync } from "node:child_process";
|
|
import { tmpdir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
import { exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts";
|
|
import { placeBootstrap, installHostService, BOOTSTRAP_PATH, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
import { dirname } from "node:path";
|
|
|
|
/** What genesis did, or where it stopped. */
|
|
export interface GenesisResult {
|
|
ok: boolean;
|
|
/** `step 7 of 10, registry` — the installer's own words, so a bed reports the cause. */
|
|
step: string;
|
|
why: string;
|
|
report: string[];
|
|
/** Everything the installer printed on its last attempt — what a bed asserts a refusal names. */
|
|
said: string;
|
|
}
|
|
|
|
export interface GenesisOptions {
|
|
instanceId: string;
|
|
/** The machine being raised into a mesh of one. */
|
|
node: string;
|
|
/** Path to the built `mesh-bootstrap` binary on this workstation. */
|
|
installer: string;
|
|
/** A checkout of mesh-catalog's `modules/` on this workstation. */
|
|
catalogDir: string;
|
|
/**
|
|
* The foundation TEMPLATE's content — not a bundle. The installer produces the bundle from it,
|
|
* replacing the control plane's image with the id of the image it carries.
|
|
*/
|
|
bundleTemplate: string;
|
|
/** Manifests the installer reads. Only these two: it opens no others. */
|
|
catalogueModules?: string[];
|
|
catalogueOnMachine?: string;
|
|
/** Where this mesh's own registry will answer. */
|
|
registry?: string;
|
|
/**
|
|
* Where the control plane is built from, and the commit.
|
|
*
|
|
* The installer carries a builder rather than a finished control plane (novox/hq ADR 0073), so
|
|
* it has to be told what to make. A commit rather than a branch, because what genesis clones is
|
|
* the trust anchor for everything the mesh will ever run (ADR 0071).
|
|
*/
|
|
source: string;
|
|
sourceRef: string;
|
|
/**
|
|
* Phase two: where the shared base and the catalogue's modules are built from.
|
|
*
|
|
* The installer no longer stops at a mesh that runs — it builds the base, a store, the
|
|
* catalogue, chooses the network and the filter, and asks a human where one must choose. This
|
|
* bed has no human, so every choice arrives as a flag, and a required choice with no flag is
|
|
* the installer refusing — which is the behaviour, not a lab problem.
|
|
*/
|
|
toolsSource: string;
|
|
catalogSource: string;
|
|
/** Where the shared library is built from — published before the base resolves it (ADR 0076). */
|
|
sdkSource: string;
|
|
sdkRef?: string;
|
|
/** What of the base repo to build. Defaults to main; a feature branch under test overrides it. */
|
|
toolsRef?: string;
|
|
/** The site the anchor is placed at on the private network. Must match how the operator/test
|
|
* places it afterwards, or the first re-place changes the overlay and recreates the control plane. */
|
|
site?: string;
|
|
/** Supervise the host as a systemd service (the real install path) instead of --host-in-background,
|
|
* so it survives a reboot. Needs hostBinary to locate the packaging. */
|
|
hostService?: boolean;
|
|
/** The built mesh-host binary on this workstation; its repo's packaging/ dir supplies the unit. */
|
|
hostBinary?: string;
|
|
/** What of the catalogue to build. A branch under test is the usual reason this is not main. */
|
|
catalogRef?: string;
|
|
/**
|
|
* Raise the machine adopted (novox/hq ADR 0100): what it runs and its firewall are kept. Without
|
|
* it the installer raises a converged node, and refuses a machine in use.
|
|
*/
|
|
adopted?: boolean;
|
|
/** Further installer flags, as the operator would type them — `--registry-port 5100`, `--dry-run`. */
|
|
flags?: string[];
|
|
/**
|
|
* How many times to run the installer. Three by default, for a pull the internet rate-limited; a
|
|
* bed that expects a REFUSAL runs it once, because a refusal is the answer, not a flake.
|
|
*/
|
|
attempts?: number;
|
|
/**
|
|
* Whether to ask the machine if it became a working mesh of one afterwards. Off for a run that is
|
|
* not meant to raise one — a dry run, or a refusal the bed expects.
|
|
*/
|
|
verify?: boolean;
|
|
log?: (m: string) => void;
|
|
}
|
|
|
|
/** The step a failed `mesh-bootstrap` names, as it prints it, or "" if it named none. */
|
|
export function stepIn(said: string): string {
|
|
return said.match(/^mesh-bootstrap: (step \d+ of \d+, [a-z-]+):/m)?.[1] ?? "";
|
|
}
|
|
|
|
export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
|
const node = o.node;
|
|
const registry = o.registry ?? "127.0.0.1:5000";
|
|
// The registry's and the builder's. Not the control plane's: its manifest lives in its own
|
|
// repository and genesis takes it from the build it runs (novox/hq ADR 0069, 04-ISSUES/072).
|
|
const modules = o.catalogueModules ?? ["distribution", "builder"];
|
|
const catalogueOnMachine = o.catalogueOnMachine ?? "/opt/mesh-catalog";
|
|
const log = o.log ?? (() => {});
|
|
|
|
const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`];
|
|
let said = "";
|
|
const stop = (step: string, why: string): GenesisResult => {
|
|
report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`);
|
|
return { ok: false, step, why, report, said };
|
|
};
|
|
|
|
const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => {
|
|
const { stdout } = await exec(o.instanceId, node, [
|
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
], timeoutMs);
|
|
const marker = stdout.lastIndexOf("__exit=");
|
|
if (marker < 0) return { out: stdout, ok: false };
|
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
};
|
|
const must = async (command: string, timeoutMs?: number): Promise<string> => {
|
|
const { out, ok } = await on(command, timeoutMs);
|
|
if (!ok) throw new Error(`${node}: ${command}\n${out}`);
|
|
return out;
|
|
};
|
|
|
|
// The installer, beside the host binary. Everything else was placed by `raise`; this one is
|
|
// placed here because only the first machine is bootstrapped.
|
|
const name = await instanceNameOf(o.instanceId, node);
|
|
const version = await placeBootstrap(name, node, o.installer, (m) => log(`genesis:${m}`));
|
|
report.push(` installer ${version} at ${BOOTSTRAP_PATH}`);
|
|
report.push(` builds from ${o.source} at ${o.sourceRef.slice(0, 8)}`);
|
|
|
|
// The catalogue. `mesh-bootstrap --catalog` reads manifests from a CHECKOUT on the machine,
|
|
// because at this moment the mesh has no forge, no build machine and — until the registry step
|
|
// finishes — no registry. A manifest is a file, and somebody has to have put it there.
|
|
//
|
|
// **The WHOLE checkout, not the three genesis names.** `--catalog` is documented as a checkout
|
|
// of the catalogue repository, and phase two reads more from it than genesis does — postgres,
|
|
// mesh-catalog, the packet filter, whatever extras. Stocking only the bootstrap three left
|
|
// phase two unable to read a manifest that was never put there (novox/hq installer step 14). The
|
|
// production equivalent is an operator with a full checkout, or the installer cloning the repo;
|
|
// the lab stands in for that by copying the whole tree once.
|
|
const bundleTar = join(tmpdir(), `mesh-lab-catalogue-${process.pid}-${node}.tar`);
|
|
execFileSync("tar", ["-cf", bundleTar, "-C", o.catalogDir, "."]);
|
|
// Cleared first: a bed that runs genesis more than once (a refusal, then the raise) finds the last
|
|
// run's staging files, and the machine refuses to open them for the push.
|
|
await must(`rm -f /tmp/catalogue.tar /tmp/foundation-template.lock && mkdir -p ${catalogueOnMachine}/modules`);
|
|
await push(o.instanceId, node, bundleTar, "/tmp/catalogue.tar");
|
|
await must(`tar -xf /tmp/catalogue.tar -C ${catalogueOnMachine}/modules`);
|
|
// The three genesis itself needs must be present, or the pivot cannot even begin — checked here
|
|
// rather than discovered at step 8, where the message is about a missing file.
|
|
for (const module of modules) {
|
|
const at = `${catalogueOnMachine}/modules/${module}/module.json`;
|
|
if (!(await on(`test -f ${at}`)).ok) {
|
|
return stop("preparing the catalogue", `the catalogue has no ${module}/module.json`);
|
|
}
|
|
}
|
|
report.push(` catalogue full checkout at ${catalogueOnMachine} (${modules.join(", ")} + phase two)`);
|
|
|
|
const local = join(tmpdir(), `mesh-lab-foundation-${process.pid}-${node}.lock`);
|
|
writeFileSync(local, o.bundleTemplate);
|
|
await push(o.instanceId, node, local, "/tmp/foundation-template.lock");
|
|
|
|
// Supervise the host as a service (the real install path) when asked — the only way it survives a
|
|
// reboot. Installs the shipped packaging in the machine, then lets --host-service start+enable it.
|
|
if (o.hostService) {
|
|
if (!o.hostBinary) {
|
|
return stop("preparing the host service", "hostService needs hostBinary to find the packaging");
|
|
}
|
|
await installHostService(name, node, join(dirname(o.hostBinary), "packaging"), (m) => log(`genesis:${m}`));
|
|
report.push(` host supervised by nox-mesh-host.service`);
|
|
}
|
|
|
|
const command = [
|
|
BOOTSTRAP_PATH,
|
|
`--source ${o.source}`,
|
|
`--source-ref ${o.sourceRef}`,
|
|
`--bundle /tmp/foundation-template.lock`,
|
|
`--catalog ${catalogueOnMachine}`,
|
|
`--node ${node}`,
|
|
`--registry ${registry}`,
|
|
`--tools-source ${o.toolsSource}`,
|
|
`--tools-ref ${o.toolsRef ?? "main"}`,
|
|
`--catalog-source ${o.catalogSource}`,
|
|
`--catalog-ref ${o.catalogRef ?? "main"}`,
|
|
`--sdk-source ${o.sdkSource}`,
|
|
`--sdk-ref ${o.sdkRef ?? "main"}`,
|
|
// The choices, answered the unattended way. Both have one option today, so these are
|
|
// redundant on purpose: the day a second filter exists, this bed keeps working instead of
|
|
// refusing, and choosing becomes a thing it visibly does.
|
|
`--site ${o.site ?? "main"}`,
|
|
`--private-network wireguard`,
|
|
`--packet-filter nftables`,
|
|
`--host ${HOST_PATH}`,
|
|
...(o.adopted ? [`--adopted`] : []),
|
|
...(o.flags ?? []),
|
|
// A service when the packaging was installed above (survives a reboot); otherwise the
|
|
// background process, which does not — the installer refuses to invent a unit either way.
|
|
...(o.hostService ? [] as string[] : [`--host-in-background`]),
|
|
].join(" ");
|
|
|
|
// Run up to three times. Not to paper over a failure — every attempt's failing step is printed —
|
|
// but because the installer is idempotent by design and says so, and because the one thing that
|
|
// fails for a reason which goes away by itself is a pull: the store, broker and registry come
|
|
// from the internet, and a rate-limited anonymous pull is not this mesh's fault.
|
|
let step = "";
|
|
const attempts = o.attempts ?? 3;
|
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
const ran = await on(command, 2_400_000);
|
|
said = ran.out;
|
|
log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`);
|
|
if (ran.ok) { step = ""; break; }
|
|
step = stepIn(said) || "an unnamed step";
|
|
if (attempt < attempts) {
|
|
log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`);
|
|
await new Promise((r) => setTimeout(r, 30_000));
|
|
}
|
|
}
|
|
if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n"));
|
|
if (o.verify === false) {
|
|
report.push(`\nThe installer finished; not asked whether it raised a mesh (verify: false).`);
|
|
return { ok: true, step: "", why: "", report, said };
|
|
}
|
|
|
|
// ------------------------------------------------------------------------------------------
|
|
// Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting
|
|
// zero (novox/hq ADR 0018).
|
|
// ------------------------------------------------------------------------------------------
|
|
|
|
// 1. The control plane answers, asked of the PERMANENT container by name.
|
|
const answered = await on(`docker exec mesh-controller /mesh-controller status`, 60_000);
|
|
report.push(` control plane ${answered.ok ? answered.out.split("\n")[0] : "NO ANSWER"}`);
|
|
if (!answered.ok) return stop("after the last step", `mesh-controller does not answer:\n${answered.out}`);
|
|
|
|
// 2. The registry replies on /v2/. A container that is up is not a registry that serves.
|
|
const v2 = await on(`curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://${registry}/v2/`);
|
|
const v2Code = v2.out.trim();
|
|
report.push(` registry /v2/ ${v2Code || "no answer"}`);
|
|
if (v2Code !== "200") return stop("after the last step", `the mesh's own registry answered ${v2Code || "nothing"}`);
|
|
|
|
// 3. THE PIVOT COMPLETED — the running control plane is pinned by a digest THIS MESH'S REGISTRY
|
|
// assigned, not by an image id (ADR 0067 states this check in as many words).
|
|
const pinnedTo = (await on(`docker inspect --format '{{.Config.Image}}' mesh-controller`)).out.trim();
|
|
report.push(` pinned to ${pinnedTo || "(nothing)"}`);
|
|
if (/^sha256:[0-9a-f]{64}$/.test(pinnedTo)) {
|
|
return stop("after the last step",
|
|
`mesh-controller is running from ${pinnedTo}, which is an IMAGE ID — the digest of the image's ` +
|
|
`own configuration, which no registry ever served. The pivot did not happen, so this mesh ` +
|
|
`cannot upgrade itself (novox/hq ADR 0067, "the pivot completed").`);
|
|
}
|
|
if (!new RegExp(`^${registry.replaceAll(".", "\\.")}/mesh-controller@sha256:[0-9a-f]{64}$`).test(pinnedTo)) {
|
|
return stop("after the last step",
|
|
`mesh-controller is running from ${pinnedTo || "nothing this bed could read"}, which is not a ` +
|
|
`digest assigned by ${registry}.`);
|
|
}
|
|
|
|
// 3a. THE CONTROL PLANE WAS BUILT, not carried.
|
|
//
|
|
// **The distinction the installer now exists to make** (novox/hq ADR 0073). A mesh running an
|
|
// image it was handed cannot rebuild the thing that runs it, and looks identical from the
|
|
// outside to one that can — same container, same digest, same registry. The difference is
|
|
// whether a build happened, and the only place that is visible is the installer saying so.
|
|
//
|
|
// Checked against the commit this bed asked for, not merely that some build occurred: an
|
|
// installer that quietly built something else would satisfy a weaker check and raise a mesh
|
|
// nobody asked for.
|
|
const wanted = o.sourceRef.slice(0, 8);
|
|
if (!new RegExp(`built mesh-controller from ${wanted}`).test(said)) {
|
|
return stop("after the last step",
|
|
`the installer never said it built mesh-controller from ${wanted}. What runs may have been ` +
|
|
`carried rather than made here, which is a mesh that cannot rebuild its own control plane. ` +
|
|
`The installer said:\n${said.split("\n").filter((l) => /built|build/.test(l)).join("\n") || "(nothing about building)"}`);
|
|
}
|
|
report.push(` built here mesh-controller from ${wanted}, by the carried builder`);
|
|
|
|
// 3b. And the registry really serves it. A reference is a claim; a tag list is the registry agreeing.
|
|
const tags = await on(`curl -s --max-time 10 http://${registry}/v2/mesh-controller/tags/list`);
|
|
report.push(` registry holds ${tags.out.trim() || "nothing"}`);
|
|
if (!tags.out.includes("genesis")) {
|
|
return stop("after the last step",
|
|
`${registry} does not serve mesh-controller, so the digest the container is pinned to names an ` +
|
|
`image nothing can pull: ${tags.out.trim()}`);
|
|
}
|
|
|
|
// 4. The temporary control plane is GONE. The name is the audit.
|
|
const temp = await on(`docker inspect --format '{{.State.Status}}' temp-mesh-controller`);
|
|
report.push(` temp-mesh-controller ${temp.ok ? `STILL HERE (${temp.out.trim()})` : "gone"}`);
|
|
if (temp.ok) {
|
|
return stop("after the last step",
|
|
`temp-mesh-controller is still ${temp.out.trim()}. Two control planes are consuming this mesh's ` +
|
|
`broker queues; neither is wrong and the pivot is not finished.`);
|
|
}
|
|
|
|
// 5. And the mesh has heard from its one node.
|
|
const nodes = await on(`docker exec mesh-controller /mesh-controller node list`);
|
|
report.push(` node list ${nodes.out.trim().split("\n").join(" | ")}`);
|
|
const line = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${node} `));
|
|
if (!line || !/^\S+\s+here\b/.test(line)) {
|
|
return stop("after the last step", `the mesh has not heard from ${node}: ${line ?? "it has no record of it at all"}`);
|
|
}
|
|
|
|
report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`);
|
|
return { ok: true, step: "", why: "", report, said };
|
|
}
|