From e427e41389fe66e8af6ed0f5ae916de1ad926c42 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 12 Sep 2026 16:46:05 +0200 Subject: [PATCH] Raise a mesh of one by the installer, in a bed of its own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four-node bed proves genesis entangled with three machines joining across a gateway, so the cheapest check of the install path costs a four-machine raise. This is genesis alone: one machine, the installer, and the question asked of the machine rather than inferred from an exit code. Genesis moves into a shared routine both beds call, rather than being described a second time here — a second description kept in step with the first is what put the whole procedure inside a fixture to begin with. The scenario needs two things the first draft missed, and both cost a full raise to discover: a way out to the internet, because the installer's first act is to pull the substrate; and a container runtime, because the installer's first refusal is a machine that has none. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- scenarios/genesis-single.yml | 40 +++++ test/integration/genesis-single.test.ts | 103 +++++++++++++ test/integration/genesis.ts | 194 ++++++++++++++++++++++++ 3 files changed, 337 insertions(+) create mode 100644 scenarios/genesis-single.yml create mode 100644 test/integration/genesis-single.test.ts create mode 100644 test/integration/genesis.ts diff --git a/scenarios/genesis-single.yml b/scenarios/genesis-single.yml new file mode 100644 index 0000000..da12b61 --- /dev/null +++ b/scenarios/genesis-single.yml @@ -0,0 +1,40 @@ +# GENESIS, on its own: one machine, no mesh, and the installer. +# +# The smallest thing that proves a mesh can be raised. One machine on a public segment with a way +# out to the internet, the host placed, and nothing else — the installer carries the control +# plane's image and the substrate's own images are pulled over the uplink, exactly as they are on +# a bare machine. +# +# The address matters: the substrate template names the broker at 192.0.2.10, and a token carries +# that address verbatim as the endpoint an enrolling node dials. With one machine, that machine +# must BE it, or the mesh would hand out an endpoint nothing answers on. +scenario: genesis-single + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24, "2001:db8:a::/48"] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10, "2001:db8:a::10"] } + # The way out. Without it the machine is sealed in, and the installer stops at its first pull: + # the store, the broker and the registry all come from the internet, exactly as they do on a + # bare machine. A scenario that needs no images can omit this; genesis cannot. + egress: true + inbound: allow + # Enough for the substrate (store, broker), the registry, and two control planes during the + # pivot. Smaller than the four-node bed's anchor, which also carries a whole service set. + memory: 8GiB + cpus: 4 + disk: 40GiB + +# The host, and a container runtime for it to drive. +# +# The runtime is not a mesh tier — it is a prerequisite of the machine, and the installer's first +# step refuses to go on without one. Placing it here is the lab preparing a machine, not the lab +# describing an installation. Everything above tier 0 — the substrate, the registry, the control +# plane — is the installer's, and the lab places none of it. That is the whole point of this bed: +# if the lab placed the substrate, it would be describing installing all over again. +place: + all: [host, runtime] diff --git a/test/integration/genesis-single.test.ts b/test/integration/genesis-single.test.ts new file mode 100644 index 0000000..1221bb1 --- /dev/null +++ b/test/integration/genesis-single.test.ts @@ -0,0 +1,103 @@ +/** + * A MESH OF ONE, raised by the installer and nothing else. + * + * The four-node bed proves genesis too, but it proves it entangled with three machines joining + * afterwards across a gateway. This bed is genesis alone: one machine, the installer, and the + * question "is this a working mesh?" asked of the machine. + * + * It shares its genesis with the four-node bed (`./genesis.ts`) rather than describing installing + * a second time — a second description kept in step with the first is the arrangement that put + * the whole procedure inside a fixture in the first place (novox/hq ADR 0067). + * + * MESH_LAB_INCUS='sudo -n incus' + * MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host + * MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap + * MESH_LAB_BUNDLE=.../mesh-host/examples/substrate-first-node.lock + * MESH_LAB_CATALOG=.../mesh-catalog/modules + * MESH_LAB_KEEP=1 to leave it standing afterwards + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy } from "../../src/lifecycle/operate.ts"; +import { bootstrapBinaryPath, hostBinaryPath } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts"; +import { genesis, type GenesisResult } from "./genesis.ts"; + +const SCENARIO = "genesis-single"; +const NODE = "anchor"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; +const KEEP = !!process.env["MESH_LAB_KEEP"]; +const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "genesis-single-live" : undefined); + +const skip = + !capability.usable ? capability.why : + !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : + !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap (mesh-host `make " + + "bootstrap IMAGE=mesh-control:development`)" : + !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate template" : + !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" : + false; + +let instanceId = ""; +let result: GenesisResult; + +before(async () => { + if (skip) return; + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), + }); + instanceId = raised.instanceId; + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); + + // Caught rather than allowed to propagate, so a failure BEFORE the installer ran is reported in + // the same shape as one the installer itself named, instead of a bare stack trace. + try { + result = await genesis({ + instanceId, + node: NODE, + installer: installer as string, + catalogDir, + // The template, not a bundle. The store and broker become the references mesh-catalog pins, + // and the machine pulls them over its uplink like any first node. mesh-control is left + // naming a registry that does not exist — the installer overwrites it, and leaving it proves + // that it does. + bundleTemplate: substrateBundle(bundle, []), + log: (m) => console.log(m), + }); + } catch (err) { + result = { + ok: false, + step: "getting the machine ready to be bootstrapped — the installer never ran", + why: (err as Error).message, + report: [], + }; + } + console.log(result.report.join("\n")); +}, { timeout: 3_600_000 }); + +after(async () => { + if (KEEP) { + console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`); + return; + } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("one machine becomes a working mesh of one, raised by the installer", { + skip, timeout: 3_600_000, +}, () => { + assert.ok(result.ok, + `GENESIS FAILED — ${result.step || "no step named"}.\n\n${result.why}\n\n` + + result.report.join("\n")); +}); diff --git a/test/integration/genesis.ts b/test/integration/genesis.ts new file mode 100644 index 0000000..3cafb53 --- /dev/null +++ b/test/integration/genesis.ts @@ -0,0 +1,194 @@ +/** + * GENESIS — a machine with no mesh becomes a mesh of one, by running the installer. + * + * This is the mesh's own install procedure, exercised the way an operator runs it: the same + * program a bare machine runs, given the same inputs, checked by asking the machine rather than + * by trusting the installer's exit code (novox/hq ADR 0018, ADR 0067). + * + * It lives here, shared, for one reason. An install procedure that exists only inside one bed is + * exercised by whoever writes that bed and never by whoever installs, and a second bed describing + * it differently is the arrangement that already failed — a fixture invented a registry that + * exists in no production and hid two separate faults for as long as it existed. There is one + * description of genesis, and both the single-node bed and the four-node bed call it. + */ +import { existsSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts"; +import { placeBootstrap, BOOTSTRAP_PATH, HOST_PATH } from "../../src/lifecycle/place.ts"; + +/** What genesis did, or where it stopped. */ +export interface GenesisResult { + ok: boolean; + /** `step 7 of 10, registry` — the installer's own words, so a bed reports the cause. */ + step: string; + why: string; + report: string[]; +} + +export interface GenesisOptions { + instanceId: string; + /** The machine being raised into a mesh of one. */ + node: string; + /** Path to the built `mesh-bootstrap` binary on this workstation. */ + installer: string; + /** A checkout of mesh-catalog's `modules/` on this workstation. */ + catalogDir: string; + /** + * The substrate TEMPLATE's content — not a bundle. The installer produces the bundle from it, + * replacing the control plane's image with the id of the image it carries. + */ + bundleTemplate: string; + /** Manifests the installer reads. Only these two: it opens no others. */ + catalogueModules?: string[]; + catalogueOnMachine?: string; + /** Where this mesh's own registry will answer. */ + registry?: string; + log?: (m: string) => void; +} + +/** The step a failed `mesh-bootstrap` names, as it prints it, or "" if it named none. */ +export function stepIn(said: string): string { + return said.match(/^mesh-bootstrap: (step \d+ of \d+, [a-z-]+):/m)?.[1] ?? ""; +} + +export async function genesis(o: GenesisOptions): Promise { + const node = o.node; + const registry = o.registry ?? "127.0.0.1:5000"; + const modules = o.catalogueModules ?? ["registry", "mesh-control"]; + const catalogueOnMachine = o.catalogueOnMachine ?? "/opt/mesh-catalog"; + const log = o.log ?? (() => {}); + + const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`]; + const stop = (step: string, why: string): GenesisResult => { + report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`); + return { ok: false, step, why, report }; + }; + + const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => { + const { stdout } = await exec(o.instanceId, node, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; + }; + const must = async (command: string, timeoutMs?: number): Promise => { + const { out, ok } = await on(command, timeoutMs); + if (!ok) throw new Error(`${node}: ${command}\n${out}`); + return out; + }; + + // The installer, beside the host binary. Everything else was placed by `raise`; this one is + // placed here because only the first machine is bootstrapped. + const name = await instanceNameOf(o.instanceId, node); + const version = await placeBootstrap(name, node, o.installer, (m) => log(`genesis:${m}`)); + report.push(` installer ${version} at ${BOOTSTRAP_PATH}`); + + // The catalogue. `mesh-bootstrap --catalog` reads manifests from a CHECKOUT on the machine, + // because at this moment the mesh has no forge, no build machine and — until the registry step + // finishes — no registry. A manifest is a file, and somebody has to have put it there. + await must(`mkdir -p ${modules.map((m) => `${catalogueOnMachine}/modules/${m}`).join(" ")}`); + for (const module of modules) { + const from = resolve(o.catalogDir, module, "module.json"); + if (!existsSync(from)) return stop("preparing the catalogue", `the catalogue has no ${module}/module.json at ${from}`); + await push(o.instanceId, node, from, `${catalogueOnMachine}/modules/${module}/module.json`); + } + report.push(` catalogue ${modules.join(", ")} at ${catalogueOnMachine}`); + + const local = join(tmpdir(), `mesh-lab-substrate-${process.pid}-${node}.lock`); + writeFileSync(local, o.bundleTemplate); + await push(o.instanceId, node, local, "/tmp/substrate-template.lock"); + + const command = [ + BOOTSTRAP_PATH, + `--bundle /tmp/substrate-template.lock`, + `--catalog ${catalogueOnMachine}`, + `--node ${node}`, + `--registry ${registry}`, + `--host ${HOST_PATH}`, + // The lab has no unit to supervise the host with, and the installer refuses to invent one — a + // unit file is a packaging decision. A host started this way does not survive a reboot. + `--host-in-background`, + ].join(" "); + + // Run up to three times. Not to paper over a failure — every attempt's failing step is printed — + // but because the installer is idempotent by design and says so, and because the one thing that + // fails for a reason which goes away by itself is a pull: the store, broker and registry come + // from the internet, and a rate-limited anonymous pull is not this mesh's fault. + let said = ""; + let step = ""; + for (let attempt = 1; attempt <= 3; attempt++) { + const ran = await on(command, 2_400_000); + said = ran.out; + log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`); + if (ran.ok) { step = ""; break; } + step = stepIn(said); + if (attempt < 3) { + log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`); + await new Promise((r) => setTimeout(r, 30_000)); + } + } + if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n")); + + // ------------------------------------------------------------------------------------------ + // Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting + // zero (novox/hq ADR 0018). + // ------------------------------------------------------------------------------------------ + + // 1. The control plane answers, asked of the PERMANENT container by name. + const answered = await on(`docker exec mesh-control /mesh-control status`, 60_000); + report.push(` control plane ${answered.ok ? answered.out.split("\n")[0] : "NO ANSWER"}`); + if (!answered.ok) return stop("after the last step", `mesh-control does not answer:\n${answered.out}`); + + // 2. The registry replies on /v2/. A container that is up is not a registry that serves. + const v2 = await on(`curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://${registry}/v2/`); + const v2Code = v2.out.trim(); + report.push(` registry /v2/ ${v2Code || "no answer"}`); + if (v2Code !== "200") return stop("after the last step", `the mesh's own registry answered ${v2Code || "nothing"}`); + + // 3. THE PIVOT COMPLETED — the running control plane is pinned by a digest THIS MESH'S REGISTRY + // assigned, not by an image id (ADR 0067 states this check in as many words). + const pinnedTo = (await on(`docker inspect --format '{{.Config.Image}}' mesh-control`)).out.trim(); + report.push(` pinned to ${pinnedTo || "(nothing)"}`); + if (/^sha256:[0-9a-f]{64}$/.test(pinnedTo)) { + return stop("after the last step", + `mesh-control is running from ${pinnedTo}, which is an IMAGE ID — the digest of the image's ` + + `own configuration, which no registry ever served. The pivot did not happen, so this mesh ` + + `cannot upgrade itself (novox/hq ADR 0067, "the pivot completed").`); + } + if (!new RegExp(`^${registry.replaceAll(".", "\\.")}/mesh-control@sha256:[0-9a-f]{64}$`).test(pinnedTo)) { + return stop("after the last step", + `mesh-control is running from ${pinnedTo || "nothing this bed could read"}, which is not a ` + + `digest assigned by ${registry}.`); + } + + // 3b. And the registry really serves it. A reference is a claim; a tag list is the registry agreeing. + const tags = await on(`curl -s --max-time 10 http://${registry}/v2/mesh-control/tags/list`); + report.push(` registry holds ${tags.out.trim() || "nothing"}`); + if (!tags.out.includes("genesis")) { + return stop("after the last step", + `${registry} does not serve mesh-control, so the digest the container is pinned to names an ` + + `image nothing can pull: ${tags.out.trim()}`); + } + + // 4. The temporary control plane is GONE. The name is the audit. + const temp = await on(`docker inspect --format '{{.State.Status}}' temp-mesh-control`); + report.push(` temp-mesh-control ${temp.ok ? `STILL HERE (${temp.out.trim()})` : "gone"}`); + if (temp.ok) { + return stop("after the last step", + `temp-mesh-control is still ${temp.out.trim()}. Two control planes are consuming this mesh's ` + + `broker queues; neither is wrong and the pivot is not finished.`); + } + + // 5. And the mesh has heard from its one node. + const nodes = await on(`docker exec mesh-control /mesh-control node list`); + report.push(` node list ${nodes.out.trim().split("\n").join(" | ")}`); + const line = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${node} `)); + if (!line || !/^\S+\s+here\b/.test(line)) { + return stop("after the last step", `the mesh has not heard from ${node}: ${line ?? "it has no record of it at all"}`); + } + + report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`); + return { ok: true, step: "", why: "", report }; +}