diff --git a/README.md b/README.md index cf672cc..d0b878f 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,18 @@ export MESH_LAB_BUNDLE=/examples/substrate-first-node.lock export MESH_LAB_MODULES=/examples/modules export MESH_LAB_BUILDER=/build/mesh-builder # build/, which is git-ignored +# The installer. `suite` builds it with mesh-host's `make bootstrap`, which embeds a `docker save` +# of the control-plane image — so it is built AFTER that image, in the same run, or it carries a +# stale one sealed inside a binary where nothing would ever notice. The whole-mesh bed raises its +# anchor by RUNNING this, rather than by applying a substrate bundle itself (novox/hq ADR 0067). +export MESH_LAB_BOOTSTRAP_BINARY=/mesh-bootstrap +export MESH_LAB_CONTROL_IMAGE=mesh-control:development # optional; what it carries + +# A checkout of the mesh's catalogue. The installer reads the registry's and the control plane's +# manifests from a copy of it ON THE MACHINE, because at genesis there is no forge, no build +# machine and — until the registry is up — nothing serving anything. +export MESH_LAB_CATALOG=/modules + # Built with `go build -o ./examples/` in mesh-control. export MESH_LAB_PROVISIONER=/postgres-provisioner export MESH_LAB_OBJECTSTORE_PROVISIONER=/objectstore-provisioner diff --git a/scenarios/whole-mesh-full.yml b/scenarios/whole-mesh-full.yml index 9ff06f5..1c9df35 100644 --- a/scenarios/whole-mesh-full.yml +++ b/scenarios/whole-mesh-full.yml @@ -42,6 +42,15 @@ # network on the container port, so the host side is free to move). Reported as a topology finding. # # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock +# MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap MESH_LAB_CATALOG=.../mesh-catalog/modules +# +# GENESIS AND JOINING ARE TWO DIFFERENT ACTS, and this bed distinguishes them. novox is brought +# into existence by `mesh-bootstrap` — the same program a bare machine runs — and is afterwards a +# working mesh of one, with a registry and a control plane that is an ordinary module pinned to an +# image that registry serves. ace, shanks and g14 then JOIN it: host binary, token, enrol, run. No +# bootstrap, no substrate, no registry. novox is never enrolled twice, because the installer +# already did it. +# # The images: are the UNION of the novox set (feat/novox-conversions @ 431310f: the slug + roundcube # fixes, so only-office/de-spiegel/amqp-email-forwarder now resolve) and the ace media/home set, and # every one of them must already be BUILT on the workstation — nothing can fetch them. @@ -76,12 +85,18 @@ machines: memory: 24GiB cpus: 8 disk: 130GiB - # The substrate's control plane, the proxy, and a runtime per module novox is assigned. - # step-ca, photos, invoicing, novox.be, only-office, de-spiegel, amqp-email-forwarder, registry, - # firewall and fail2ban are not here because they carry no runtime image of their own — what - # they run is third-party or is the node itself. + # The proxy, and a runtime per module novox is assigned. step-ca, photos, invoicing, novox.be, + # only-office, de-spiegel, amqp-email-forwarder, registry, firewall and fail2ban are not here + # because they carry no runtime image of their own — what they run is third-party or is the + # node itself. + # + # **mesh-control is NOT here, and its absence is the point** (novox/hq ADR 0067). The anchor is + # brought into existence by the installer, and the installer carries the control plane's image + # inside itself — that is the whole reason a machine that can reach no registry can still raise + # a mesh. Handing it over from the workstation as well would mean the bed never found out + # whether the installer really carries it: the load would say "already held" and the fiction + # would be invisible, which is exactly the class of thing the lab's own registry used to hide. images: - - mesh-control:development - mesh-route-proxy:development - mesh-runtime-postgres:development - mesh-runtime-redis:development @@ -164,7 +179,6 @@ machines: # a machine holds it because it was handed it. Everything else the modules run comes from the # internet and is not named here at all. images: - - mesh-control:development # --- per-module runtimes (union) --- - mesh-runtime-postgres:development - mesh-runtime-redis:development diff --git a/src/lifecycle/operate.ts b/src/lifecycle/operate.ts index 9dc8165..e85a712 100644 --- a/src/lifecycle/operate.ts +++ b/src/lifecycle/operate.ts @@ -61,11 +61,45 @@ export async function exec( */ timeoutMs = 120_000, ): Promise<{ stdout: string; stderr: string }> { + const name = await instanceNameOf(instanceId, machine); + return incus(["exec", name, "--", ...command], timeoutMs); +} + +/** + * What the hypervisor calls one of a scenario's machines. + * + * Asked of the daemon by the metadata each instance carries, and only derived from the naming + * rule when it answers nothing — the same order `exec` has always used. It is exported because + * the placement stage takes this name rather than the pair, and a caller that wants to put + * something on one machine after the raise (the installer, a catalogue checkout) would otherwise + * have to reimplement the lookup and get the fallback wrong. + */ +export async function instanceNameOf(instanceId: string, machine: string): Promise { const found = (await taggedInstances()).find( (i) => i.instanceId === instanceId && i.machine === machine, ); - const name = found?.name ?? machineName(instanceId, machine); - return incus(["exec", name, "--", ...command], timeoutMs); + return found?.name ?? machineName(instanceId, machine); +} + +/** + * Put a file from this workstation inside a machine. + * + * The long default timeout is not generosity: what goes through here is an installer carrying a + * container image, which is tens of megabytes, and a push that is merely slow must not look like + * a push that is stuck. + */ +export async function push( + instanceId: string, + machine: string, + local: string, + remote: string, + mode?: string, + timeoutMs = 900_000, +): Promise { + const name = await instanceNameOf(instanceId, machine); + const args = ["file", "push", local, `${name}${remote}`]; + if (mode) args.push("--mode", mode); + await incus(args, timeoutMs); } /** diff --git a/src/lifecycle/place.ts b/src/lifecycle/place.ts index 19f6794..be85aec 100644 --- a/src/lifecycle/place.ts +++ b/src/lifecycle/place.ts @@ -42,6 +42,17 @@ export function isPlaceable(artifact: string): boolean { /** Where the host binary lives on a machine once placed. */ export const HOST_PATH = "/usr/local/bin/mesh-host"; +/** + * Where the installer lives on a machine once placed. + * + * **Beside the host, because it is the same tier and the same delivery** (novox/hq ADR 0067): + * bootstrapping is done by hand and it changes a machine, which is what tier 0 is — but `mesh-host` + * says of itself that it connects to nothing and listens on nothing, and an installer that loads + * images and interrogates a control plane cannot be folded into it without making that sentence + * false. Two programs, one shelf. + */ +export const BOOTSTRAP_PATH = "/usr/local/bin/mesh-bootstrap"; + export interface Placement { machine: string; artifacts: string[]; @@ -78,6 +89,14 @@ export function hostBinaryPath(): string | null { return process.env["MESH_LAB_HOST_BINARY"] ?? null; } +/** + * The installer to place, from the environment. Same rule as the host binary: an explicit path, + * and nothing is guessed. + */ +export function bootstrapBinaryPath(): string | null { + return process.env["MESH_LAB_BOOTSTRAP_BINARY"] ?? null; +} + export class PlacementError extends Error { readonly machine: string; @@ -146,6 +165,41 @@ export async function placeHost( return { machine, profile, version }; } +/** + * Put the installer on a machine and ask it what it is. + * + * The same shape as {@link placeHost} and for the same reason: the version is read back from the + * running binary, because a file arriving is not a program working (novox/hq ADR 0018). The + * installer is the larger of the two by an order of magnitude — it carries a saved container image + * — so a copy that half-arrived is a real possibility rather than a theoretical one. + * + * It is placed on ONE machine, not all of them. Only the anchor is bootstrapped; every other + * machine joins a mesh that already exists, with the host binary and a token and nothing else. + */ +export async function placeBootstrap( + instanceName: string, + machine: string, + binary: string, + log: (message: string) => void = () => {}, +): Promise { + await incus(["file", "push", binary, `${instanceName}${BOOTSTRAP_PATH}`, "--mode", "0755"], + 900_000); + + const version = (await incusOk( + ["exec", instanceName, "--", BOOTSTRAP_PATH, "version"], 60_000, + ))?.trim(); + if (!version) { + throw new PlacementError( + machine, + `the installer was copied to ${machine} and does not run there. It carries a saved ` + + `container image and is twenty megabytes or so, which is exactly the size at which a ` + + `truncated copy stops being theoretical.`, + ); + } + log(` placed the installer on ${machine} (${version})`); + return version; +} + /** Place everything a scenario declares. */ export async function applyPlacements( scenario: Scenario, diff --git a/src/rebuild.ts b/src/rebuild.ts index c945348..d12336c 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -2,7 +2,8 @@ * Rebuild what the lab runs, from source, before it runs. * * **A stale artifact reporting success against old rules is the fault this project keeps writing - * down** (novox/hq 04-ISSUES/005). The lab consumes three artifacts from two repositories, and they + * down** (novox/hq 04-ISSUES/005). The lab consumes a handful of artifacts from two repositories, + * and they * were rebuilt by hand, one at a time, from memory. A rename in the control plane's catalogue needs * both the control-plane image *and* the builder binary, because both parse manifests; rebuilding * one left a binary eleven hours old refusing a field the mesh had just renamed, and cost a full @@ -73,9 +74,40 @@ export function planned(env: NodeJS.ProcessEnv = process.env): Build[] { }); } } + + // **The installer, carrying the control plane's image.** + // + // Last, and that is an ordering rather than a preference: `make bootstrap` embeds the output of + // `docker save `, so the image has to have been built by the step above or the installer + // carries whatever was lying around — the eleven-hour-old artifact again, this time inside a + // binary where nothing would ever notice. + // + // It is built here at all because the bed now bootstraps THROUGH it (novox/hq ADR 0067): the + // anchor is brought into existence by running the same program a bare machine runs, rather than + // by the bed applying a substrate bundle by hand and calling that an install. An installer that + // was stale would be a bed proving something about last week's procedure. + const installer = env["MESH_LAB_BOOTSTRAP_BINARY"]; + if (installer && where["mesh-host"]) { + builds.push({ + what: "installer", + in: where["mesh-host"], + argv: ["make", "bootstrap", `IMAGE=${controlPlaneImage(env)}`, `BOOTSTRAP_OUT=${installer}`], + }); + } return builds; } +/** + * The control-plane image the installer carries. + * + * `mesh-control:development` is what mesh-control's `make image` tags, and what the scenarios name + * — one tag, said in one place. It is overridable because a release installer carries a release + * image, and nothing about that is the lab's business. + */ +export function controlPlaneImage(env: NodeJS.ProcessEnv = process.env): string { + return env["MESH_LAB_CONTROL_IMAGE"] ?? "mesh-control:development"; +} + /** rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. */ export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] { const built: string[] = []; diff --git a/test/integration/whole-mesh-full.test.ts b/test/integration/whole-mesh-full.test.ts index dbc3e62..1d12af0 100644 --- a/test/integration/whole-mesh-full.test.ts +++ b/test/integration/whole-mesh-full.test.ts @@ -12,6 +12,25 @@ * There is NO separate anchor: novox IS the anchor. The substrate runs on novox, and novox also * enrols as a node and receives its own service set — the substrate host and a service node at once. * + * TWO ACTS, AND THE BED NOW DISTINGUISHES THEM (novox/hq ADR 0067). + * + * GENESIS — novox is brought into existence by `mesh-bootstrap`, the installer, run on the + * machine exactly as a person would run it on a bare one: preflight, load the carried + * control-plane image, write the bundle, apply, verify, enrol itself, install the registry + * module, push the control-plane image into it, reinstall the control plane as an ordinary + * module pinned to the digest that push produced, retire the temporary one. Afterwards novox + * is a WORKING MESH OF ONE, and this bed asserts exactly that before going any further. + * + * JOINING — ace, shanks and g14 then join a mesh that already exists: host binary, token, + * `enrol`, run the agent. No bootstrap, no substrate, no registry. novox is NOT enrolled again. + * + * The bed used to do neither. It applied the substrate bundle itself and looped enrolment over all + * four machines as one continuous operation — which got the order right by accident and modelled + * the wrong shape, and is why ADR 0067's own acceptance check ("the bed bootstraps through the + * installer rather than around it") went unmet. Genesis GATES joining: if it stops, the bed says + * which of the installer's ten steps it stopped at and goes no further, because a second machine + * joining a mesh that is not ready is a different failure and must not be mistaken for this one. + * * THE THING THIS BED EXISTS TO PROVE (the flat beds never could): does the WireGuard overlay tunnel * FORM across the access point? A home node (ace/shanks/g14) dials novox's PUBLIC hub endpoint * 192.0.2.20:51820/udp OUT through the household gateway's masquerade; the handshake has to complete @@ -32,21 +51,26 @@ * behaves like every other: raise in before(), destroy in after(). * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock + * MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap MESH_LAB_CATALOG=.../mesh-catalog/modules */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; -import { destroy, exec } from "../../src/lifecycle/operate.ts"; -import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { destroy, exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts"; +import { + bootstrapBinaryPath, hostBinaryPath, placeBootstrap, BOOTSTRAP_PATH, HOST_PATH, +} from "../../src/lifecycle/place.ts"; import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; @@ -56,15 +80,51 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" - : false; + : !installer || !existsSync(installer) + ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap (mesh-host `make " + + "bootstrap IMAGE=mesh-control:development`). The anchor is raised BY the installer now, " + + "so a run without one would be testing the procedure this bed exists to stop testing" + : false; const SCENARIO = "whole-mesh-full"; /** novox hosts the substrate and the control plane; it is where `mesh` commands run. */ const CONTROL = "novox"; -/** Every node that enrols. novox is on hosting; the rest are behind the home gateway. */ +/** Every node in the mesh. novox is on hosting; the rest are behind the home gateway. */ const NODES = ["novox", "ace", "shanks", "g14"]; +/** + * The machines that JOIN. novox is not one of them, and that is the distinction this bed was + * restructured to make: novox is brought into existence by the installer, which enrols it as part + * of genesis. Enrolling it again here would be a second identity the mesh does not know. + */ const HOME_NODES = ["ace", "shanks", "g14"]; +/** A checkout of the mesh's catalogue, on the anchor, for the installer to read manifests from. */ +const CATALOGUE_ON_MACHINE = "/opt/mesh-catalog"; + +/** + * The manifests `mesh-bootstrap` reads out of that checkout — and only those. + * + * Named here rather than pushing the whole repository because the whole repository is a hundred + * megabytes of `node_modules` and the installer opens exactly two files: mesh-host's + * `internal/bootstrap` RegistryModule and ControlPlaneModule. If it ever opens a third, this list + * is where the bed finds out, by the installer saying which manifest it could not read. + */ +const CATALOGUE_MODULES = ["registry", "mesh-control"]; + +/** + * Where this mesh keeps its own images, as the anchor reaches it. + * + * **Loopback, and that is a finding rather than a shortcut.** The mesh's registry is plain HTTP on + * purpose — it is reached over the mesh's own network, which is already the encrypted and + * authenticated thing — and a container runtime refuses a plain-HTTP registry at any address + * EXCEPT a loopback one unless it has been told to allow it. So genesis can push to 127.0.0.1:5000 + * with no configuration, and the reference the control-plane module is then pinned to is one only + * the anchor can pull. On this bed that is enough, because only the anchor runs the control plane. + * A mesh where a second machine had to pull it would need the runtimes told about the registry + * first, and nothing in the design says who does that. + */ +const MESH_REGISTRY = "127.0.0.1:5000"; + /** * ADR 0066 — the domain each public-facing node composes its routed names under. * @@ -127,6 +187,11 @@ const NOVOX: Mod[] = [ { name: "amqp-email-forwarder", containers: ["amqp-email-forwarder"] }, { name: "portainer", containers: ["portainer", "mesh-portainer"] }, { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] }, + // Already installed, assigned and running — genesis needed it to publish the control plane's + // image. Left in the plan on purpose: registering the same manifest and assigning it again is + // what an operator's `module add` + `assign` would do on a mesh that already has it, and a + // module the installer put there had better survive being asked for a second time. It also keeps + // mesh-registry in the convergence report, where a reader expects to see it. { name: "registry", containers: ["mesh-registry"] }, { name: "mailu", @@ -404,6 +469,227 @@ async function overlayAddr(node: string): Promise { return out.split("\n").map((l) => l.trim()).find(Boolean) ?? ""; } +// ================================================================================================== +// PHASE 1 — GENESIS. novox is brought into existence by the installer. +// ================================================================================================== + +/** What genesis did, or where it stopped. */ +interface GenesisResult { + ok: boolean; + /** `step 7 of 10, registry` — the installer's own words, so the bed reports the cause. */ + step: string; + why: string; + report: string[]; +} + +/** The step a failed `mesh-bootstrap` names, as it prints it, or "" if it named none. */ +function stepIn(said: string): string { + return said.match(/^mesh-bootstrap: (step \d+ of \d+, [a-z-]+):/m)?.[1] ?? ""; +} + +/** + * Put on the anchor what the installer needs to read, and run it. + * + * **This is the whole of what changed, and it is not a refactor.** The bed used to apply the + * substrate bundle itself, by hand, and then enrol four machines in one loop. It got the order + * right by accident and it modelled the wrong shape: an install procedure that exists only as a + * test fixture is exercised by whoever writes tests and never by whoever installs, which is why + * every bootstrap fault this year was found late (novox/hq ADR 0067). The anchor is now raised by + * running the same program a bare machine runs, and the bed only reads the result. + * + * It is run up to three times. Not to paper over a failure — every attempt's failing step is + * printed — but because `mesh-bootstrap` is idempotent by design and says so, and because the one + * thing here that fails for a reason which goes away by itself is a pull: the store, the broker and + * the registry come from the internet, through a household gateway's masquerade, and Docker Hub + * rate-limiting an anonymous pull is not this mesh's fault. A re-run is the retry, and it is the + * retry the installer's own documentation names. + */ +async function genesis(images: HeldImage[]): Promise { + const report: string[] = ["================ GENESIS: novox becomes a mesh of one ================"]; + const stop = (step: string, why: string): GenesisResult => { + report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`); + return { ok: false, step, why, report }; + }; + + // The installer, beside the host binary. Everything else on this machine was placed by `raise`; + // this one is placed here because only the anchor is bootstrapped. + const name = await instanceNameOf(instanceId, CONTROL); + const version = await placeBootstrap(name, CONTROL, installer as string, + (m) => console.log(`genesis:${m}`)); + report.push(` installer ${version} at ${BOOTSTRAP_PATH}`); + + // The catalogue. `mesh-bootstrap --catalog` reads manifests from a CHECKOUT on the machine, + // because at this moment the mesh has no forge, no build machine and — until step 7 finishes — + // no registry. A manifest is a file, and somebody has to have put it there. + await must(CONTROL, `mkdir -p ${CATALOGUE_MODULES.map((m) => `${CATALOGUE_ON_MACHINE}/modules/${m}`).join(" ")}`); + for (const module of CATALOGUE_MODULES) { + const from = resolve(catalogDir, module, "module.json"); + assert.ok(existsSync(from), `the catalogue has no ${module}/module.json at ${from}`); + await push(instanceId, CONTROL, from, `${CATALOGUE_ON_MACHINE}/modules/${module}/module.json`); + } + report.push(` catalogue ${CATALOGUE_MODULES.join(", ")} at ${CATALOGUE_ON_MACHINE}`); + + // The substrate TEMPLATE — not the bundle. The installer produces the bundle from it: it replaces + // the control plane's image with the id of the image it carries, renames that container + // `temp-mesh-control`, and writes the result where a person can read it. + // + // Two substitutions still happen here, and both belong to the bed rather than to the installer. + // The example names three images at a registry the lab no longer raises: the store and the broker + // become the upstream references mesh-catalog pins (harness), and the machine pulls them over its + // uplink like any first node. The third, mesh-control, is deliberately LEFT naming that dead + // registry — the installer overwrites it, and leaving it proves that it does. + // + // And the broker's advertised address. The template hardcodes 192.0.2.10:5671, the old + // separate-anchor address; a token carries MESH_BROKER_ADDRESS verbatim as the endpoint an + // enrolling node dials, so with the substrate on novox it must be novox's own public address or + // every node would enrol against a dead one. The installer refuses to guess this and says so + // loudly, which is right — it does not know what this machine is called from outside. + const template = bundleFor(images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671"); + const local = join(tmpdir(), `mesh-lab-substrate-${process.pid}.lock`); + writeFileSync(local, template); + await push(instanceId, CONTROL, local, "/tmp/substrate-template.lock"); + + const command = [ + BOOTSTRAP_PATH, + `--bundle /tmp/substrate-template.lock`, + `--catalog ${CATALOGUE_ON_MACHINE}`, + `--node ${CONTROL}`, + `--registry ${MESH_REGISTRY}`, + `--host ${HOST_PATH}`, + // The lab has no unit to supervise the host with, and the installer refuses to invent one — a + // unit file is a packaging decision. This is the arrangement it offers instead, and it is loud + // about what it is: a host started this way does not survive a reboot. + `--host-in-background`, + ].join(" "); + + let said = ""; + let step = ""; + for (let attempt = 1; attempt <= 3; attempt++) { + const ran = await on(CONTROL, command, 2_400_000); + said = ran.out; + console.log(`\n---- mesh-bootstrap on ${CONTROL} (attempt ${attempt}) ----\n${said}`); + if (ran.ok) { + step = ""; + break; + } + step = stepIn(said); + if (attempt < 3) { + console.log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; ` + + `re-running in 30s — every step it already did will say so`); + await new Promise((r) => setTimeout(r, 30_000)); + } + } + if (step) { + return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n")); + } + + // ------------------------------------------------------------------------------------------ + // And now the only thing that matters: is novox a WORKING MESH OF ONE? Asked of the machine, + // never inferred from the installer exiting zero (novox/hq ADR 0018). + // ------------------------------------------------------------------------------------------ + + // 1. The control plane answers. Asked of the PERMANENT container by name — `status` opens all + // three stores, so a reply proves the connections it was given are the substrate's own. + const answered = await on(CONTROL, `docker exec mesh-control /mesh-control status`, 60_000); + report.push(` control plane ${answered.ok ? answered.out.split("\n")[0] : "NO ANSWER"}`); + if (!answered.ok) return stop("after step 10", `mesh-control does not answer:\n${answered.out}`); + + // 2. The registry replies on /v2/ — the registry API's own "yes, I am one and I am ready". A + // container that is up is not a registry that serves. + const v2 = await on(CONTROL, + `curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://${MESH_REGISTRY}/v2/`); + const v2Code = v2.out.trim(); + report.push(` registry /v2/ ${v2Code || "no answer"}`); + if (v2Code !== "200") return stop("after step 10", `the mesh's own registry answered ${v2Code || "nothing"}`); + + // 3. THE PIVOT COMPLETED. ADR 0067 states this check in as many words: after installing, the + // running control plane's image is pinned by a digest THE MESH'S OWN REGISTRY ASSIGNED — not + // by an image id. If it is still an image id, the substrate's container is what is running, + // nothing was published, and this mesh can never roll out its own upgrades. + const pinnedTo = (await on(CONTROL, `docker inspect --format '{{.Config.Image}}' mesh-control`)) + .out.trim(); + report.push(` pinned to ${pinnedTo || "(nothing)"}`); + if (/^sha256:[0-9a-f]{64}$/.test(pinnedTo)) { + return stop("after step 10", + `mesh-control is running from ${pinnedTo}, which is an IMAGE ID — the digest of the image's ` + + `own configuration, which no registry ever served. The pivot did not happen: what is ` + + `running is the image the installer carried, not one this mesh published, so this mesh ` + + `cannot upgrade itself (novox/hq ADR 0067, "the pivot completed").`); + } + if (!new RegExp(`^${MESH_REGISTRY.replaceAll(".", "\\.")}/mesh-control@sha256:[0-9a-f]{64}$`) + .test(pinnedTo)) { + return stop("after step 10", + `mesh-control is running from ${pinnedTo || "nothing this bed could read"}, which is not a ` + + `digest assigned by ${MESH_REGISTRY}.`); + } + + // 3b. And the registry really serves it, asked of the registry rather than of the container. A + // reference is a claim; a tag list is the registry agreeing. + const tags = await on(CONTROL, + `curl -s --max-time 10 http://${MESH_REGISTRY}/v2/mesh-control/tags/list`); + report.push(` registry holds ${tags.out.trim() || "nothing"}`); + if (!tags.out.includes("genesis")) { + return stop("after step 10", + `${MESH_REGISTRY} does not serve mesh-control, so the digest the container is pinned to ` + + `names an image nothing can pull: ${tags.out.trim()}`); + } + + // 4. The temporary control plane is GONE. Two control planes is the half-finished state, and the + // name is the audit: a machine running mesh-control and not temp-mesh-control has pivoted. + const temp = await on(CONTROL, `docker inspect --format '{{.State.Status}}' temp-mesh-control`); + report.push(` temp-mesh-control ${temp.ok ? `STILL HERE (${temp.out.trim()})` : "gone"}`); + if (temp.ok) { + return stop("after step 10", + `temp-mesh-control is still ${temp.out.trim()}. Two control planes are consuming this ` + + `mesh's broker queues; neither is wrong and the pivot is not finished.`); + } + + // 5. And the mesh has heard from its one node. Everything the join phase does next depends on it. + const nodes = await on(CONTROL, `docker exec mesh-control /mesh-control node list`); + report.push(` node list ${nodes.out.trim().split("\n").join(" | ")}`); + const line = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${CONTROL} `)); + if (!line || !/^\S+\s+here\b/.test(line)) { + return stop("after step 10", + `the mesh has not heard from ${CONTROL}: ${line ?? "it has no record of it at all"}`); + } + + report.push(`\nVERDICT: ${CONTROL} is a working mesh of one, bootstrapped through the installer.`); + return { ok: true, step: "", why: "", report }; +} + +// ================================================================================================== +// PHASE 2 — JOINING. Everything else is a machine joining a mesh that already exists. +// ================================================================================================== + +/** + * ace, shanks and g14 join. Host binary plus a token — no bootstrap, no substrate, no registry. + * + * **novox is not in this loop.** It was enrolled by the installer, as part of becoming a mesh, and + * enrolling it again would present the mesh with a second identity for a node it already knows — + * which `mesh-host enrol` refuses, and rightly. + * + * The home nodes reach novox's public 192.0.2.20:5671 by dialling OUT through the household + * gateway, so the enrol itself is the first proof that outbound home→public works. + */ +async function joinTheMesh(): Promise { + // ADR 0066: said as soon as the record exists, because everything routed is composed from it. A + // node that faces the outside has one; the workstations do not, and are given none. The anchor's + // is set here rather than in genesis because it is a fact about the mesh, not part of raising + // one — and the installer has an opinion about neither. + const anchorDomain = PUBLIC_DOMAIN[CONTROL]; + if (anchorDomain) await mesh(`node public-domain ${CONTROL} ${anchorDomain}`); + + for (const machine of HOME_NODES) { + await mesh(`node add ${machine}`); + const domain = PUBLIC_DOMAIN[machine]; + if (domain) await mesh(`node public-domain ${machine} ${domain}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); + assert.match(said, new RegExp(`enrolled as ${machine}`), said); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + } +} + before(async () => { if (skip) return; assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); @@ -416,67 +702,35 @@ before(async () => { held = raised.images; console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); - // novox raises the substrate from its bundle. The store and the broker keep upstream references - // and novox PULLS them, over its uplink, the way any first node does; mesh-control exists in no - // registry, so it becomes the ID novox holds it under — the whole of what changed here. + // ---- PHASE 1, and it GATES phase 2 --------------------------------------------------------- // - // The rest is the collapse: the substrate rides novox, not a separate anchor. The bundle hardcodes the - // broker's advertised address as 192.0.2.10:5671 (the OLD separate-anchor address) — and a token - // carries MESH_BROKER_ADDRESS verbatim as the endpoint an enrolling node dials. With the substrate - // on novox that endpoint must be novox's own public address, or every node (novox included) would - // enrol against a dead address. The broker serves its cert on all interfaces and the token pins by - // fingerprint, not hostname, so only the address needs correcting. - const bundleText = bundleFor(raised.images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671"); - await must(CONTROL, `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleText}\nMESHBUNDLE`); + // Caught rather than allowed to propagate, so that a failure BEFORE the installer ran — a + // manifest that is not where the bed thought, a binary that would not copy — is reported in the + // same shape as one the installer itself named, instead of as a bare stack trace from a helper. + let genesisResult: GenesisResult; + try { + genesisResult = await genesis(raised.images); + } catch (err) { + genesisResult = { + ok: false, + step: "getting the anchor ready to be bootstrapped — the installer never ran", + why: (err as Error).message, + report: ["================ GENESIS: novox becomes a mesh of one ================"], + }; + } + console.log(genesisResult.report.join("\n")); + assert.ok(genesisResult.ok, + `GENESIS FAILED — ${genesisResult.step || "no step named"}.\n\n${genesisResult.why}\n\n` + + `No other machine was asked to join. A second machine joining a mesh that is not ready is a ` + + `different failure with a different cause, and running it now would bury this one under it.\n\n` + + genesisResult.report.join("\n")); - // **Belt as well as braces on fetching.** `raise` refuses to return until every machine with - // egress has resolved a name and reached the internet, so the first attempt should be the only - // one. This retry is here because of what the failure looked like when there was no such - // guarantee: the apply died on a pull, `before` threw, and the instance was left a bare shell — - // VMs, no substrate, no enrolment, nothing to read. - // - // And a pull is now genuinely the one step that can fail for a reason which goes away by itself: - // the store and the broker come from the internet, through a household gateway's masquerade, and - // a registry elsewhere having a bad minute is not this mesh's fault. That is the trade this bed - // accepts — it is no longer hermetic, because a real node is not either, and the faults it was - // hiding were exactly the ones that only appear when a machine has to fetch for itself. - { - let applied = false; - let said = ""; - for (let attempt = 1; attempt <= 3 && !applied; attempt++) { - const tried = await on(CONTROL, `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); - applied = tried.ok; - said = tried.out; - if (!applied && attempt < 3) { - console.log(`substrate apply attempt ${attempt} failed; retrying in 30s:\n${said.split("\n").slice(-8).join("\n")}`); - await new Promise((r) => setTimeout(r, 30_000)); - } - } - assert.ok(applied, `the substrate did not apply on novox after three attempts:\n${said}`); - } - const up = await must(CONTROL, `docker ps --format '{{.Names}}'`); - for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { - assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); - } - - // Every node joins the one mesh and runs a host. The home nodes reach novox's public 192.0.2.20:5671 - // by dialling OUT through the household gateway — the enrol itself is the first proof that outbound - // home→public works. novox enrols too: substrate host and service node at once. - for (const machine of NODES) { - await mesh(`node add ${machine}`); - // ADR 0066: said as soon as the record exists, because everything routed is composed from it. - // A node that faces the outside has one; the workstations do not, and are given none. - const domain = PUBLIC_DOMAIN[machine]; - if (domain) await mesh(`node public-domain ${machine} ${domain}`); - const token = tokenFrom(await mesh(`token issue --node ${machine}`)); - const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); - assert.match(said, new RegExp(`enrolled as ${machine}`), said); - await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); - } + // ---- PHASE 2 -------------------------------------------------------------------------------- + await joinTheMesh(); // The operator provides ace's media library (ADR 0051 accesses confirm the paths, create nothing). await must("ace", `mkdir -p ${MEDIA_DIRS.join(" ")}`); -}, { timeout: 3_600_000 }); +}, { timeout: 5_400_000 }); after(async () => { if (KEEP) { diff --git a/test/rebuild.test.ts b/test/rebuild.test.ts index cffe81b..d8b04e3 100644 --- a/test/rebuild.test.ts +++ b/test/rebuild.test.ts @@ -1,7 +1,8 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { planned } from "../src/rebuild.ts"; +import { planned, controlPlaneImage } from "../src/rebuild.ts"; import { repositories } from "../src/repos.ts"; +import { loadScenario } from "../src/declaration/parse.ts"; // The control plane's image and the builder are one step, not two. // @@ -39,6 +40,52 @@ test("every image the lab runs is rebuilt, not only the control plane's", () => } }); +// The installer is built, and it is built AFTER the image it carries. +// +// `make bootstrap` embeds the output of `docker save `, so an installer built before the +// control plane's image is one carrying whatever was lying around — 04-ISSUES/005 again, this time +// sealed inside a binary where nothing would ever notice. The bed raises its anchor by running this +// program (novox/hq ADR 0067), so a stale one is a bed proving something about last week. +test("the installer is built, carrying the image built in the same run", () => { + const builds = planned({ + MESH_LAB_HOST_BINARY: "/repo/host/mesh-host", + MESH_LAB_MODULES: "/repo/control/examples/modules", + MESH_LAB_BOOTSTRAP_BINARY: "/repo/host/mesh-bootstrap", + }); + const what = builds.map((b) => b.what); + assert.ok(what.includes("installer"), "the installer is never built, so the bed carries a stale one"); + assert.ok( + what.indexOf("images") < what.indexOf("installer"), + `the installer is built before the image it embeds: ${what.join(", ")}`, + ); + + const installer = builds.find((b) => b.what === "installer")!; + assert.equal(installer.in, "/repo/host"); + assert.ok(installer.argv.includes(`IMAGE=${controlPlaneImage({})}`), installer.argv.join(" ")); + assert.ok(installer.argv.includes("BOOTSTRAP_OUT=/repo/host/mesh-bootstrap"), installer.argv.join(" ")); +}); + +// The anchor must NOT be handed the control plane's image. +// +// The installer carries it, which is the whole reason a machine that can reach no registry can +// raise a mesh (novox/hq ADR 0067). Hand it over from the workstation as well and the installer's +// load says "already held", the carrying is never exercised, and the bed goes green on a fiction — +// the same class of thing the lab's own registry used to hide. Asserted on the file rather than +// remembered, because a list of images is exactly the kind of thing somebody tops up. +test("the whole-mesh bed hands its anchor no control-plane image", () => { + const scenario = loadScenario("scenarios/whole-mesh-full.yml"); + const named = [ + ...(scenario.images ?? []), + ...Object.values(scenario.machines).flatMap((m) => m.images ?? []), + ]; + assert.deepEqual( + named.filter((i) => i.startsWith("mesh-control")), + [], + "the anchor is handed mesh-control, so genesis would never find out whether the installer " + + "really carries it", + ); +}); + // A repository this run was not pointed at is not built, and not claimed. test("only what this run was pointed at is built", () => { assert.deepEqual(planned({}), []);