From 7641bb20592fb470db8d20d9c0d47ffa9e9b1325 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 14 Sep 2026 21:52:24 +0200 Subject: [PATCH] A one-node mesh, and twelve things that have to be true of it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The common case, and the one that was never tested as a whole. What existed asked whether four machines converged; it never asked whether ONE machine ends up holding a mesh. The order was wrong too. Three machines were enrolled second, into a mesh that could not yet produce a single module, and that was reported as though something had been shown. 17-raising-a-mesh is explicit: genesis ends with a mesh that RUNS, and what remains after the core modules are built is "adding machines". So the core comes first and machines arrive last — here, not at all, because a second node is only meaningful once the first is complete. Three things were missing entirely and nothing complained, because nothing asked: the mesh never built its own catalogue, never had a store of its own for that catalogue to use, and never rebuilt its own control plane through the module path. And four checks that were absent rather than failing: - it can describe itself — status, module list, plan --json, and the catalogue's five tools ASKED rather than observed. A container being up was being read as the catalogue working, which is the same error as matching a container by substring and finding the wrong one. - its networking is what the modules asked for — default closed, ssh open, declared ports open, .internal names written, module networks present. Left out altogether, which is hard to defend given the firewall work this week. - a change to a module's source reaches the machine on its own. The capability the migration depends on. - it comes back after a reboot. Never once tested; the lab had no way to restart a machine, because nothing had ever needed one. Machines are named by role now — anchor, home-server, workstation, laptop — not after the operator's own nodes, which made test output and real state hard to tell apart. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- scenarios/fresh-mesh.yml | 22 +- scenarios/one-node-mesh.yml | 56 +++ test/integration/fresh-mesh.test.ts | 216 +++++--- test/integration/one-node-mesh.test.ts | 649 +++++++++++++++++++++++++ 4 files changed, 859 insertions(+), 84 deletions(-) create mode 100644 scenarios/one-node-mesh.yml create mode 100644 test/integration/one-node-mesh.test.ts diff --git a/scenarios/fresh-mesh.yml b/scenarios/fresh-mesh.yml index 4a0370d..2a7f5c9 100644 --- a/scenarios/fresh-mesh.yml +++ b/scenarios/fresh-mesh.yml @@ -25,9 +25,9 @@ # the gap is a named failure rather than an absence. # # hosting (public, routable) home (private, behind the access point) -# novox 192.0.2.20 ── anchor ace 10.99.1.10 home server -# substrate, registry, shanks 10.99.1.20 workstation -# builder, control plane g14 10.99.1.30 workstation +# anchor 192.0.2.20 ── anchor home-server 10.99.1.10 home server +# substrate, registry, workstation 10.99.1.20 workstation +# builder, control plane laptop 10.99.1.30 workstation # # EGRESS IS NOT OPTIONAL HERE. With nothing loaded, a sealed machine stops at the installer's first # pull. Every machine has a way out, and it is a SECOND path: each still reaches the rest of the @@ -41,7 +41,7 @@ scenario: fresh-mesh segments: - # The routable segment. novox lives here; its public address is the broker endpoint every token + # The routable segment. anchor lives here; its public address is the broker endpoint every token # carries and the overlay hub the home nodes dial. hosting: kind: public @@ -63,10 +63,10 @@ machines: # The anchor. Raised by the installer into a mesh of one, and then asked to build. # # Sized for what it actually does here: the store, the broker, the registry, TWO control planes - # during the pivot, the builder, and a build workspace holding a Node toolchain image and an npm - # cache. It is NOT sized for the whole novox service set, because this bed does not run one — it + # during the pivot, the builder, and a build worksphome-server holding a Node toolchain image and an npm + # cache. It is NOT sized for the whole anchor service set, because this bed does not run one — it # proves the machinery that would produce it. - novox: + anchor: at: { segment: hosting, address: [192.0.2.20] } egress: true inbound: allow @@ -77,21 +77,21 @@ machines: # Three machines that JOIN. Host binary and a token, nothing else — no bootstrap, no substrate, # no registry. They are deliberately small: what they are here to prove is that a joined machine # can be given a module the mesh built, which is a question about credentials and not about load. - ace: + home-server: at: { segment: home, address: [10.99.1.10] } egress: true inbound: allow memory: 4GiB cpus: 2 disk: 25GiB - shanks: + workstation: at: { segment: home, address: [10.99.1.20] } egress: true inbound: allow memory: 3GiB cpus: 2 disk: 20GiB - g14: + laptop: at: { segment: home, address: [10.99.1.30] } egress: true inbound: allow @@ -102,5 +102,5 @@ machines: # **No `images:` key, and that is the whole point of this file.** Anything a machine holds here, it # pulled or the mesh built. See the header. -place: +plhome-server: all: [host, runtime] diff --git a/scenarios/one-node-mesh.yml b/scenarios/one-node-mesh.yml new file mode 100644 index 0000000..fb62b03 --- /dev/null +++ b/scenarios/one-node-mesh.yml @@ -0,0 +1,56 @@ +# ONE MACHINE, AND EVERYTHING A MESH HAS TO BE. Nothing is handed to it. +# +# The common case, and the one worth getting right first: a person with a single machine runs the +# installer and ends up with a mesh that works. Not a mesh that *runs* — `17-raising-a-mesh` is +# careful about that difference, and so is this scenario. Genesis ends with a substrate, a registry, +# a built control plane and a builder, and a mesh in that state cannot produce anything and holds no +# record of what it has. Calling that "up" is how the catalogue came to be missing from a test for +# weeks without anything complaining. +# +# So the test driving this asks the harder question: can this machine, given nothing but a container +# runtime and the host binary, end up holding +# +# - a substrate and a registry it pulled from the internet, +# - a control plane it BUILT, and then rebuilt from its own repository through the module path, +# - a builder that takes work over the broker, +# - the shared base every module with code of its own stands on, +# - a store of its own — the substrate's is the control plane's own plumbing, not a provider, +# - a catalogue, so it can say what it has and what a change reaches, +# - and a module of its own, built, provisioned and running. +# +# **There is no `images:` key, and that is the whole point of this file.** The four-machine scenario +# next door loads thirty-four of the mesh's own images from the workstation because it does not +# build them — a shape no real installation has. Here nothing is loaded. What the machine holds it +# either pulled from the internet or made. +# +# EGRESS IS NOT OPTIONAL. With nothing loaded, a sealed machine stops at the installer's first pull. +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap +# MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock +# MESH_LAB_CATALOG=.../mesh-catalog/modules +# MESH_LAB_SOURCE= MESH_LAB_SOURCE_REF= +scenario: one-node-mesh + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + # The address matters: the substrate template names the broker at a fixed address, and a token + # carries that verbatim as the endpoint an enrolling node dials. With one machine, that machine + # must BE it, or the mesh hands out an endpoint nothing answers on. + # + # Sized for what it actually does: the store, the broker, the registry, two control planes during + # the pivot, the builder, a Node toolchain and an npm cache in the build workspace, and then every + # core module it builds and runs on top of that. + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 12GiB + cpus: 6 + disk: 60GiB + +place: + all: [host, runtime] diff --git a/test/integration/fresh-mesh.test.ts b/test/integration/fresh-mesh.test.ts index 83c83fe..4f30d54 100644 --- a/test/integration/fresh-mesh.test.ts +++ b/test/integration/fresh-mesh.test.ts @@ -48,10 +48,10 @@ import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts"; import { genesis, type GenesisResult } from "./genesis.ts"; const SCENARIO = "fresh-mesh"; -const CONTROL = "novox"; -const HOME_NODES = ["ace", "shanks", "g14"]; +const CONTROL = "anchor"; +const HOME_NODES = ["home-server", "workstation", "laptop"]; /** The joined machine asked to run a mesh-built module. Any of the three would do. */ -const SECOND = "ace"; +const SECOND = "home-server"; /** The anchor's public address — what every other machine dials, and what its own token must name. */ const ANCHOR = "192.0.2.20"; @@ -74,9 +74,30 @@ const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" }; * record of, so it provides nothing to anything. A module asking for `amqp` is asking for a * provider in the graph, and this is it. No build: its image is upstream. */ -const PROVIDER = { module: "lavinmq", repo: "mesh-catalog", path: "modules/lavinmq" }; +const PROVIDER = { module: "lavinmq", repo: "mesh-catalog", path: "modules/lavinmq", container: "mesh-lavinmq" }; +/** + * The store, and the catalogue that cannot exist without it. + * + * Both were missing from this test entirely, and nothing complained, because nothing asked. A mesh + * with no catalogue holds no module graph — it cannot say what it has, what a module is made of, + * what a change reaches, or what must be rebuilt. It ran anyway, which is the point: "the mesh is + * up" was being read off genesis finishing. + */ +const STORE = { module: "postgres", repo: "mesh-catalog", path: "modules/postgres", container: "mesh-postgres" }; +const CATALOGUE = { module: "mesh-catalog", repo: "mesh-catalog", path: "modules/mesh-catalog", container: "mesh-catalog" }; +/** The control plane, rebuilt from its own repository — the step that ends the installer's tenure. */ +const CONTROL_PLANE = { module: "mesh-control", repo: "mesh-control", path: "", container: "mesh-control" }; /** Named once, because the step title is also how later steps say what they waited on. */ const NEEDS = "the mesh runs a broker for that module to talk to"; +const GENESIS = "a bare machine becomes a mesh of one, raised by the installer"; +const BASE_BUILT = "the mesh builds the shared base from source"; +const STORE_RUNS = "the mesh builds and runs a store of its own"; +const CATALOGUE_RUNS = "the mesh builds and runs its own catalogue"; +const CONTROL_REBUILT = "the mesh rebuilds its own control plane from source"; +const MODULE_BUILT = "the mesh builds a module standing on that base"; +const ANCHOR_RUNS = "the anchor runs the module the mesh built"; +const JOINED = "three machines join the mesh, and reach it over its private network"; +const SECOND_RUNS = "a joined machine runs the module the mesh built"; const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping" }; const capability = await labIsUsable(); @@ -162,6 +183,34 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } +/** A module the mesh has to make for itself: where its source is, and what it runs when it works. */ +interface CoreModule { module: string; repo: string; path: string; container: string } + +/** + * Build a module from source, install it, and wait for it to actually run. + * + * The whole sequence a module goes through, in one place because the mesh has to do it for several + * before it is a mesh at all: register the manifest, build it from its repository and path, issue + * the broker account its runtime needs, assign it, send it, and then ask the machine whether the + * container is up. + * + * **The account is not optional and is not swallowed.** A module's runtime is a tool host: it + * connects to the mesh's broker before doing anything. Without an account the mesh still fills the + * secret the module declares it owns — with a generated value — so the container starts, fails to + * parse a password as a credential document, and loops on a JSON syntax error mentioning no + * missing account. That cost a step that reported PASS. + */ +async function bringUp(m: CoreModule): Promise { + await registerModule(m.module, resolve(catalogDir, m.module, "module.json")); + const built = await mesh( + `build ${forgeUrl(m.repo)} --path ${m.path} --ref ${refFor(m.repo)} --wait 1200s`, 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + await mesh(`module issue ${m.module} --node ${CONTROL}`); + await mesh(`assign ${CONTROL} ${m.module}`); + await mesh(`push ${CONTROL}`, 600_000); + return `${built}\n${await waitForContainer(CONTROL, m.container)}`; +} + /** * Wait for one container, BY NAME, to be running. * @@ -262,7 +311,7 @@ before(async () => { // // The shared description, the same one `genesis-single` calls. The bundle is the TEMPLATE with // nothing held: no image is pre-resolved, because none is here to resolve to. - await step("novox becomes a mesh of one, raised by the installer", null, async () => { + await step("a bare machine becomes a mesh of one, raised by the installer", null, async () => { try { raised = await genesis({ instanceId, @@ -289,41 +338,29 @@ before(async () => { return raised.report.join("\n"); }); - // ---- 2. JOINING ----------------------------------------------------------------------------- + // ---- 2..6. THE MESH BECOMES ONE -------------------------------------------------------------- // - // Host binary and a token. novox is NOT in this loop — the installer enrolled it, and enrolling - // it again would offer the mesh a second identity for a node it already knows. - await step("three machines join it across the household gateway", - "novox becomes a mesh of one, raised by the installer", async () => { - const said: string[] = []; - for (const machine of HOME_NODES) { - await mesh(`node add ${machine}`); - const token = tokenFrom(await mesh(`token issue --node ${machine}`)); - const out = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); - assert.match(out, new RegExp(`enrolled as ${machine}`), out); - await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); - said.push(` ${machine} enrolled and running`); - } - const nodes = await mesh("node list"); - said.push(nodes.trim()); - return said.join("\n"); - }); + // **Joining used to be here, second, and that was the wrong order.** Three machines were enrolled + // into a mesh that could not yet produce a single module, and the step was reported as though + // something had been shown. They do join — reliably — but joining a mesh that can build nothing + // proves only that enrolment works, which was never the doubtful part. + // + // `17-raising-a-mesh` says it plainly: genesis ends with a mesh of one that RUNS, and that is not + // the same as a mesh that WORKS; what remains after the core modules are built is "adding + // machines, and deciding what they run". So everything a mesh needs to be a mesh happens first, + // and machines arrive at the end. - // ---- 3. THE MESH BUILDS THE SHARED BASE ----------------------------------------------------- - // // The toolchain and runtime every module with code of its own stands on. It is a module, and it // is built like one — cloned from the forge by the builder installing put here, compiled on the // machine, published into the mesh's own registry. - await step("the mesh builds the shared base from source", - "three machines join it across the household gateway", async () => { + await step(BASE_BUILT, GENESIS, async () => { // Registered from the manifest the builder will also read, so what the mesh holds and what it // builds are the same description of the same module. // // **Not swallowed.** This call used to end in `.catch(() => {})`, on the reasoning that the // base might already be known. It hid a real failure — the manifest was being named at a path // inside a container that had never seen it — and the step passed anyway, because a base with - // nothing to stand on builds whether or not the mesh has a record of it. The next step, which - // needs that record, is where it surfaced. + // nothing to stand on builds whether or not the mesh has a record of it. await registerModule(BASE.module, baseManifest); const built = await mesh( `build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000); @@ -331,9 +368,39 @@ before(async () => { return built; }); - // ---- 4. AND A MODULE STANDING ON IT --------------------------------------------------------- - await step("the mesh builds a module standing on that base", - "the mesh builds the shared base from source", async () => { + // A store of its own. **Not the substrate's.** The installer raises a store for the control + // plane to keep its own records in, the way it raises a broker — plumbing, not a module the mesh + // has any record of, so it provides nothing to anything. A module that wants a database wants a + // provider in the graph, and the catalogue below is the first thing to want one. + await step(STORE_RUNS, BASE_BUILT, () => bringUp(STORE)); + + // And the catalogue, which was missing from this test altogether. + // + // Without it the mesh holds no module graph: it cannot say what it has, what a module is made + // of, what a change to one reaches, or what must be rebuilt. A mesh in that state still runs, + // which is exactly how its absence went unnoticed — "the mesh is up" was being read off the + // installer finishing rather than off the mesh being able to answer anything. + await step(CATALOGUE_RUNS, STORE_RUNS, () => bringUp(CATALOGUE)); + + // The control plane, rebuilt from its own repository and rolled out. + // + // The installer built it once, which is what got the mesh running. Building it again THROUGH THE + // MODULE PATH — build, notice the version moved, roll it out — is a different claim: it is the + // moment the mesh stops depending on the installer for anything, and the first time the thing + // that performs an upgrade performs one on itself. + await step(CONTROL_REBUILT, CATALOGUE_RUNS, async () => { + const built = await mesh( + `build ${forgeUrl(CONTROL_PLANE.repo)} --ref ${sourceRef} --wait 1200s`, 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + const rolled = await mesh(`upgrade ${CONTROL_PLANE.module} roll-out`, 900_000); + // Asked of the machine rather than believed from the command: the control plane that answers + // afterwards is the one that has to be running for anything below this line to mean anything. + await waitForContainer(CONTROL, CONTROL_PLANE.container); + return `${built}\n${rolled}`; + }); + + // ---- 7..8. SOMETHING TO RUN --------------------------------------------------------------- + await step(MODULE_BUILT, CONTROL_REBUILT, async () => { await registerModule(MODULE.module, resolve(catalogDir, MODULE.module, "module.json")); const built = await mesh( `build ${forgeUrl(MODULE.repo)} --path ${MODULE.path} --ref ${refFor(MODULE.repo)} --wait 1200s`, @@ -347,61 +414,64 @@ before(async () => { return `${built}\n${builds}`; }); - // ---- 4b. WHAT THE MODULE NEEDS -------------------------------------------------------------- - // // `amqp-ping` requires the `amqp` provision, and the mesh refused to place it: "nothing provides // amqp, wanted by amqp-ping". That refusal is correct and is the reason this step exists rather - // than the reason to pick an easier module. **The substrate's broker is not a provider.** It is - // raised by the installer as part of the bundle, so it is a running container and not a module - // with something to offer — the mesh's own plumbing, not an entry in its graph. A module that - // wants a broker wants one the mesh knows about. + // than the reason to pick an easier module. // // `lavinmq` is that module. It was first added here on the belief that it needed no building — - // its broker is an upstream image — and the mesh refused it: two of its three containers named a - // placeholder digest, "which is never a real image". That was right. The broker is upstream, but - // the module is not only the broker: it carries a run-once bootstrap that writes the broker's + // its broker is an upstream image — and the mesh refused it again: two of its three containers + // named a placeholder digest, "which is never a real image". Also right. The broker is upstream, + // but the module is not only the broker: it carries a run-once bootstrap that writes the broker's // configuration, a provisioner that grants each consumer its own vhost and user, tools and an - // event consumer. All of that is its own code and has to be built like anything else. - await step(NEEDS, "the mesh builds a module standing on that base", async () => { - await registerModule(PROVIDER.module, resolve(catalogDir, PROVIDER.module, "module.json")); - const built = await mesh( - `build ${forgeUrl(PROVIDER.repo)} --path ${PROVIDER.path} --ref ${refFor(PROVIDER.repo)} --wait 1200s`, - 1_500_000); - assert.doesNotMatch(built, /failed/i, built); - await mesh(`assign ${CONTROL} ${PROVIDER.module}`); - // **Its account on the mesh's own broker, and not swallowed.** A module's runtime is a tool - // host: it connects to the mesh broker before it does anything, and what it reads is a sealed - // credential document. Without an account the mesh still fills the secret this module declares - // it owns — with a generated value — so the container starts, fails to parse a password as a - // credential, and crash-loops on a JSON syntax error that says nothing about the missing - // account. Issuing it is not optional and neither is hearing that it failed. - await mesh(`module issue ${PROVIDER.module} --node ${CONTROL}`); - await mesh(`push ${CONTROL}`, 600_000); - return waitForContainer(CONTROL, "mesh-lavinmq"); - }); + // event consumer, all of it its own code. + await step(NEEDS, MODULE_BUILT, () => bringUp(PROVIDER)); - // ---- 5. THE ANCHOR RUNS IT ------------------------------------------------------------------ - // // The machine that built it. This is the case every earlier proof covered, and it is here as the - // control for step 6: if this fails, step 6's failure says nothing about fetching. - await step("the anchor runs the module the mesh built", NEEDS, async () => { + // control for the last step: if this fails, that one's failure says nothing about fetching. + await step(ANCHOR_RUNS, NEEDS, async () => { await mesh(`module issue ${MODULE.module} --node ${CONTROL}`); await mesh(`assign ${CONTROL} ${MODULE.module}`); await mesh(`push ${CONTROL}`, 600_000); return waitForContainer(CONTROL, MODULE.module); }); - // ---- 6. AND A MACHINE THAT DID NOT BUILD IT ------------------------------------------------- + // ---- 9. NOW MACHINES MAY ARRIVE --------------------------------------------------------------- // - // **The thing nothing has ever checked.** ace did not build this image and has never seen it. To - // run it, it must fetch it from the mesh's registry — and a joined node has no account there. - // The mesh grants a consumer a credential for a database; it does not yet do so for the store - // its own images live in (novox/hq issue 042). + // Host binary and a token, and nothing else. The anchor is NOT in this loop — the installer + // enrolled it, and enrolling it again would offer the mesh a second identity for a node it + // already knows. // - // Asked anyway, and asked LAST, so that when it fails the five steps above still stand as - // evidence of what does work. - await step(`a joined machine runs the module the mesh built`, - "the anchor runs the module the mesh built", async () => { + // And they are placed on the mesh's private network, which the earlier ordering never did. The + // mesh refuses to bind a consumer to a provider on another machine when either is missing from + // it — "they are not both on the private network" — so without this the last step fails for a + // reason that has nothing to do with what it is asking. + await step(JOINED, ANCHOR_RUNS, async () => { + const said: string[] = []; + await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site hosting`); + for (const machine of HOME_NODES) { + await mesh(`node add ${machine}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const out = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); + assert.match(out, new RegExp(`enrolled as ${machine}`), out); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + await mesh(`overlay place ${machine} --site home`); + said.push(` ${machine} enrolled, running, and on the private network`); + } + said.push((await mesh("node list")).trim()); + said.push((await mesh("overlay show")).trim()); + return said.join("\n"); + }); + + // ---- 10. AND A MACHINE THAT DID NOT BUILD IT -------------------------------------------------- + // + // **The thing nothing has ever checked.** This machine did not build the image and has never seen + // it. To run it, it must fetch it from the mesh's registry — and a joined node has no account + // there (novox/hq issue 042), nor any reason to trust a registry serving plain HTTP over the + // network (issue 048). Both are open, and both are invisible on a mesh of one. + // + // Asked anyway, and asked LAST, so that when it fails everything above still stands as evidence + // of what does work. + await step(SECOND_RUNS, JOINED, async () => { await mesh(`module issue ${MODULE.module} --node ${SECOND}`); await mesh(`assign ${SECOND} ${MODULE.module}`); await mesh(`push ${SECOND}`, 600_000); @@ -427,7 +497,7 @@ after(async () => { }, { timeout: 900_000 }); for (const name of [ - "novox becomes a mesh of one, raised by the installer", + "a bare machine becomes a mesh of one, raised by the installer", "three machines join it across the household gateway", "the mesh builds the shared base from source", "the mesh builds a module standing on that base", diff --git a/test/integration/one-node-mesh.test.ts b/test/integration/one-node-mesh.test.ts new file mode 100644 index 0000000..710961d --- /dev/null +++ b/test/integration/one-node-mesh.test.ts @@ -0,0 +1,649 @@ +/** + * FOUR FRESH MACHINES, AND NOTHING HANDED TO THEM. + * + * The four-machine bed (`whole-mesh-full`) proves the mesh converges. It does so by loading + * thirty-four of the mesh's own images onto its machines from the workstation, because it does not + * build them — something beside the bed built them and copied them in. No real installation looks + * like that, and the lab has been burned by exactly this shape before: it used to raise a registry + * inside the scenario, and a bootstrap that only worked against that registry went green here and + * would have failed on any bare machine. + * + * This bed hands over nothing. The scenario has no `images:` list at all. What the machines get is + * a container runtime and the host binary — prerequisites of a machine, not parts of a mesh — and + * from there: + * + * 1. novox is raised into a mesh of one by the installer, which BUILDS the control plane. + * 2. ace, shanks and g14 JOIN it, across a household NAT, with a token and nothing else. + * 3. The mesh builds the shared base from source, with its own builder. + * 4. The mesh builds a real module standing on that base. + * 5. The anchor runs it, pinned to a digest the mesh's own registry assigned. + * 6. A JOINED machine runs it — which means pulling from a registry that asks who it is. + * + * Steps 1 and 2 are proven elsewhere and are here because the later ones need them. **Steps 3 + * through 6 are what this bed exists for**, and 6 is the one nothing has ever checked: genesis + * puts the builder, the registry and everything they produce on ONE machine, so every earlier + * proof of a mesh-built module running is a proof about the machine that built it. A second + * machine has to fetch, and fetching needs an account nothing yet grants (novox/hq issue 042). + * + * Each step is recorded separately rather than allowed to throw, so a gap at 6 reports as a gap at + * 6 instead of erasing the evidence for 3, 4 and 5. + * + * MESH_LAB_INCUS='sudo -n incus' + * MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host + * MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap + * MESH_LAB_BUNDLE=.../mesh-host/examples/substrate-first-node.lock + * MESH_LAB_CATALOG=.../mesh-catalog/modules + * MESH_LAB_SOURCE=/mesh-control.git MESH_LAB_SOURCE_REF= + * MESH_LAB_KEEP=1 to leave it standing afterwards + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec, push } from "../../src/lifecycle/operate.ts"; +import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts"; +import { genesis, type GenesisResult } from "./genesis.ts"; +import { incus } from "../../src/incus/client.ts"; +import { instanceNameOf } from "../../src/lifecycle/operate.ts"; +import { waitUntilAllUsable } from "../../src/lifecycle/ready.ts"; + +const SCENARIO = "one-node-mesh"; +const CONTROL = "anchor"; + +/** The anchor's public address — what every other machine dials, and what its own token must name. */ +const ANCHOR = "192.0.2.10"; +/** Where this mesh's registry answers, on the anchor's public address so a joined node can reach it. */ +const REGISTRY = `${ANCHOR}:5000`; + +/** + * The module built on top of the base, and the base it stands on. + * + * `amqp-ping` is deliberately small and deliberately REAL: its own TypeScript, compiled by the + * shared toolchain, running on the shared runtime, talking to the broker. A module whose artifact + * is a mirrored public image would pass every assertion below while skipping the whole of what is + * under test (novox/hq SELF-UPGRADE-PLAN, rule 1). + */ +const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" }; +/** + * What `amqp-ping` requires, and what the substrate does not supply. + * + * The installer raises a broker, but as a bundle resource — plumbing, not a module the mesh has a + * record of, so it provides nothing to anything. A module asking for `amqp` is asking for a + * provider in the graph, and this is it. No build: its image is upstream. + */ +const PROVIDER = { module: "lavinmq", repo: "mesh-catalog", path: "modules/lavinmq", container: "mesh-lavinmq" }; +/** + * The store, and the catalogue that cannot exist without it. + * + * Both were missing from this test entirely, and nothing complained, because nothing asked. A mesh + * with no catalogue holds no module graph — it cannot say what it has, what a module is made of, + * what a change reaches, or what must be rebuilt. It ran anyway, which is the point: "the mesh is + * up" was being read off genesis finishing. + */ +const STORE = { module: "postgres", repo: "mesh-catalog", path: "modules/postgres", container: "mesh-postgres" }; +const CATALOGUE = { module: "mesh-catalog", repo: "mesh-catalog", path: "modules/mesh-catalog", container: "mesh-catalog" }; +/** The control plane, rebuilt from its own repository — the step that ends the installer's tenure. */ +const CONTROL_PLANE = { module: "mesh-control", repo: "mesh-control", path: "", container: "mesh-control" }; + +/** + * What this mesh must hold when it is finished, and what must be RUNNING on the machine. + * + * Written down as a list rather than checked one step at a time, because "is this a mesh" is a + * question about the set. The three the build loop cannot produce for itself — the control plane, + * the registry and the builder — are here too: they are carried in, and a mesh missing any of them + * is not one. + */ +const MUST_HOLD = ["mesh-control", "registry", "builder", "mesh-tools", "postgres", + "mesh-catalog", "lavinmq", "amqp-ping"]; +const MUST_RUN = ["mesh-control", "mesh-registry", "mesh-broker", "mesh-store", + "mesh-postgres", "mesh-catalog", "mesh-lavinmq", "amqp-ping"]; +/** Named once, because the step title is also how later steps say what they waited on. */ +const NEEDS = "the mesh runs a broker for that module to talk to"; +const GENESIS = "a bare machine becomes a mesh of one, raised by the installer"; +const BASE_BUILT = "the mesh builds the shared base from source"; +const STORE_RUNS = "the mesh builds and runs a store of its own"; +const CATALOGUE_RUNS = "the mesh builds and runs its own catalogue"; +const CONTROL_REBUILT = "the mesh rebuilds its own control plane from source"; +const MODULE_BUILT = "the mesh builds a module standing on that base"; +const ANCHOR_RUNS = "the anchor runs the module the mesh built"; +const DESCRIBES = "the mesh can describe itself, and what it says is true"; +const NETWORK = "the machine's networking is what the modules asked for"; +const FOLLOWS = "a change to a module's source reaches the machine on its own"; +const SURVIVES = "the mesh comes back after the machine reboots"; +const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping" }; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; +const source = process.env["MESH_LAB_SOURCE"] ?? ""; +const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? ""; +const KEEP = !!process.env["MESH_LAB_KEEP"]; +const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "fresh-mesh-live" : undefined); + +/** + * Where a repository other than the control plane's lives. + * + * Derived from `MESH_LAB_SOURCE` by swapping the last path segment, because every one of these + * repositories sits beside the others under the same owner on the same forge. Overridable, so a + * forge that is arranged differently does not need this bed edited. + */ +function forgeUrl(repo: string): string { + const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`]; + if (override) return override; + return source.replace(/[^/]+\.git$/, `${repo}.git`); +} +/** + * What to build, per repository. + * + * A branch is acceptable for an ordinary build; only genesis insists on a commit (ADR 0071). Per + * repository rather than one value for all of them, because a change under test usually lives in + * one repository and the rest should be built from what everyone else has — building them all from + * a feature branch would prove that branch against itself. + * + * MESH_LAB_BUILD_REF the default for every repository + * MESH_LAB_BUILD_REF_MESH_CATALOG ...overridden for one + */ +function refFor(repo: string): string { + const override = process.env[`MESH_LAB_BUILD_REF_${repo.toUpperCase().replaceAll("-", "_")}`]; + return override ?? process.env["MESH_LAB_BUILD_REF"] ?? "main"; +} + +/** + * The shared base's manifest, on this workstation. + * + * The base is a repository with a manifest at its root (novox/hq ADR 0069), so unlike the + * catalogue's modules it is not under `MESH_LAB_CATALOG`. Derived from that path on the convention + * that the checkouts sit beside each other, and overridable for a layout where they do not. + */ +const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ?? + resolve(catalogDir, "..", "..", BASE.repo, "module.json"); + +const skip = + !capability.usable ? capability.why : + !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : + !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" : + !source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" : + !sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" : + !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate template" : + !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" : + false; + +let instanceId = ""; +let raised: GenesisResult; + +// ---- talking to the machines ------------------------------------------------------------------ + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} +async function mesh(command: string, timeoutMs?: number): Promise { + return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs); +} + +/** + * Stop the machine and start it again, the way a power cut or a kernel upgrade would. + * + * There is no restart in the lab's own vocabulary, which is its own small finding: nothing had ever + * needed one, because nothing had ever asked whether a mesh comes back. + */ +async function restartMachine(machine: string): Promise { + const name = await instanceNameOf(instanceId, machine); + await incus(["restart", name], 180_000); + await waitUntilAllUsable([name], 300, (m) => console.log(` restart: ${m}`)); +} + +/** A module the mesh has to make for itself: where its source is, and what it runs when it works. */ +interface CoreModule { module: string; repo: string; path: string; container: string } + +/** + * Build a module from source, install it, and wait for it to actually run. + * + * The whole sequence a module goes through, in one place because the mesh has to do it for several + * before it is a mesh at all: register the manifest, build it from its repository and path, issue + * the broker account its runtime needs, assign it, send it, and then ask the machine whether the + * container is up. + * + * **The account is not optional and is not swallowed.** A module's runtime is a tool host: it + * connects to the mesh's broker before doing anything. Without an account the mesh still fills the + * secret the module declares it owns — with a generated value — so the container starts, fails to + * parse a password as a credential document, and loops on a JSON syntax error mentioning no + * missing account. That cost a step that reported PASS. + */ +async function bringUp(m: CoreModule): Promise { + await registerModule(m.module, resolve(catalogDir, m.module, "module.json")); + const built = await mesh( + `build ${forgeUrl(m.repo)} --path ${m.path} --ref ${refFor(m.repo)} --wait 1200s`, 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + await mesh(`module issue ${m.module} --node ${CONTROL}`); + await mesh(`assign ${CONTROL} ${m.module}`); + await mesh(`push ${CONTROL}`, 600_000); + return `${built}\n${await waitForContainer(CONTROL, m.container)}`; +} + +/** + * Wait for one container, BY NAME, to be running. + * + * **Named exactly, because substring matching passed a step that had failed.** Waiting for "a + * container whose name contains lavinmq" was satisfied by the broker — `lavinmq`, up and healthy — + * while the thing actually under test, the module's own runtime `mesh-lavinmq`, was crash-looping + * beside it. The step went green and the fault was found by reading `docker ps` by hand. + */ +async function waitForContainer(node: string, container: string, seconds = 200): Promise { + const deadline = Date.now() + seconds * 1_000; + let last = ""; + while (Date.now() < deadline) { + const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + last = ps; + const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container); + if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps; + await new Promise((r) => setTimeout(r, 5_000)); + } + const logs = (await on(node, `docker logs --tail 15 ${container} 2>&1`)).out; + throw new Error( + `${container} is not running on ${node}.\n\ncontainers:\n${last}\n\nwhat it said:\n${logs}`); +} + +/** + * Register a module from a manifest on this workstation. + * + * **The control plane runs in a container, so a file on the machine is not a file it can open.** + * Pushing the manifest to the machine and naming that path got `no such file or directory` from + * inside mesh-control, which is correct and was briefly mistaken for a missing manifest. It is + * copied the last step of the way with `docker cp`. + * + * **Into the root, not into /tmp.** The control plane's image is a minimal one and has no `/tmp` + * to copy into — `docker cp` says so in those words. `/` is the one directory every image has. + */ +async function registerModule(module: string, manifest: string): Promise { + assert.ok(existsSync(manifest), `no manifest for ${module} at ${manifest}`); + const onMachine = `/tmp/${module}.json`; + const inContainer = `/${module}.json`; + await push(instanceId, CONTROL, manifest, onMachine); + await must(CONTROL, `docker cp ${onMachine} mesh-control:${inContainer}`); + return mesh(`module add ${inContainer}`); +} +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +// ---- steps, recorded rather than thrown -------------------------------------------------------- + +interface Step { ok: boolean; why: string; said: string } +const steps = new Map(); +const order: string[] = []; + +/** Run a step, remember what it said, and never throw. A step whose predecessor failed is skipped. */ +async function step(name: string, after_: string | null, fn: () => Promise): Promise { + order.push(name); + if (after_ && !steps.get(after_)?.ok) { + steps.set(name, { ok: false, why: `not attempted — "${after_}" did not succeed`, said: "" }); + console.log(`SKIPPED ${name}`); + return; + } + console.log(`\n======== ${name} ========`); + try { + const said = await fn(); + steps.set(name, { ok: true, why: "", said }); + console.log(`OK ${name}`); + } catch (err) { + const why = (err as Error).message; + steps.set(name, { ok: false, why, said: "" }); + console.log(`FAILED ${name}\n${why.split("\n").slice(0, 25).join("\n")}`); + } +} + +function report(name: string): string { + const s = steps.get(name); + if (!s) return `${name}: never ran`; + const lines = order.map((n) => { + const it = steps.get(n); + return ` ${it?.ok ? "PASS" : "FAIL"} ${n}`; + }); + return `${s.why}\n\nWhere this bed got to:\n${lines.join("\n")}`; +} + +before(async () => { + if (skip) return; + + const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), + }); + instanceId = bed.instanceId; + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); + console.log(`NOTHING WAS LOADED: this scenario names no images. Every image on every machine ` + + `below was pulled from the internet or built by the mesh.`); + + // ---- 1. GENESIS ----------------------------------------------------------------------------- + // + // The shared description, the same one `genesis-single` calls. The bundle is the TEMPLATE with + // nothing held: no image is pre-resolved, because none is here to resolve to. + await step("a bare machine becomes a mesh of one, raised by the installer", null, async () => { + try { + raised = await genesis({ + instanceId, + node: CONTROL, + installer: installer as string, + catalogDir, + // The broker's advertised address, corrected. + // + // The template hardcodes 192.0.2.10:5671 — the address of the anchor in the single-machine + // bed. A token carries this verbatim as the endpoint an enrolling node dials, so on a mesh + // whose anchor is somewhere else every node, including this one, would enrol against an + // address nothing answers on. The installer refuses to guess it and says so, which is + // right: it does not know what this machine is called from outside. + bundleTemplate: substrateBundle(bundle, []).replaceAll("192.0.2.10:5671", `${ANCHOR}:5671`), + registry: REGISTRY, + source, + sourceRef, + log: (m) => console.log(m), + }); + } catch (err) { + throw new Error(`the installer never ran: ${(err as Error).message}`); + } + if (!raised.ok) throw new Error(`${raised.step || "no step named"}: ${raised.why}\n\n${raised.report.join("\n")}`); + return raised.report.join("\n"); + }); + + // ---- 2..6. THE MESH BECOMES ONE -------------------------------------------------------------- + // + // **Joining used to be here, second, and that was the wrong order.** Three machines were enrolled + // into a mesh that could not yet produce a single module, and the step was reported as though + // something had been shown. They do join — reliably — but joining a mesh that can build nothing + // proves only that enrolment works, which was never the doubtful part. + // + // `17-raising-a-mesh` says it plainly: genesis ends with a mesh of one that RUNS, and that is not + // the same as a mesh that WORKS; what remains after the core modules are built is "adding + // machines, and deciding what they run". So everything a mesh needs to be a mesh happens first, + // and machines arrive at the end. + + // The toolchain and runtime every module with code of its own stands on. It is a module, and it + // is built like one — cloned from the forge by the builder installing put here, compiled on the + // machine, published into the mesh's own registry. + await step(BASE_BUILT, GENESIS, async () => { + // Registered from the manifest the builder will also read, so what the mesh holds and what it + // builds are the same description of the same module. + // + // **Not swallowed.** This call used to end in `.catch(() => {})`, on the reasoning that the + // base might already be known. It hid a real failure — the manifest was being named at a path + // inside a container that had never seen it — and the step passed anyway, because a base with + // nothing to stand on builds whether or not the mesh has a record of it. + await registerModule(BASE.module, baseManifest); + const built = await mesh( + `build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + return built; + }); + + // A store of its own. **Not the substrate's.** The installer raises a store for the control + // plane to keep its own records in, the way it raises a broker — plumbing, not a module the mesh + // has any record of, so it provides nothing to anything. A module that wants a database wants a + // provider in the graph, and the catalogue below is the first thing to want one. + await step(STORE_RUNS, BASE_BUILT, () => bringUp(STORE)); + + // And the catalogue, which was missing from this test altogether. + // + // Without it the mesh holds no module graph: it cannot say what it has, what a module is made + // of, what a change to one reaches, or what must be rebuilt. A mesh in that state still runs, + // which is exactly how its absence went unnoticed — "the mesh is up" was being read off the + // installer finishing rather than off the mesh being able to answer anything. + await step(CATALOGUE_RUNS, STORE_RUNS, () => bringUp(CATALOGUE)); + + // The control plane, rebuilt from its own repository and rolled out. + // + // The installer built it once, which is what got the mesh running. Building it again THROUGH THE + // MODULE PATH — build, notice the version moved, roll it out — is a different claim: it is the + // moment the mesh stops depending on the installer for anything, and the first time the thing + // that performs an upgrade performs one on itself. + await step(CONTROL_REBUILT, CATALOGUE_RUNS, async () => { + const built = await mesh( + `build ${forgeUrl(CONTROL_PLANE.repo)} --ref ${sourceRef} --wait 1200s`, 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + const rolled = await mesh(`upgrade ${CONTROL_PLANE.module} roll-out`, 900_000); + // Asked of the machine rather than believed from the command: the control plane that answers + // afterwards is the one that has to be running for anything below this line to mean anything. + await waitForContainer(CONTROL, CONTROL_PLANE.container); + return `${built}\n${rolled}`; + }); + + // ---- 7..8. SOMETHING TO RUN --------------------------------------------------------------- + await step(MODULE_BUILT, CONTROL_REBUILT, async () => { + await registerModule(MODULE.module, resolve(catalogDir, MODULE.module, "module.json")); + const built = await mesh( + `build ${forgeUrl(MODULE.repo)} --path ${MODULE.path} --ref ${refFor(MODULE.repo)} --wait 1200s`, + 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + // The point of the whole step: what came out is named by a digest this mesh's registry + // assigned, not by a placeholder and not by a tag. + const builds = await mesh(`builds ${MODULE.module}`); + assert.match(builds, /sha256:[0-9a-f]{12}/, + `the build recorded no digest — the module is not pinned to anything this registry serves:\n${builds}`); + return `${built}\n${builds}`; + }); + + // `amqp-ping` requires the `amqp` provision, and the mesh refused to place it: "nothing provides + // amqp, wanted by amqp-ping". That refusal is correct and is the reason this step exists rather + // than the reason to pick an easier module. + // + // `lavinmq` is that module. It was first added here on the belief that it needed no building — + // its broker is an upstream image — and the mesh refused it again: two of its three containers + // named a placeholder digest, "which is never a real image". Also right. The broker is upstream, + // but the module is not only the broker: it carries a run-once bootstrap that writes the broker's + // configuration, a provisioner that grants each consumer its own vhost and user, tools and an + // event consumer, all of it its own code. + await step(NEEDS, MODULE_BUILT, () => bringUp(PROVIDER)); + + // The machine that built it. This is the case every earlier proof covered, and it is here as the + // control for the last step: if this fails, that one's failure says nothing about fetching. + await step(ANCHOR_RUNS, NEEDS, async () => { + await mesh(`module issue ${MODULE.module} --node ${CONTROL}`); + await mesh(`assign ${CONTROL} ${MODULE.module}`); + await mesh(`push ${CONTROL}`, 600_000); + return waitForContainer(CONTROL, MODULE.module); + }); + + // ---- 9. IT CAN DESCRIBE ITSELF --------------------------------------------------------------- + // + // **Presence is not function, and this step exists because I kept confusing them.** A container + // being up was taken as the catalogue working; a name containing "lavinmq" was taken as the + // module running. The mesh holds a graph — nodes, what each is assigned, what capabilities each + // has, which claims are occupied, what each module is configured with, which provisions exist and + // who holds them — and none of it was ever asked a question. + await step(DESCRIBES, ANCHOR_RUNS, async () => { + const said: string[] = []; + + // The mesh's own verdict on itself. Nothing wrong, nothing waiting, nothing behind. + const state = JSON.parse(await mesh("status --json")) as { + wrong: { node: string; outcome: string; refused?: string }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + assert.equal(state.wrong.length, 0, + `the mesh reports something wrong:\n${JSON.stringify(state.wrong, null, 2)}`); + assert.equal(state.waiting.length, 0, + `the mesh is waiting on a node:\n${JSON.stringify(state.waiting, null, 2)}`); + const node = state.reported.find((r) => r.node === CONTROL); + assert.ok(node, `the mesh does not report the only machine it has:\n${JSON.stringify(state)}`); + assert.equal(node.outcome, "applied", `${CONTROL} did not apply what it was sent: ${node.outcome}`); + assert.ok(node.current, `${CONTROL} is not running what the mesh would send it`); + said.push(` status one node, applied, current, nothing wrong`); + + // Every module a mesh has to hold, including the three it cannot build for itself. + const modules = await mesh("module list"); + for (const m of MUST_HOLD) { + assert.match(modules, new RegExp(`^${m}\\b`, "m"), + `the mesh holds no ${m}. A mesh without it is not finished:\n${modules}`); + } + said.push(` module list ${MUST_HOLD.length} modules, all present`); + + // What the machine would be sent, and what it names. **No placeholder may survive here** — a + // digest of all zeroes is never a real image, and a declaration carrying one reaches a machine + // that will try to fetch it. + const plan = await mesh(`plan ${CONTROL} --json`); + assert.doesNotMatch(plan, /sha256:0{64}/, + `the machine's own plan names a placeholder digest, which is never a real image`); + assert.match(plan, /sha256:[0-9a-f]{64}/, `the plan pins nothing by digest at all`); + said.push(` plan every image pinned, no placeholders`); + + // And the catalogue, ASKED rather than observed. These five questions are what it exists for. + const ask = async (tool: string, args = "{}") => + must(CONTROL, `docker exec mesh-catalog mesh-tools invoke mesh-catalog ${tool} ${quote(args)}`, + 120_000); + + const held = await ask("catalog_modules"); + for (const m of MUST_HOLD) { + if (m === "registry" || m === "builder" || m === "mesh-control") continue; // carried, not built here + assert.ok(held.includes(m), `the catalogue does not know about ${m}:\n${held}`); + } + assert.doesNotMatch(held, /sha256:0{64}/, `the catalogue holds a placeholder version`); + said.push(` catalog_modules every built module, each with a version this mesh made`); + + const provides = await ask("catalog_provides", JSON.stringify({ provision: "amqp" })); + assert.ok(provides.includes(PROVIDER.module), + `the catalogue cannot say what provides amqp, which is the question it exists to answer:\n${provides}`); + said.push(` catalog_provides amqp is answered by ${PROVIDER.module}`); + + const stale = await ask("catalog_stale"); + said.push(` catalog_stale ${stale.trim().slice(0, 120)}`); + return said.join("\n"); + }); + + // ---- 10. AND ITS NETWORKING IS WHAT WAS ASKED FOR --------------------------------------------- + // + // **Left out of this test entirely until it was pointed out**, which is hard to defend: the + // firewall is generated from what modules declare they listen on, and a firewall that opens the + // wrong set is either a service nobody can reach or a port nobody meant to publish. Neither shows + // up as a failed container. + await step(NETWORK, DESCRIBES, async () => { + const said: string[] = []; + + const ruleset = (await on(CONTROL, `nft list table inet mesh 2>&1`)).out; + assert.match(ruleset, /chain input/, `the mesh's own firewall table is not there:\n${ruleset}`); + // Closed by default, or the rules below decide nothing. + assert.match(ruleset, /policy drop/, `the firewall does not default to closed:\n${ruleset}`); + // The floor: a machine that cannot be reached over ssh is a machine nobody can repair. + assert.match(ruleset, /\b22\b/, `ssh is not allowed anywhere in the ruleset`); + // What a module actually declared. The registry says it listens on 5000 for the mesh. + assert.match(ruleset, /\b5000\b/, + `the registry declares it listens on 5000 and nothing opened it:\n${ruleset}`); + said.push(` firewall default closed, ssh open, declared ports open`); + + // The names the mesh writes for itself. A consumer reaching a provider by its `.internal` + // address depends on this file, and on it reaching inside containers. + const hosts = (await on(CONTROL, `cat /etc/hosts`)).out; + assert.match(hosts, /\.internal/, `the mesh wrote no .internal names:\n${hosts}`); + said.push(` names ${(hosts.match(/[a-z0-9-]+\.internal/g) ?? []).join(" ")}`); + + // The networks the declarations asked for, rather than whatever the runtime had lying around. + const networks = (await on(CONTROL, `docker network ls --format '{{.Name}}'`)).out; + for (const wanted of ["lavinmq", "amqp-ping"]) { + assert.match(networks, new RegExp(`^${wanted}$`, "m"), + `the ${wanted} module declares a network and none exists:\n${networks}`); + } + said.push(` networks module networks present`); + return said.join("\n"); + }); + + // ---- 11. A CHANGE REACHES THE MACHINE ON ITS OWN ---------------------------------------------- + // + // The whole point of the mesh, and the capability the migration depends on: move a module's + // source and the running copy follows, with nobody driving the steps. Everything above is + // machinery; this is what the machinery is for. + await step(FOLLOWS, NETWORK, async () => { + const before = await mesh(`builds ${MODULE.module}`); + const wasPinned = before.match(/sha256:[0-9a-f]{64}/)?.[0] ?? ""; + assert.ok(wasPinned, `nothing is pinned to rebuild from:\n${before}`); + + // The mesh is told its copy is older than the source. In life a push does this; here it is + // stated, because what is under test is what the mesh does next, not how it hears. + const head = (await must(CONTROL, `git ls-remote ${forgeUrl(MODULE.repo)} ` + + `${refFor(MODULE.repo)} | cut -f1`, 120_000)).trim(); + assert.match(head, /^[0-9a-f]{40}$/, `could not read the source's head: ${head}`); + await mesh(`module moved ${MODULE.module} ${head}`); + + const behind = await mesh(`status`); + assert.match(behind, /behind|build --behind/, + `the mesh does not report a module behind its source:\n${behind}`); + + await mesh(`build --behind --wait 1200s`, 1_500_000); + const rolled = await mesh(`upgrade ${MODULE.module} roll-out`, 900_000); + await waitForContainer(CONTROL, MODULE.module); + + const after = await mesh(`builds ${MODULE.module}`); + assert.match(after, /sha256:[0-9a-f]{64}/, `nothing was pinned after the rebuild:\n${after}`); + return `${behind}\n${rolled}\n${after}`; + }); + + // ---- 12. AND IT SURVIVES THE MACHINE STOPPING ------------------------------------------------- + // + // **Never once tested.** A mesh that works until the machine reboots is a demonstration, not + // something to move real services onto — and the installer is explicit that a host started the + // way the lab starts it does not survive a reboot, which makes this the check that says whether + // that matters. + await step(SURVIVES, FOLLOWS, async () => { + await restartMachine(CONTROL); + const missing: string[] = []; + for (const container of MUST_RUN) { + try { + await waitForContainer(CONTROL, container, 240); + } catch { + missing.push(container); + } + } + const ps = (await on(CONTROL, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + assert.equal(missing.length, 0, + `after a reboot these are not running: ${missing.join(", ")}\n\ncontainers:\n${ps}`); + return ps; + }); + + console.log(`\n================ WHAT THIS MESH DID FOR ITSELF ================`); + for (const n of order) console.log(` ${steps.get(n)?.ok ? "PASS" : "FAIL"} ${n}`); +}, { timeout: 7_200_000 }); + +after(async () => { + if (KEEP) { + console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`); + return; + } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +for (const name of [ + GENESIS, + BASE_BUILT, + STORE_RUNS, + CATALOGUE_RUNS, + CONTROL_REBUILT, + MODULE_BUILT, + NEEDS, + ANCHOR_RUNS, + DESCRIBES, + NETWORK, + FOLLOWS, + SURVIVES, +]) { + test(name, { skip, timeout: 60_000 }, () => { + assert.ok(steps.get(name)?.ok, report(name)); + }); +}