diff --git a/scenarios/fresh-mesh.yml b/scenarios/fresh-mesh.yml new file mode 100644 index 0000000..4a0370d --- /dev/null +++ b/scenarios/fresh-mesh.yml @@ -0,0 +1,106 @@ +# FOUR FRESH MACHINES AND THE INSTALLER. Nothing is handed to them. +# +# This is `whole-mesh-full`'s topology with `genesis-single`'s honesty. The four-machine bed loads +# thirty-four of the mesh's own images onto its machines from the workstation, because it does not +# build them — they are produced by hand beside the bed and copied in. That is a shape no real +# installation has, and it is the same class of fiction the lab already removed once: it used to +# raise a registry inside the scenario, and a bootstrap that only worked against it went green here +# and would have failed on any real machine. +# +# So this bed hands over NOTHING. There is no `images:` list. Every machine gets a container +# runtime and the host binary, which are prerequisites of the machine rather than parts of the +# mesh, and after that the mesh is on its own: +# +# - the substrate, the registry and the builder's own dependencies are PULLED from the internet, +# which is where a bare machine gets them; +# - the control plane is BUILT, by the builder the installer carries, from a repository and a +# commit it is told to use; +# - every module after that is BUILT by the mesh's own builder and published into the mesh's own +# registry, and a machine that runs one PULLS it from there. +# +# That last clause is the thing no bed has ever checked, and it is why this one has four machines +# rather than one. Genesis puts the builder, the registry and everything they make on ONE machine. +# A second machine running a mesh-built module has to fetch it from a registry that asks who it is, +# and nothing yet gives a joined node an account for it. The bed asks anyway, in its own test, so +# the gap is a named failure rather than an absence. +# +# hosting (public, routable) home (private, behind the access point) +# novox 192.0.2.20 ── anchor ace 10.99.1.10 home server +# substrate, registry, shanks 10.99.1.20 workstation +# builder, control plane g14 10.99.1.30 workstation +# +# EGRESS IS NOT OPTIONAL HERE. With nothing loaded, a sealed machine stops at the installer's first +# pull. Every machine has a way out, and it is a SECOND path: each still reaches the rest of the +# scenario through its declared gateway, and the uplink carries only what leaves the scenario — +# the public images, and the forge the control plane is cloned from. +# +# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap +# MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock +# MESH_LAB_CATALOG=.../mesh-catalog/modules +# MESH_LAB_SOURCE= MESH_LAB_SOURCE_REF= +scenario: fresh-mesh + +segments: + # The routable segment. novox lives here; its public address is the broker endpoint every token + # carries and the overlay hub the home nodes dial. + hosting: + kind: public + cidr: [192.0.2.0/24] + + # The household segment behind an ordinary home router: masquerades v4 outbound, forwards + # inbound, expires idle mappings after two minutes. + home: + kind: private + cidr: [10.99.1.0/24] + gateway: + to: hosting + address: [192.0.2.50] + nat: [v4] + forwardable: true + mapping_ttl: 120s + +machines: + # The anchor. Raised by the installer into a mesh of one, and then asked to build. + # + # Sized for what it actually does here: the store, the broker, the registry, TWO control planes + # during the pivot, the builder, and a build workspace holding a Node toolchain image and an npm + # cache. It is NOT sized for the whole novox service set, because this bed does not run one — it + # proves the machinery that would produce it. + novox: + at: { segment: hosting, address: [192.0.2.20] } + egress: true + inbound: allow + memory: 12GiB + cpus: 6 + disk: 60GiB + + # Three machines that JOIN. Host binary and a token, nothing else — no bootstrap, no substrate, + # no registry. They are deliberately small: what they are here to prove is that a joined machine + # can be given a module the mesh built, which is a question about credentials and not about load. + ace: + at: { segment: home, address: [10.99.1.10] } + egress: true + inbound: allow + memory: 4GiB + cpus: 2 + disk: 25GiB + shanks: + at: { segment: home, address: [10.99.1.20] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + disk: 20GiB + g14: + at: { segment: home, address: [10.99.1.30] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + disk: 20GiB + +# **No `images:` key, and that is the whole point of this file.** Anything a machine holds here, it +# pulled or the mesh built. See the header. + +place: + all: [host, runtime] diff --git a/src/lifecycle/egress.ts b/src/lifecycle/egress.ts index e2b4995..ebda7fa 100644 --- a/src/lifecycle/egress.ts +++ b/src/lifecycle/egress.ts @@ -89,9 +89,40 @@ export async function confirmEgress( // the retry is tightened so a silent server costs seconds rather than the step. A broken uplink // still fails the check above, before any of this. await resilientResolver(name); + + // **And then prove it is STEADY, because one success proved nothing.** + // + // The check above is satisfied by a single answer, and that is how a machine resolving one + // query in three passed it and then killed two installs twenty minutes later. What a run needs + // is not "a name resolved once" but "names resolve reliably", and those differ by exactly the + // failure that has cost the most time here. Consecutive, because alternating success and + // timeout is the observed shape — a total count would pass on the same machine. + const steady = await steadilyResolves(name, 5); + if (steady < 5) { + throw new EgressError( + `${machine} resolves ${UPSTREAM} only ${steady} time(s) in five consecutive tries.\n` + + ` It has egress and an unreliable resolver, which does not fail here — it fails later, ` + + `inside a pull or a clone, as "could not resolve host" with the cause long out of view.\n` + + ` The uplink's own resolver is the usual culprit and is deliberately last in the list; ` + + `a machine still failing this has something wrong upstream of the lab.`, + ); + } + log(` ${machine} resolves steadily (5/5)`); } } +/** How many of `tries` consecutive lookups answered. Stops at the first failure. */ +async function steadilyResolves(name: string, tries: number): Promise { + for (let i = 0; i < tries; i++) { + const said = await incusOk( + ["exec", name, "--", "sh", "-c", + `timeout 8 getent hosts ${UPSTREAM} >/dev/null && echo yes`], 20_000, + ); + if (said?.trim() !== "yes") return i; + } + return tries; +} + async function resolves(name: string, waitSeconds: number): Promise { const deadline = Date.now() + waitSeconds * 1_000; while (Date.now() < deadline) { @@ -140,11 +171,22 @@ async function reaches(name: string, waitSeconds: number): Promise { await incus([ @@ -152,7 +194,8 @@ async function resilientResolver(name: string): Promise { `link=$(ip -4 route show default | awk '{print $5}' | head -n1); ` + `via=$(ip -4 route show default | awk '{print $3}' | head -n1); ` + `if [ -n "$link" ] && command -v resolvectl >/dev/null 2>&1; then ` + - `resolvectl dns "$link" $via 1.1.1.1 8.8.8.8 >/dev/null 2>&1 || true; fi; true`, + `resolvectl dns "$link" 1.1.1.1 8.8.8.8 9.9.9.9 $via >/dev/null 2>&1 || true; ` + + `resolvectl flush-caches >/dev/null 2>&1 || true; fi; true`, ], 30_000); } diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index 0c1aa9c..96e402e 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -307,9 +307,18 @@ export async function raise( enter("waiting for machines to become usable"); await waitUntilAllUsable(created, readyTimeout, log); + // **Before the uplink lease is asked for, make sure each machine asks as itself.** + enter("giving each machine its own identity"); + await distinguishMachines(created, log); + enter("applying declared addresses"); await applyAddresses(scenario, instanceId, byMachine, log); + // The positive control for the step above: if two machines share an uplink address, say so + // HERE, where it is one obvious sentence, rather than letting it surface an hour later as + // intermittent name resolution on some machines and not others. + await noTwoMachinesShareAnAddress(scenario, byMachine, log); + // Transit first: a gateway's default route points at it, so it has to exist. enter("wiring the public networks together"); const transit = await raiseTransit(scenario, instanceId, log); @@ -356,3 +365,85 @@ export async function raise( throw new RaiseError(instanceId, step, cause); } } + +/** + * Give every machine a machine-id of its own, before any of them asks for an uplink lease. + * + * **Every machine in a bed is a clone of one base image, so they all boot with the SAME + * `/etc/machine-id`.** systemd's DHCP client derives its client identifier from that file, and the + * uplink's dnsmasq keys leases on the client identifier rather than on the MAC — so four machines + * with four distinct MACs were all handed *the same address*, and the host kept one ARP entry for + * it. Whichever machine last answered an ARP request got everybody's replies. + * + * This is the fault behind every "the lab's DNS is flaky" run. It does not present as an address + * conflict; it presents as name resolution that works two times in three, as pulls that succeed on + * a retry, and as one machine out of four being fine — because that machine happened to be holding + * the address. It survived an earlier diagnosis that blamed resolver ordering, because reordering + * resolvers on a machine that has just won the ARP race does appear to fix it. + * + * **Ordering is the whole of it, and the first version got it wrong in the other direction.** That + * version also restarted networkd and waited for a new lease, which is what you would do to repair + * a machine already holding a shared address. Here there is nothing to repair yet: the uplink is + * still down at this point and `applyAddresses` is what asks for the lease. So this writes the + * identity and stops, and the request that follows is made as somebody. + * + * Machines without `systemd-machine-id-setup` are left alone; the step is advisory. + */ +async function distinguishMachines(names: string[], log: (m: string) => void): Promise { + for (const name of names) { + const said = await incusOk([ + "exec", name, "--", "sh", "-c", + // Regenerated rather than written: `systemd-machine-id-setup` owns the format, and a + // hand-made value that is not 32 hex characters is rejected by systemd at next boot. + `command -v systemd-machine-id-setup >/dev/null 2>&1 || { echo unchanged; exit 0; }; ` + + `rm -f /etc/machine-id && systemd-machine-id-setup >/dev/null 2>&1; ` + + `cat /etc/machine-id`, + ], 60_000); + log(` ${name} is ${said?.trim().slice(0, 12) || "unchanged"}`); + } +} + +/** + * Refuse to go on if two machines took the same uplink address. + * + * A check rather than a comment, because the failure it guards is invisible where it happens and + * unrecognisable where it surfaces. Every machine that declares egress is asked what address it + * holds; two the same is a stop, named as what it is. + */ +async function noTwoMachinesShareAnAddress( + scenario: Scenario, + byMachine: Map, + log: (m: string) => void, +): Promise { + const held = new Map(); + for (const [machine, spec] of Object.entries(scenario.machines)) { + if (!spec.egress || spec.at === "detached") continue; + const name = byMachine.get(machine); + if (!name) continue; + const said = (await incusOk([ + "exec", name, "--", "sh", "-c", + // The `src` of the default route, NOT its last field — the first version of this took + // `$NF` and compared four machines' route METRIC, which is identical by construction and + // made the guard fire on every bed. A check that cannot be wrong is worth less than one + // that is read carefully once. + `ip -4 -o route show default 2>/dev/null | ` + + `awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}' | head -n1`, + ], 30_000))?.trim(); + if (!said) continue; + held.set(said, [...(held.get(said) ?? []), machine]); + } + const shared = [...held.entries()].filter(([, who]) => who.length > 1); + if (shared.length === 0) { + if (held.size > 0) log(` every machine with egress took an address of its own`); + return; + } + throw new Error( + `two machines took the SAME uplink address: ` + + shared.map(([a, who]) => `${a} held by ${who.join(" and ")}`).join("; ") + `.\n` + + ` They are clones of one image, so they present one DHCP client identity unless each is ` + + `given its own machine-id before the lease is asked for.\n` + + ` This does not fail as an address conflict. It fails later, as name resolution that works ` + + `about two times in three and as pulls that succeed on a retry, because the host holds one ` + + `ARP entry and whichever machine answered last receives the replies.`, + ); +} diff --git a/test/integration/fresh-mesh.test.ts b/test/integration/fresh-mesh.test.ts new file mode 100644 index 0000000..2c7198b --- /dev/null +++ b/test/integration/fresh-mesh.test.ts @@ -0,0 +1,355 @@ +/** + * FOUR FRESH MACHINES, AND NOTHING HANDED TO THEM. + * + * The four-machine bed (`whole-mesh-full`) proves the mesh converges. It does so by loading + * thirty-four of the mesh's own images onto its machines from the workstation, because it does not + * build them — something beside the bed built them and copied them in. No real installation looks + * like that, and the lab has been burned by exactly this shape before: it used to raise a registry + * inside the scenario, and a bootstrap that only worked against that registry went green here and + * would have failed on any bare machine. + * + * This bed hands over nothing. The scenario has no `images:` list at all. What the machines get is + * a container runtime and the host binary — prerequisites of a machine, not parts of a mesh — and + * from there: + * + * 1. novox is raised into a mesh of one by the installer, which BUILDS the control plane. + * 2. ace, shanks and g14 JOIN it, across a household NAT, with a token and nothing else. + * 3. The mesh builds the shared base from source, with its own builder. + * 4. The mesh builds a real module standing on that base. + * 5. The anchor runs it, pinned to a digest the mesh's own registry assigned. + * 6. A JOINED machine runs it — which means pulling from a registry that asks who it is. + * + * Steps 1 and 2 are proven elsewhere and are here because the later ones need them. **Steps 3 + * through 6 are what this bed exists for**, and 6 is the one nothing has ever checked: genesis + * puts the builder, the registry and everything they produce on ONE machine, so every earlier + * proof of a mesh-built module running is a proof about the machine that built it. A second + * machine has to fetch, and fetching needs an account nothing yet grants (novox/hq issue 042). + * + * Each step is recorded separately rather than allowed to throw, so a gap at 6 reports as a gap at + * 6 instead of erasing the evidence for 3, 4 and 5. + * + * MESH_LAB_INCUS='sudo -n incus' + * MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host + * MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap + * MESH_LAB_BUNDLE=.../mesh-host/examples/substrate-first-node.lock + * MESH_LAB_CATALOG=.../mesh-catalog/modules + * MESH_LAB_SOURCE=/mesh-control.git MESH_LAB_SOURCE_REF= + * MESH_LAB_KEEP=1 to leave it standing afterwards + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec, push } from "../../src/lifecycle/operate.ts"; +import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts"; +import { genesis, type GenesisResult } from "./genesis.ts"; + +const SCENARIO = "fresh-mesh"; +const CONTROL = "novox"; +const HOME_NODES = ["ace", "shanks", "g14"]; +/** The joined machine asked to run a mesh-built module. Any of the three would do. */ +const SECOND = "ace"; + +/** The anchor's public address — what every other machine dials, and what its own token must name. */ +const ANCHOR = "192.0.2.20"; +/** Where this mesh's registry answers, on the anchor's public address so a joined node can reach it. */ +const REGISTRY = `${ANCHOR}:5000`; + +/** + * The module built on top of the base, and the base it stands on. + * + * `amqp-ping` is deliberately small and deliberately REAL: its own TypeScript, compiled by the + * shared toolchain, running on the shared runtime, talking to the broker. A module whose artifact + * is a mirrored public image would pass every assertion below while skipping the whole of what is + * under test (novox/hq SELF-UPGRADE-PLAN, rule 1). + */ +const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" }; +const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping" }; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; +const source = process.env["MESH_LAB_SOURCE"] ?? ""; +const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? ""; +const KEEP = !!process.env["MESH_LAB_KEEP"]; +const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "fresh-mesh-live" : undefined); + +/** + * Where a repository other than the control plane's lives. + * + * Derived from `MESH_LAB_SOURCE` by swapping the last path segment, because every one of these + * repositories sits beside the others under the same owner on the same forge. Overridable, so a + * forge that is arranged differently does not need this bed edited. + */ +function forgeUrl(repo: string): string { + const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`]; + if (override) return override; + return source.replace(/[^/]+\.git$/, `${repo}.git`); +} +/** A branch is acceptable for an ordinary build; only genesis insists on a commit (ADR 0071). */ +const buildRef = process.env["MESH_LAB_BUILD_REF"] ?? "main"; + +/** + * The shared base's manifest, on this workstation. + * + * The base is a repository with a manifest at its root (novox/hq ADR 0069), so unlike the + * catalogue's modules it is not under `MESH_LAB_CATALOG`. Derived from that path on the convention + * that the checkouts sit beside each other, and overridable for a layout where they do not. + */ +const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ?? + resolve(catalogDir, "..", "..", BASE.repo, "module.json"); + +const skip = + !capability.usable ? capability.why : + !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : + !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" : + !source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" : + !sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" : + !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate template" : + !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" : + false; + +let instanceId = ""; +let raised: GenesisResult; + +// ---- talking to the machines ------------------------------------------------------------------ + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} +async function mesh(command: string, timeoutMs?: number): Promise { + return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs); +} +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +// ---- steps, recorded rather than thrown -------------------------------------------------------- + +interface Step { ok: boolean; why: string; said: string } +const steps = new Map(); +const order: string[] = []; + +/** Run a step, remember what it said, and never throw. A step whose predecessor failed is skipped. */ +async function step(name: string, after_: string | null, fn: () => Promise): Promise { + order.push(name); + if (after_ && !steps.get(after_)?.ok) { + steps.set(name, { ok: false, why: `not attempted — "${after_}" did not succeed`, said: "" }); + console.log(`SKIPPED ${name}`); + return; + } + console.log(`\n======== ${name} ========`); + try { + const said = await fn(); + steps.set(name, { ok: true, why: "", said }); + console.log(`OK ${name}`); + } catch (err) { + const why = (err as Error).message; + steps.set(name, { ok: false, why, said: "" }); + console.log(`FAILED ${name}\n${why.split("\n").slice(0, 25).join("\n")}`); + } +} + +function report(name: string): string { + const s = steps.get(name); + if (!s) return `${name}: never ran`; + const lines = order.map((n) => { + const it = steps.get(n); + return ` ${it?.ok ? "PASS" : "FAIL"} ${n}`; + }); + return `${s.why}\n\nWhere this bed got to:\n${lines.join("\n")}`; +} + +before(async () => { + if (skip) return; + + const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), + }); + instanceId = bed.instanceId; + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); + console.log(`NOTHING WAS LOADED: this scenario names no images. Every image on every machine ` + + `below was pulled from the internet or built by the mesh.`); + + // ---- 1. GENESIS ----------------------------------------------------------------------------- + // + // The shared description, the same one `genesis-single` calls. The bundle is the TEMPLATE with + // nothing held: no image is pre-resolved, because none is here to resolve to. + await step("novox becomes a mesh of one, raised by the installer", null, async () => { + try { + raised = await genesis({ + instanceId, + node: CONTROL, + installer: installer as string, + catalogDir, + // The broker's advertised address, corrected. + // + // The template hardcodes 192.0.2.10:5671 — the address of the anchor in the single-machine + // bed. A token carries this verbatim as the endpoint an enrolling node dials, so on a mesh + // whose anchor is somewhere else every node, including this one, would enrol against an + // address nothing answers on. The installer refuses to guess it and says so, which is + // right: it does not know what this machine is called from outside. + bundleTemplate: substrateBundle(bundle, []).replaceAll("192.0.2.10:5671", `${ANCHOR}:5671`), + registry: REGISTRY, + source, + sourceRef, + log: (m) => console.log(m), + }); + } catch (err) { + throw new Error(`the installer never ran: ${(err as Error).message}`); + } + if (!raised.ok) throw new Error(`${raised.step || "no step named"}: ${raised.why}\n\n${raised.report.join("\n")}`); + return raised.report.join("\n"); + }); + + // ---- 2. JOINING ----------------------------------------------------------------------------- + // + // Host binary and a token. novox is NOT in this loop — the installer enrolled it, and enrolling + // it again would offer the mesh a second identity for a node it already knows. + await step("three machines join it across the household gateway", + "novox becomes a mesh of one, raised by the installer", async () => { + const said: string[] = []; + for (const machine of HOME_NODES) { + await mesh(`node add ${machine}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const out = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); + assert.match(out, new RegExp(`enrolled as ${machine}`), out); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + said.push(` ${machine} enrolled and running`); + } + const nodes = await mesh("node list"); + said.push(nodes.trim()); + return said.join("\n"); + }); + + // ---- 3. THE MESH BUILDS THE SHARED BASE ----------------------------------------------------- + // + // The toolchain and runtime every module with code of its own stands on. It is a module, and it + // is built like one — cloned from the forge by the builder installing put here, compiled on the + // machine, published into the mesh's own registry. + await step("the mesh builds the shared base from source", + "three machines join it across the household gateway", async () => { + assert.ok(existsSync(baseManifest), + `no manifest for the shared base at ${baseManifest} — set MESH_LAB_BASE_MANIFEST`); + await push(instanceId, CONTROL, baseManifest, `/tmp/${BASE.module}.json`); + // Registered from the manifest the builder will also read, so what the mesh holds and what it + // builds are the same description of the same module. + await mesh(`module add /tmp/${BASE.module}.json`).catch(() => {}); + const built = await mesh( + `build ${forgeUrl(BASE.repo)} --ref ${buildRef} --wait 1200s`, 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + return built; + }); + + // ---- 4. AND A MODULE STANDING ON IT --------------------------------------------------------- + await step("the mesh builds a module standing on that base", + "the mesh builds the shared base from source", async () => { + const manifest = resolve(catalogDir, MODULE.module, "module.json"); + assert.ok(existsSync(manifest), `no manifest at ${manifest}`); + await push(instanceId, CONTROL, manifest, `/tmp/${MODULE.module}.json`); + await mesh(`module add /tmp/${MODULE.module}.json`); + const built = await mesh( + `build ${forgeUrl(MODULE.repo)} --path ${MODULE.path} --ref ${buildRef} --wait 1200s`, + 1_500_000); + assert.doesNotMatch(built, /failed/i, built); + // The point of the whole step: what came out is named by a digest this mesh's registry + // assigned, not by a placeholder and not by a tag. + const builds = await mesh(`builds ${MODULE.module}`); + assert.match(builds, /sha256:[0-9a-f]{12}/, + `the build recorded no digest — the module is not pinned to anything this registry serves:\n${builds}`); + return `${built}\n${builds}`; + }); + + // ---- 5. THE ANCHOR RUNS IT ------------------------------------------------------------------ + // + // The machine that built it. This is the case every earlier proof covered, and it is here as the + // control for step 6: if this fails, step 6's failure says nothing about fetching. + await step("the anchor runs the module the mesh built", + "the mesh builds a module standing on that base", async () => { + await mesh(`module issue ${MODULE.module} --node ${CONTROL}`).catch(() => {}); + await mesh(`assign ${CONTROL} ${MODULE.module}`); + await mesh(`push ${CONTROL}`, 600_000); + for (let i = 0; i < 40; i++) { + const ps = (await on(CONTROL, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out; + if (/amqp-ping/.test(ps) && /Up /.test(ps.split("\n").find((l) => l.includes("amqp-ping")) ?? "")) { + return ps; + } + await new Promise((r) => setTimeout(r, 5_000)); + } + throw new Error(`amqp-ping never came up on ${CONTROL}:\n` + + (await on(CONTROL, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out); + }); + + // ---- 6. AND A MACHINE THAT DID NOT BUILD IT ------------------------------------------------- + // + // **The thing nothing has ever checked.** ace did not build this image and has never seen it. To + // run it, it must fetch it from the mesh's registry — and a joined node has no account there. + // The mesh grants a consumer a credential for a database; it does not yet do so for the store + // its own images live in (novox/hq issue 042). + // + // Asked anyway, and asked LAST, so that when it fails the five steps above still stand as + // evidence of what does work. + await step(`a joined machine runs the module the mesh built`, + "the anchor runs the module the mesh built", async () => { + await mesh(`module issue ${MODULE.module} --node ${SECOND}`).catch(() => {}); + await mesh(`assign ${SECOND} ${MODULE.module}`); + await mesh(`push ${SECOND}`, 600_000); + for (let i = 0; i < 40; i++) { + const ps = (await on(SECOND, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out; + const line = ps.split("\n").find((l) => l.includes("amqp-ping")); + if (line && /Up /.test(line)) return ps; + await new Promise((r) => setTimeout(r, 5_000)); + } + const state = (await on(SECOND, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + const log = (await on(SECOND, `tail -40 /var/log/mesh-host.log`)).out; + throw new Error( + `amqp-ping never came up on ${SECOND} — the machine that did NOT build it.\n\n` + + `containers:\n${state}\n\nwhat the host said:\n${log}`); + }); + + console.log(`\n================ WHAT THIS MESH DID FOR ITSELF ================`); + for (const n of order) console.log(` ${steps.get(n)?.ok ? "PASS" : "FAIL"} ${n}`); +}, { timeout: 7_200_000 }); + +after(async () => { + if (KEEP) { + console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`); + return; + } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +for (const name of [ + "novox becomes a mesh of one, raised by the installer", + "three machines join it across the household gateway", + "the mesh builds the shared base from source", + "the mesh builds a module standing on that base", + "the anchor runs the module the mesh built", + "a joined machine runs the module the mesh built", +]) { + test(name, { skip, timeout: 60_000 }, () => { + assert.ok(steps.get(name)?.ok, report(name)); + }); +}