jschoubben
09a22de78f
The installer asks where a human must choose, and this bed has no human — so every choice arrives as a flag, and a required choice with no flag is the installer refusing, which is the behaviour rather than a lab problem. Both choices have one option today, so the flags are redundant on purpose: the day a second filter exists this bed keeps working instead of refusing, and choosing becomes a thing it visibly does. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx
515 lines
26 KiB
TypeScript
515 lines
26 KiB
TypeScript
/**
|
|
* FOUR FRESH MACHINES, AND NOTHING HANDED TO THEM.
|
|
*
|
|
* The four-machine bed (`whole-mesh-full`) proves the mesh converges. It does so by loading
|
|
* thirty-four of the mesh's own images onto its machines from the workstation, because it does not
|
|
* build them — something beside the bed built them and copied them in. No real installation looks
|
|
* like that, and the lab has been burned by exactly this shape before: it used to raise a registry
|
|
* inside the scenario, and a bootstrap that only worked against that registry went green here and
|
|
* would have failed on any bare machine.
|
|
*
|
|
* This bed hands over nothing. The scenario has no `images:` list at all. What the machines get is
|
|
* a container runtime and the host binary — prerequisites of a machine, not parts of a mesh — and
|
|
* from there:
|
|
*
|
|
* 1. novox is raised into a mesh of one by the installer, which BUILDS the control plane.
|
|
* 2. ace, shanks and g14 JOIN it, across a household NAT, with a token and nothing else.
|
|
* 3. The mesh builds the shared base from source, with its own builder.
|
|
* 4. The mesh builds a real module standing on that base.
|
|
* 5. The anchor runs it, pinned to a digest the mesh's own registry assigned.
|
|
* 6. A JOINED machine runs it — which means pulling from a registry that asks who it is.
|
|
*
|
|
* Steps 1 and 2 are proven elsewhere and are here because the later ones need them. **Steps 3
|
|
* through 6 are what this bed exists for**, and 6 is the one nothing has ever checked: genesis
|
|
* puts the builder, the registry and everything they produce on ONE machine, so every earlier
|
|
* proof of a mesh-built module running is a proof about the machine that built it. A second
|
|
* machine has to fetch, and fetching needs an account nothing yet grants (novox/hq issue 042).
|
|
*
|
|
* Each step is recorded separately rather than allowed to throw, so a gap at 6 reports as a gap at
|
|
* 6 instead of erasing the evidence for 3, 4 and 5.
|
|
*
|
|
* MESH_LAB_INCUS='sudo -n incus'
|
|
* MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
|
|
* MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap
|
|
* MESH_LAB_BUNDLE=.../mesh-host/examples/substrate-first-node.lock
|
|
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
|
* MESH_LAB_SOURCE=<forge>/mesh-control.git MESH_LAB_SOURCE_REF=<commit>
|
|
* MESH_LAB_KEEP=1 to leave it standing afterwards
|
|
*/
|
|
import { test, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { existsSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
import { raise } from "../../src/lifecycle/raise.ts";
|
|
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
|
import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts";
|
|
import { genesis, type GenesisResult } from "./genesis.ts";
|
|
|
|
const SCENARIO = "fresh-mesh";
|
|
const CONTROL = "anchor";
|
|
const HOME_NODES = ["home-server", "workstation", "laptop"];
|
|
/** The joined machine asked to run a mesh-built module. Any of the three would do. */
|
|
const SECOND = "home-server";
|
|
|
|
/** The anchor's public address — what every other machine dials, and what its own token must name. */
|
|
const ANCHOR = "192.0.2.20";
|
|
/** Where this mesh's registry answers, on the anchor's public address so a joined node can reach it. */
|
|
const REGISTRY = `${ANCHOR}:5000`;
|
|
|
|
/**
|
|
* The module built on top of the base, and the base it stands on.
|
|
*
|
|
* `amqp-ping` is deliberately small and deliberately REAL: its own TypeScript, compiled by the
|
|
* shared toolchain, running on the shared runtime, talking to the broker. A module whose artifact
|
|
* is a mirrored public image would pass every assertion below while skipping the whole of what is
|
|
* under test (novox/hq SELF-UPGRADE-PLAN, rule 1).
|
|
*/
|
|
const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" };
|
|
/**
|
|
* What `amqp-ping` requires, and what the substrate does not supply.
|
|
*
|
|
* The installer raises a broker, but as a bundle resource — plumbing, not a module the mesh has a
|
|
* record of, so it provides nothing to anything. A module asking for `amqp` is asking for a
|
|
* provider in the graph, and this is it. No build: its image is upstream.
|
|
*/
|
|
const PROVIDER = { module: "lavinmq", repo: "mesh-catalog", path: "modules/lavinmq", container: "mesh-lavinmq" };
|
|
/**
|
|
* The store, and the catalogue that cannot exist without it.
|
|
*
|
|
* Both were missing from this test entirely, and nothing complained, because nothing asked. A mesh
|
|
* with no catalogue holds no module graph — it cannot say what it has, what a module is made of,
|
|
* what a change reaches, or what must be rebuilt. It ran anyway, which is the point: "the mesh is
|
|
* up" was being read off genesis finishing.
|
|
*/
|
|
const STORE = { module: "postgres", repo: "mesh-catalog", path: "modules/postgres", container: "mesh-postgres" };
|
|
const CATALOGUE = { module: "mesh-catalog", repo: "mesh-catalog", path: "modules/mesh-catalog", container: "mesh-catalog" };
|
|
/** The control plane, rebuilt from its own repository — the step that ends the installer's tenure. */
|
|
const CONTROL_PLANE = { module: "mesh-control", repo: "mesh-control", path: "", container: "mesh-control" };
|
|
/** Named once, because the step title is also how later steps say what they waited on. */
|
|
const NEEDS = "the mesh runs a broker for that module to talk to";
|
|
const GENESIS = "a bare machine becomes a mesh of one, raised by the installer";
|
|
const BASE_BUILT = "the mesh builds the shared base from source";
|
|
const STORE_RUNS = "the mesh builds and runs a store of its own";
|
|
const CATALOGUE_RUNS = "the mesh builds and runs its own catalogue";
|
|
const CONTROL_REBUILT = "the mesh rebuilds its own control plane from source";
|
|
const MODULE_BUILT = "the mesh builds a module standing on that base";
|
|
const ANCHOR_RUNS = "the anchor runs the module the mesh built";
|
|
const JOINED = "three machines join the mesh, and reach it over its private network";
|
|
const SECOND_RUNS = "a joined machine runs the module the mesh built";
|
|
const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping" };
|
|
|
|
const capability = await labIsUsable();
|
|
const binary = hostBinaryPath();
|
|
const installer = bootstrapBinaryPath();
|
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
|
const source = process.env["MESH_LAB_SOURCE"] ?? "";
|
|
const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? "";
|
|
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
|
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "fresh-mesh-live" : undefined);
|
|
|
|
/**
|
|
* Where a repository other than the control plane's lives.
|
|
*
|
|
* Derived from `MESH_LAB_SOURCE` by swapping the last path segment, because every one of these
|
|
* repositories sits beside the others under the same owner on the same forge. Overridable, so a
|
|
* forge that is arranged differently does not need this bed edited.
|
|
*/
|
|
function forgeUrl(repo: string): string {
|
|
const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`];
|
|
if (override) return override;
|
|
return source.replace(/[^/]+\.git$/, `${repo}.git`);
|
|
}
|
|
/**
|
|
* What to build, per repository.
|
|
*
|
|
* A branch is acceptable for an ordinary build; only genesis insists on a commit (ADR 0071). Per
|
|
* repository rather than one value for all of them, because a change under test usually lives in
|
|
* one repository and the rest should be built from what everyone else has — building them all from
|
|
* a feature branch would prove that branch against itself.
|
|
*
|
|
* MESH_LAB_BUILD_REF the default for every repository
|
|
* MESH_LAB_BUILD_REF_MESH_CATALOG ...overridden for one
|
|
*/
|
|
function refFor(repo: string): string {
|
|
const override = process.env[`MESH_LAB_BUILD_REF_${repo.toUpperCase().replaceAll("-", "_")}`];
|
|
return override ?? process.env["MESH_LAB_BUILD_REF"] ?? "main";
|
|
}
|
|
|
|
/**
|
|
* The shared base's manifest, on this workstation.
|
|
*
|
|
* The base is a repository with a manifest at its root (novox/hq ADR 0069), so unlike the
|
|
* catalogue's modules it is not under `MESH_LAB_CATALOG`. Derived from that path on the convention
|
|
* that the checkouts sit beside each other, and overridable for a layout where they do not.
|
|
*/
|
|
const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ??
|
|
resolve(catalogDir, "..", "..", BASE.repo, "module.json");
|
|
|
|
const skip =
|
|
!capability.usable ? capability.why :
|
|
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
|
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
|
!source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" :
|
|
!sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" :
|
|
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate template" :
|
|
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
|
false;
|
|
|
|
let instanceId = "";
|
|
let raised: GenesisResult;
|
|
|
|
// ---- talking to the machines ------------------------------------------------------------------
|
|
|
|
function quote(s: string): string {
|
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
}
|
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
const { stdout } = await exec(instanceId, machine, [
|
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
], timeoutMs);
|
|
const marker = stdout.lastIndexOf("__exit=");
|
|
if (marker < 0) return { out: stdout, ok: false };
|
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
}
|
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
|
const { out, ok } = await on(machine, command, timeoutMs);
|
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
|
return out;
|
|
}
|
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
|
|
}
|
|
|
|
/** A module the mesh has to make for itself: where its source is, and what it runs when it works. */
|
|
interface CoreModule { module: string; repo: string; path: string; container: string }
|
|
|
|
/**
|
|
* Build a module from source, install it, and wait for it to actually run.
|
|
*
|
|
* The whole sequence a module goes through, in one place because the mesh has to do it for several
|
|
* before it is a mesh at all: register the manifest, build it from its repository and path, issue
|
|
* the broker account its runtime needs, assign it, send it, and then ask the machine whether the
|
|
* container is up.
|
|
*
|
|
* **The account is not optional and is not swallowed.** A module's runtime is a tool host: it
|
|
* connects to the mesh's broker before doing anything. Without an account the mesh still fills the
|
|
* secret the module declares it owns — with a generated value — so the container starts, fails to
|
|
* parse a password as a credential document, and loops on a JSON syntax error mentioning no
|
|
* missing account. That cost a step that reported PASS.
|
|
*/
|
|
async function bringUp(m: CoreModule): Promise<string> {
|
|
await registerModule(m.module, resolve(catalogDir, m.module, "module.json"));
|
|
const built = await mesh(
|
|
`build ${forgeUrl(m.repo)} --path ${m.path} --ref ${refFor(m.repo)} --wait 1200s`, 1_500_000);
|
|
assert.doesNotMatch(built, /failed/i, built);
|
|
await mesh(`module issue ${m.module} --node ${CONTROL}`);
|
|
await mesh(`assign ${CONTROL} ${m.module}`);
|
|
await mesh(`push ${CONTROL}`, 600_000);
|
|
return `${built}\n${await waitForContainer(CONTROL, m.container)}`;
|
|
}
|
|
|
|
/**
|
|
* Wait for one container, BY NAME, to be running.
|
|
*
|
|
* **Named exactly, because substring matching passed a step that had failed.** Waiting for "a
|
|
* container whose name contains lavinmq" was satisfied by the broker — `lavinmq`, up and healthy —
|
|
* while the thing actually under test, the module's own runtime `mesh-lavinmq`, was crash-looping
|
|
* beside it. The step went green and the fault was found by reading `docker ps` by hand.
|
|
*/
|
|
async function waitForContainer(node: string, container: string, seconds = 200): Promise<string> {
|
|
const deadline = Date.now() + seconds * 1_000;
|
|
let last = "";
|
|
while (Date.now() < deadline) {
|
|
const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
|
last = ps;
|
|
const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container);
|
|
if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps;
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
const logs = (await on(node, `docker logs --tail 15 ${container} 2>&1`)).out;
|
|
throw new Error(
|
|
`${container} is not running on ${node}.\n\ncontainers:\n${last}\n\nwhat it said:\n${logs}`);
|
|
}
|
|
|
|
/**
|
|
* Register a module from a manifest on this workstation.
|
|
*
|
|
* **The control plane runs in a container, so a file on the machine is not a file it can open.**
|
|
* Pushing the manifest to the machine and naming that path got `no such file or directory` from
|
|
* inside mesh-control, which is correct and was briefly mistaken for a missing manifest. It is
|
|
* copied the last step of the way with `docker cp`.
|
|
*
|
|
* **Into the root, not into /tmp.** The control plane's image is a minimal one and has no `/tmp`
|
|
* to copy into — `docker cp` says so in those words. `/` is the one directory every image has.
|
|
*/
|
|
async function registerModule(module: string, manifest: string): Promise<string> {
|
|
assert.ok(existsSync(manifest), `no manifest for ${module} at ${manifest}`);
|
|
const onMachine = `/tmp/${module}.json`;
|
|
const inContainer = `/${module}.json`;
|
|
await push(instanceId, CONTROL, manifest, onMachine);
|
|
await must(CONTROL, `docker cp ${onMachine} mesh-control:${inContainer}`);
|
|
return mesh(`module add ${inContainer}`);
|
|
}
|
|
function tokenFrom(said: string): string {
|
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
assert.ok(found, `no token in:\n${said}`);
|
|
return found;
|
|
}
|
|
|
|
// ---- steps, recorded rather than thrown --------------------------------------------------------
|
|
|
|
interface Step { ok: boolean; why: string; said: string }
|
|
const steps = new Map<string, Step>();
|
|
const order: string[] = [];
|
|
|
|
/** Run a step, remember what it said, and never throw. A step whose predecessor failed is skipped. */
|
|
async function step(name: string, after_: string | null, fn: () => Promise<string>): Promise<void> {
|
|
order.push(name);
|
|
if (after_ && !steps.get(after_)?.ok) {
|
|
steps.set(name, { ok: false, why: `not attempted — "${after_}" did not succeed`, said: "" });
|
|
console.log(`SKIPPED ${name}`);
|
|
return;
|
|
}
|
|
console.log(`\n======== ${name} ========`);
|
|
try {
|
|
const said = await fn();
|
|
steps.set(name, { ok: true, why: "", said });
|
|
console.log(`OK ${name}`);
|
|
} catch (err) {
|
|
const why = (err as Error).message;
|
|
steps.set(name, { ok: false, why, said: "" });
|
|
console.log(`FAILED ${name}\n${why.split("\n").slice(0, 25).join("\n")}`);
|
|
}
|
|
}
|
|
|
|
function report(name: string): string {
|
|
const s = steps.get(name);
|
|
if (!s) return `${name}: never ran`;
|
|
const lines = order.map((n) => {
|
|
const it = steps.get(n);
|
|
return ` ${it?.ok ? "PASS" : "FAIL"} ${n}`;
|
|
});
|
|
return `${s.why}\n\nWhere this bed got to:\n${lines.join("\n")}`;
|
|
}
|
|
|
|
before(async () => {
|
|
if (skip) return;
|
|
|
|
const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
onProgress: (m) => console.log(`raise: ${m}`),
|
|
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
|
});
|
|
instanceId = bed.instanceId;
|
|
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
|
console.log(`NOTHING WAS LOADED: this scenario names no images. Every image on every machine ` +
|
|
`below was pulled from the internet or built by the mesh.`);
|
|
|
|
// ---- 1. GENESIS -----------------------------------------------------------------------------
|
|
//
|
|
// The shared description, the same one `genesis-single` calls. The bundle is the TEMPLATE with
|
|
// nothing held: no image is pre-resolved, because none is here to resolve to.
|
|
await step("a bare machine becomes a mesh of one, raised by the installer", null, async () => {
|
|
try {
|
|
raised = await genesis({
|
|
instanceId,
|
|
node: CONTROL,
|
|
installer: installer as string,
|
|
catalogDir,
|
|
// The broker's advertised address, corrected.
|
|
//
|
|
// The template hardcodes 192.0.2.10:5671 — the address of the anchor in the single-machine
|
|
// bed. A token carries this verbatim as the endpoint an enrolling node dials, so on a mesh
|
|
// whose anchor is somewhere else every node, including this one, would enrol against an
|
|
// address nothing answers on. The installer refuses to guess it and says so, which is
|
|
// right: it does not know what this machine is called from outside.
|
|
bundleTemplate: substrateBundle(bundle, []).replaceAll("192.0.2.10:5671", `${ANCHOR}:5671`),
|
|
registry: REGISTRY,
|
|
source,
|
|
sourceRef,
|
|
toolsSource: forgeUrl(BASE.repo),
|
|
catalogSource: forgeUrl(MODULE.repo),
|
|
catalogRef: refFor(MODULE.repo),
|
|
log: (m) => console.log(m),
|
|
});
|
|
} catch (err) {
|
|
throw new Error(`the installer never ran: ${(err as Error).message}`);
|
|
}
|
|
if (!raised.ok) throw new Error(`${raised.step || "no step named"}: ${raised.why}\n\n${raised.report.join("\n")}`);
|
|
return raised.report.join("\n");
|
|
});
|
|
|
|
// ---- 2..6. THE MESH BECOMES ONE --------------------------------------------------------------
|
|
//
|
|
// **Joining used to be here, second, and that was the wrong order.** Three machines were enrolled
|
|
// into a mesh that could not yet produce a single module, and the step was reported as though
|
|
// something had been shown. They do join — reliably — but joining a mesh that can build nothing
|
|
// proves only that enrolment works, which was never the doubtful part.
|
|
//
|
|
// `17-raising-a-mesh` says it plainly: genesis ends with a mesh of one that RUNS, and that is not
|
|
// the same as a mesh that WORKS; what remains after the core modules are built is "adding
|
|
// machines, and deciding what they run". So everything a mesh needs to be a mesh happens first,
|
|
// and machines arrive at the end.
|
|
|
|
// The toolchain and runtime every module with code of its own stands on. It is a module, and it
|
|
// is built like one — cloned from the forge by the builder installing put here, compiled on the
|
|
// machine, published into the mesh's own registry.
|
|
await step(BASE_BUILT, GENESIS, async () => {
|
|
// Registered from the manifest the builder will also read, so what the mesh holds and what it
|
|
// builds are the same description of the same module.
|
|
//
|
|
// **Not swallowed.** This call used to end in `.catch(() => {})`, on the reasoning that the
|
|
// base might already be known. It hid a real failure — the manifest was being named at a path
|
|
// inside a container that had never seen it — and the step passed anyway, because a base with
|
|
// nothing to stand on builds whether or not the mesh has a record of it.
|
|
await registerModule(BASE.module, baseManifest);
|
|
const built = await mesh(
|
|
`build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000);
|
|
assert.doesNotMatch(built, /failed/i, built);
|
|
return built;
|
|
});
|
|
|
|
// A store of its own. **Not the substrate's.** The installer raises a store for the control
|
|
// plane to keep its own records in, the way it raises a broker — plumbing, not a module the mesh
|
|
// has any record of, so it provides nothing to anything. A module that wants a database wants a
|
|
// provider in the graph, and the catalogue below is the first thing to want one.
|
|
await step(STORE_RUNS, BASE_BUILT, () => bringUp(STORE));
|
|
|
|
// And the catalogue, which was missing from this test altogether.
|
|
//
|
|
// Without it the mesh holds no module graph: it cannot say what it has, what a module is made
|
|
// of, what a change to one reaches, or what must be rebuilt. A mesh in that state still runs,
|
|
// which is exactly how its absence went unnoticed — "the mesh is up" was being read off the
|
|
// installer finishing rather than off the mesh being able to answer anything.
|
|
await step(CATALOGUE_RUNS, STORE_RUNS, () => bringUp(CATALOGUE));
|
|
|
|
// The control plane, rebuilt from its own repository and rolled out.
|
|
//
|
|
// The installer built it once, which is what got the mesh running. Building it again THROUGH THE
|
|
// MODULE PATH — build, notice the version moved, roll it out — is a different claim: it is the
|
|
// moment the mesh stops depending on the installer for anything, and the first time the thing
|
|
// that performs an upgrade performs one on itself.
|
|
await step(CONTROL_REBUILT, CATALOGUE_RUNS, async () => {
|
|
const built = await mesh(
|
|
`build ${forgeUrl(CONTROL_PLANE.repo)} --ref ${sourceRef} --wait 1200s`, 1_500_000);
|
|
assert.doesNotMatch(built, /failed/i, built);
|
|
const rolled = await mesh(`upgrade ${CONTROL_PLANE.module} roll-out`, 900_000);
|
|
// Asked of the machine rather than believed from the command: the control plane that answers
|
|
// afterwards is the one that has to be running for anything below this line to mean anything.
|
|
await waitForContainer(CONTROL, CONTROL_PLANE.container);
|
|
return `${built}\n${rolled}`;
|
|
});
|
|
|
|
// ---- 7..8. SOMETHING TO RUN ---------------------------------------------------------------
|
|
await step(MODULE_BUILT, CONTROL_REBUILT, async () => {
|
|
await registerModule(MODULE.module, resolve(catalogDir, MODULE.module, "module.json"));
|
|
const built = await mesh(
|
|
`build ${forgeUrl(MODULE.repo)} --path ${MODULE.path} --ref ${refFor(MODULE.repo)} --wait 1200s`,
|
|
1_500_000);
|
|
assert.doesNotMatch(built, /failed/i, built);
|
|
// The point of the whole step: what came out is named by a digest this mesh's registry
|
|
// assigned, not by a placeholder and not by a tag.
|
|
const builds = await mesh(`builds ${MODULE.module}`);
|
|
assert.match(builds, /sha256:[0-9a-f]{12}/,
|
|
`the build recorded no digest — the module is not pinned to anything this registry serves:\n${builds}`);
|
|
return `${built}\n${builds}`;
|
|
});
|
|
|
|
// `amqp-ping` requires the `amqp` provision, and the mesh refused to place it: "nothing provides
|
|
// amqp, wanted by amqp-ping". That refusal is correct and is the reason this step exists rather
|
|
// than the reason to pick an easier module.
|
|
//
|
|
// `lavinmq` is that module. It was first added here on the belief that it needed no building —
|
|
// its broker is an upstream image — and the mesh refused it again: two of its three containers
|
|
// named a placeholder digest, "which is never a real image". Also right. The broker is upstream,
|
|
// but the module is not only the broker: it carries a run-once bootstrap that writes the broker's
|
|
// configuration, a provisioner that grants each consumer its own vhost and user, tools and an
|
|
// event consumer, all of it its own code.
|
|
await step(NEEDS, MODULE_BUILT, () => bringUp(PROVIDER));
|
|
|
|
// The machine that built it. This is the case every earlier proof covered, and it is here as the
|
|
// control for the last step: if this fails, that one's failure says nothing about fetching.
|
|
await step(ANCHOR_RUNS, NEEDS, async () => {
|
|
await mesh(`module issue ${MODULE.module} --node ${CONTROL}`);
|
|
await mesh(`assign ${CONTROL} ${MODULE.module}`);
|
|
await mesh(`push ${CONTROL}`, 600_000);
|
|
return waitForContainer(CONTROL, MODULE.module);
|
|
});
|
|
|
|
// ---- 9. NOW MACHINES MAY ARRIVE ---------------------------------------------------------------
|
|
//
|
|
// Host binary and a token, and nothing else. The anchor is NOT in this loop — the installer
|
|
// enrolled it, and enrolling it again would offer the mesh a second identity for a node it
|
|
// already knows.
|
|
//
|
|
// And they are placed on the mesh's private network, which the earlier ordering never did. The
|
|
// mesh refuses to bind a consumer to a provider on another machine when either is missing from
|
|
// it — "they are not both on the private network" — so without this the last step fails for a
|
|
// reason that has nothing to do with what it is asking.
|
|
await step(JOINED, ANCHOR_RUNS, async () => {
|
|
const said: string[] = [];
|
|
await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site hosting`);
|
|
for (const machine of HOME_NODES) {
|
|
await mesh(`node add ${machine}`);
|
|
const token = tokenFrom(await mesh(`token issue --node ${machine}`));
|
|
const out = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
|
|
assert.match(out, new RegExp(`enrolled as ${machine}`), out);
|
|
await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
await mesh(`overlay place ${machine} --site home`);
|
|
said.push(` ${machine} enrolled, running, and on the private network`);
|
|
}
|
|
said.push((await mesh("node list")).trim());
|
|
said.push((await mesh("overlay show")).trim());
|
|
return said.join("\n");
|
|
});
|
|
|
|
// ---- 10. AND A MACHINE THAT DID NOT BUILD IT --------------------------------------------------
|
|
//
|
|
// **The thing nothing has ever checked.** This machine did not build the image and has never seen
|
|
// it. To run it, it must fetch it from the mesh's registry — and a joined node has no account
|
|
// there (novox/hq issue 042), nor any reason to trust a registry serving plain HTTP over the
|
|
// network (issue 048). Both are open, and both are invisible on a mesh of one.
|
|
//
|
|
// Asked anyway, and asked LAST, so that when it fails everything above still stands as evidence
|
|
// of what does work.
|
|
await step(SECOND_RUNS, JOINED, async () => {
|
|
await mesh(`module issue ${MODULE.module} --node ${SECOND}`);
|
|
await mesh(`assign ${SECOND} ${MODULE.module}`);
|
|
await mesh(`push ${SECOND}`, 600_000);
|
|
try {
|
|
return await waitForContainer(SECOND, MODULE.module);
|
|
} catch (err) {
|
|
const log = (await on(SECOND, `tail -40 /var/log/mesh-host.log`)).out;
|
|
throw new Error(`${(err as Error).message}\n\nwhat the host said:\n${log}`);
|
|
}
|
|
});
|
|
|
|
console.log(`\n================ WHAT THIS MESH DID FOR ITSELF ================`);
|
|
for (const n of order) console.log(` ${steps.get(n)?.ok ? "PASS" : "FAIL"} ${n}`);
|
|
}, { timeout: 7_200_000 });
|
|
|
|
after(async () => {
|
|
if (KEEP) {
|
|
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
|
|
return;
|
|
}
|
|
if (instanceId) await destroy(instanceId);
|
|
await destroyAll(`${SCENARIO}-`);
|
|
}, { timeout: 900_000 });
|
|
|
|
for (const name of [
|
|
"a bare machine becomes a mesh of one, raised by the installer",
|
|
"three machines join it across the household gateway",
|
|
"the mesh builds the shared base from source",
|
|
"the mesh builds a module standing on that base",
|
|
NEEDS,
|
|
"the anchor runs the module the mesh built",
|
|
"a joined machine runs the module the mesh built",
|
|
]) {
|
|
test(name, { skip, timeout: 60_000 }, () => {
|
|
assert.ok(steps.get(name)?.ok, report(name));
|
|
});
|
|
}
|