1062 lines
55 KiB
TypeScript
1062 lines
55 KiB
TypeScript
/**
|
||
* FOUR FRESH MACHINES, AND NOTHING HANDED TO THEM.
|
||
*
|
||
* The four-machine bed (`whole-mesh-full`) proves the mesh converges. It does so by loading
|
||
* thirty-four of the mesh's own images onto its machines from the workstation, because it does not
|
||
* build them — something beside the bed built them and copied them in. No real installation looks
|
||
* like that, and the lab has been burned by exactly this shape before: it used to raise a registry
|
||
* inside the scenario, and a bootstrap that only worked against that registry went green here and
|
||
* would have failed on any bare machine.
|
||
*
|
||
* This bed hands over nothing. The scenario has no `images:` list at all. What the machines get is
|
||
* a container runtime and the host binary — prerequisites of a machine, not parts of a mesh — and
|
||
* from there:
|
||
*
|
||
* 1. novox is raised into a mesh of one by the installer, which BUILDS the control plane.
|
||
* 2. ace, shanks and g14 JOIN it, across a household NAT, with a token and nothing else.
|
||
* 3. The mesh builds the shared base from source, with its own builder.
|
||
* 4. The mesh builds a real module standing on that base.
|
||
* 5. The anchor runs it, pinned to a digest the mesh's own registry assigned.
|
||
* 6. A JOINED machine runs it — which means pulling from a registry that asks who it is.
|
||
*
|
||
* Steps 1 and 2 are proven elsewhere and are here because the later ones need them. **Steps 3
|
||
* through 6 are what this bed exists for**, and 6 is the one nothing has ever checked: genesis
|
||
* puts the builder, the registry and everything they produce on ONE machine, so every earlier
|
||
* proof of a mesh-built module running is a proof about the machine that built it. A second
|
||
* machine has to fetch, and fetching needs an account nothing yet grants (novox/hq issue 042).
|
||
*
|
||
* Each step is recorded separately rather than allowed to throw, so a gap at 6 reports as a gap at
|
||
* 6 instead of erasing the evidence for 3, 4 and 5.
|
||
*
|
||
* MESH_LAB_INCUS='sudo -n incus'
|
||
* MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
|
||
* MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap
|
||
* MESH_LAB_BUNDLE=.../mesh-host/examples/substrate-first-node.lock
|
||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||
* MESH_LAB_SOURCE=<forge>/mesh-control.git MESH_LAB_SOURCE_REF=<commit>
|
||
* MESH_LAB_KEEP=1 to leave it standing afterwards
|
||
*/
|
||
import { test, before, after } from "node:test";
|
||
import assert from "node:assert/strict";
|
||
import { existsSync, writeFileSync, appendFileSync } from "node:fs";
|
||
import { execFileSync } from "node:child_process";
|
||
import { resolve } from "node:path";
|
||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||
import { raise } from "../../src/lifecycle/raise.ts";
|
||
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
||
import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||
import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts";
|
||
import { genesis, type GenesisResult } from "./genesis.ts";
|
||
import { incus } from "../../src/incus/client.ts";
|
||
import { instanceNameOf } from "../../src/lifecycle/operate.ts";
|
||
import { waitUntilAllUsable } from "../../src/lifecycle/ready.ts";
|
||
|
||
const SCENARIO = "one-node-mesh";
|
||
const CONTROL = "anchor";
|
||
|
||
/** The anchor's public address — what every other machine dials, and what its own token must name. */
|
||
const ANCHOR = "192.0.2.10";
|
||
/** Where this mesh's registry answers, on the anchor's public address so a joined node can reach it. */
|
||
const REGISTRY = `${ANCHOR}:5000`;
|
||
|
||
/**
|
||
* The module built on top of the base, and the base it stands on.
|
||
*
|
||
* `amqp-ping` is deliberately small and deliberately REAL: its own TypeScript, compiled by the
|
||
* shared toolchain, running on the shared runtime, talking to the broker. A module whose artifact
|
||
* is a mirrored public image would pass every assertion below while skipping the whole of what is
|
||
* under test (novox/hq SELF-UPGRADE-PLAN, rule 1).
|
||
*/
|
||
const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" };
|
||
/**
|
||
* What `amqp-ping` requires, and what the substrate does not supply.
|
||
*
|
||
* The installer raises a broker, but as a bundle resource — plumbing, not a module the mesh has a
|
||
* record of, so it provides nothing to anything. A module asking for `amqp` is asking for a
|
||
* provider in the graph, and this is it. No build: its image is upstream.
|
||
*/
|
||
const PROVIDER = { module: "lavinmq", repo: "mesh-catalog", path: "modules/lavinmq", container: "mesh-lavinmq" };
|
||
/**
|
||
* The store, and the catalogue that cannot exist without it.
|
||
*
|
||
* Both were missing from this test entirely, and nothing complained, because nothing asked. A mesh
|
||
* with no catalogue holds no module graph — it cannot say what it has, what a module is made of,
|
||
* what a change reaches, or what must be rebuilt. It ran anyway, which is the point: "the mesh is
|
||
* up" was being read off genesis finishing.
|
||
*/
|
||
/** The one word that puts a mesh on a private network and gives its machines names. */
|
||
const NETWORK_MODULE = "networking";
|
||
/**
|
||
* The packet filter, which is a module too and was assigned to nothing.
|
||
*
|
||
* The rules are generated from what every module declares it listens on, so a mesh with no filter
|
||
* is not "open by accident" — it is a mesh where the whole of that generation has never run. It
|
||
* claims a seat (`the-packet-filter`) because a machine has one of these and two things writing
|
||
* rules is a coin toss about which survives.
|
||
*/
|
||
const FILTER_MODULE = "nftables";
|
||
const STORE = { module: "postgres", repo: "mesh-catalog", path: "modules/postgres", container: "mesh-postgres" };
|
||
const CATALOGUE = { module: "mesh-catalog", repo: "mesh-catalog", path: "modules/mesh-catalog", container: "mesh-catalog" };
|
||
/** The control plane, rebuilt from its own repository — the step that ends the installer's tenure. */
|
||
const CONTROL_PLANE = { module: "mesh-control", repo: "mesh-control", path: "", container: "mesh-control" };
|
||
|
||
/**
|
||
* What this mesh must hold when it is finished, and what must be RUNNING on the machine.
|
||
*
|
||
* Written down as a list rather than checked one step at a time, because "is this a mesh" is a
|
||
* question about the set. The three the build loop cannot produce for itself — the control plane,
|
||
* the registry and the builder — are here too: they are carried in, and a mesh missing any of them
|
||
* is not one.
|
||
*/
|
||
const MUST_HOLD = ["mesh-control", "distribution", "builder", "mesh-tools", "postgres",
|
||
"mesh-catalog", "lavinmq", "amqp-ping"];
|
||
const MUST_RUN = ["mesh-control", "mesh-registry", "mesh-broker", "mesh-store",
|
||
"mesh-postgres", "mesh-catalog", "mesh-lavinmq", "amqp-ping"];
|
||
/** Named once, because the step title is also how later steps say what they waited on. */
|
||
const NEEDS = "the mesh runs a broker for that module to talk to";
|
||
const GENESIS = "a bare machine becomes a mesh of one, raised by the installer";
|
||
const SUBSTRATE = "the substrate is up — a store and a broker of the mesh's own";
|
||
const BUILT_CP = "the control plane is one this mesh built, not one it was handed";
|
||
const PIVOTED = "the pivot finished — what raised the mesh is gone";
|
||
const HAS_REGISTRY = "the registry serves this mesh its own images";
|
||
const ENROLLED = "the machine is enrolled, and an agent is running on it";
|
||
const HAS_BUILDER = "the builder is installed as a module, with an account";
|
||
const BASE_BUILT = "the mesh builds the shared base from source";
|
||
const STORE_RUNS = "the mesh builds and runs a store of its own";
|
||
const CATALOGUE_RUNS = "the mesh builds and runs its own catalogue";
|
||
const CONTROL_REBUILT = "the mesh rebuilds its own control plane from source";
|
||
const NETWORKED = "the mesh puts itself on a private network, and its machine has a name";
|
||
const FILTERED = "the machine has a packet filter, loaded from what modules declared";
|
||
const MODULE_BUILT = "the mesh builds a module standing on that base";
|
||
const ANCHOR_RUNS = "the anchor runs the module the mesh built";
|
||
const DESCRIBES = "the control plane can describe the mesh, and what it says is true";
|
||
const CATALOGUED = "the catalogue holds every module this mesh built";
|
||
const NETWORK = "the machine's networking is what the modules asked for";
|
||
const DECLARED = "every resource the mesh declared is true on the machine";
|
||
const FOLLOWS = "a change to a module's source reaches the machine on its own";
|
||
const SURVIVES = "the mesh comes back after the machine reboots";
|
||
const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping", container: "amqp-ping" };
|
||
|
||
const capability = await labIsUsable();
|
||
const binary = hostBinaryPath();
|
||
const installer = bootstrapBinaryPath();
|
||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
||
const source = process.env["MESH_LAB_SOURCE"] ?? "";
|
||
const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? "";
|
||
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
||
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "fresh-mesh-live" : undefined);
|
||
|
||
/**
|
||
* Where a repository other than the control plane's lives.
|
||
*
|
||
* Derived from `MESH_LAB_SOURCE` by swapping the last path segment, because every one of these
|
||
* repositories sits beside the others under the same owner on the same forge. Overridable, so a
|
||
* forge that is arranged differently does not need this bed edited.
|
||
*/
|
||
function forgeUrl(repo: string): string {
|
||
const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`];
|
||
if (override) return override;
|
||
return source.replace(/[^/]+\.git$/, `${repo}.git`);
|
||
}
|
||
/**
|
||
* What to build, per repository.
|
||
*
|
||
* A branch is acceptable for an ordinary build; only genesis insists on a commit (ADR 0071). Per
|
||
* repository rather than one value for all of them, because a change under test usually lives in
|
||
* one repository and the rest should be built from what everyone else has — building them all from
|
||
* a feature branch would prove that branch against itself.
|
||
*
|
||
* MESH_LAB_BUILD_REF the default for every repository
|
||
* MESH_LAB_BUILD_REF_MESH_CATALOG ...overridden for one
|
||
*/
|
||
function refFor(repo: string): string {
|
||
const override = process.env[`MESH_LAB_BUILD_REF_${repo.toUpperCase().replaceAll("-", "_")}`];
|
||
return override ?? process.env["MESH_LAB_BUILD_REF"] ?? "main";
|
||
}
|
||
|
||
/**
|
||
* The shared base's manifest, on this workstation.
|
||
*
|
||
* The base is a repository with a manifest at its root (novox/hq ADR 0069), so unlike the
|
||
* catalogue's modules it is not under `MESH_LAB_CATALOG`. Derived from that path on the convention
|
||
* that the checkouts sit beside each other, and overridable for a layout where they do not.
|
||
*/
|
||
const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ??
|
||
resolve(catalogDir, "..", "..", BASE.repo, "module.json");
|
||
|
||
const skip =
|
||
!capability.usable ? capability.why :
|
||
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
||
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
||
!source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" :
|
||
!sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" :
|
||
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate template" :
|
||
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
||
false;
|
||
|
||
let instanceId = "";
|
||
let raised: GenesisResult;
|
||
|
||
// ---- talking to the machines ------------------------------------------------------------------
|
||
|
||
function quote(s: string): string {
|
||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||
}
|
||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||
const { stdout } = await exec(instanceId, machine, [
|
||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||
], timeoutMs);
|
||
const marker = stdout.lastIndexOf("__exit=");
|
||
if (marker < 0) return { out: stdout, ok: false };
|
||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||
}
|
||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||
const { out, ok } = await on(machine, command, timeoutMs);
|
||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||
return out;
|
||
}
|
||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||
return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
|
||
}
|
||
|
||
/**
|
||
* Stop the machine and start it again, the way a power cut or a kernel upgrade would.
|
||
*
|
||
* There is no restart in the lab's own vocabulary, which is its own small finding: nothing had ever
|
||
* needed one, because nothing had ever asked whether a mesh comes back.
|
||
*/
|
||
async function restartMachine(machine: string): Promise<void> {
|
||
const name = await instanceNameOf(instanceId, machine);
|
||
await incus(["restart", name], 180_000);
|
||
await waitUntilAllUsable([name], 300, (m) => console.log(` restart: ${m}`));
|
||
}
|
||
|
||
/** A module the mesh has to make for itself: where its source is, and what it runs when it works. */
|
||
interface CoreModule { module: string; repo: string; path: string; container: string }
|
||
|
||
/**
|
||
* Build a module from source, install it, and wait for it to actually run.
|
||
*
|
||
* The whole sequence a module goes through, in one place because the mesh has to do it for several
|
||
* before it is a mesh at all: register the manifest, build it from its repository and path, issue
|
||
* the broker account its runtime needs, assign it, send it, and then ask the machine whether the
|
||
* container is up.
|
||
*
|
||
* **The account is not optional and is not swallowed.** A module's runtime is a tool host: it
|
||
* connects to the mesh's broker before doing anything. Without an account the mesh still fills the
|
||
* secret the module declares it owns — with a generated value — so the container starts, fails to
|
||
* parse a password as a credential document, and loops on a JSON syntax error mentioning no
|
||
* missing account. That cost a step that reported PASS.
|
||
*/
|
||
async function bringUp(m: CoreModule): Promise<string> {
|
||
await registerModule(m.module, resolve(catalogDir, m.module, "module.json"));
|
||
const built = await mesh(
|
||
`build ${forgeUrl(m.repo)} --path ${m.path} --ref ${refFor(m.repo)} --wait 1200s`, 1_500_000);
|
||
assert.doesNotMatch(built, /failed/i, built);
|
||
await mesh(`module issue ${m.module} --node ${CONTROL}`);
|
||
await mesh(`assign ${CONTROL} ${m.module}`);
|
||
await mesh(`push ${CONTROL}`, 600_000);
|
||
return `${built}\n${await waitForContainer(CONTROL, m.container)}`;
|
||
}
|
||
|
||
/**
|
||
* Wait for one container, BY NAME, to be running.
|
||
*
|
||
* **Named exactly, because substring matching passed a step that had failed.** Waiting for "a
|
||
* container whose name contains lavinmq" was satisfied by the broker — `lavinmq`, up and healthy —
|
||
* while the thing actually under test, the module's own runtime `mesh-lavinmq`, was crash-looping
|
||
* beside it. The step went green and the fault was found by reading `docker ps` by hand.
|
||
*/
|
||
async function waitForContainer(node: string, container: string, seconds = 200): Promise<string> {
|
||
const deadline = Date.now() + seconds * 1_000;
|
||
let last = "";
|
||
while (Date.now() < deadline) {
|
||
const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
||
last = ps;
|
||
const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container);
|
||
if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps;
|
||
await new Promise((r) => setTimeout(r, 5_000));
|
||
}
|
||
const logs = (await on(node, `docker logs --tail 15 ${container} 2>&1`)).out;
|
||
throw new Error(
|
||
`${container} is not running on ${node}.\n\ncontainers:\n${last}\n\nwhat it said:\n${logs}`);
|
||
}
|
||
|
||
/**
|
||
* Register a module from a manifest on this workstation.
|
||
*
|
||
* **The control plane runs in a container, so a file on the machine is not a file it can open.**
|
||
* Pushing the manifest to the machine and naming that path got `no such file or directory` from
|
||
* inside mesh-control, which is correct and was briefly mistaken for a missing manifest. It is
|
||
* copied the last step of the way with `docker cp`.
|
||
*
|
||
* **Into the root, not into /tmp.** The control plane's image is a minimal one and has no `/tmp`
|
||
* to copy into — `docker cp` says so in those words. `/` is the one directory every image has.
|
||
*/
|
||
async function registerModule(module: string, manifest: string): Promise<string> {
|
||
assert.ok(existsSync(manifest), `no manifest for ${module} at ${manifest}`);
|
||
const onMachine = `/tmp/${module}.json`;
|
||
const inContainer = `/${module}.json`;
|
||
await push(instanceId, CONTROL, manifest, onMachine);
|
||
await must(CONTROL, `docker cp ${onMachine} mesh-control:${inContainer}`);
|
||
return mesh(`module add ${inContainer}`);
|
||
}
|
||
function tokenFrom(said: string): string {
|
||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||
assert.ok(found, `no token in:\n${said}`);
|
||
return found;
|
||
}
|
||
|
||
// ---- steps, recorded rather than thrown --------------------------------------------------------
|
||
|
||
interface Step {
|
||
code: string; title: string; ok: boolean; why: string; said: string; seconds: number;
|
||
}
|
||
const steps = new Map<string, Step>();
|
||
const order: string[] = [];
|
||
|
||
/**
|
||
* Run one step of the plan, remember what it said, and never throw.
|
||
*
|
||
* **Each step carries a code, and the code is the point.** A step used to be identified by its own
|
||
* sentence, so "which one failed" meant reading prose, and rewording a step silently made it a
|
||
* different step with no history. A code is stable, sorts, and can be pointed at: "V1 failed" says
|
||
* something that a whole sentence does not.
|
||
*
|
||
* A step whose dependency did not succeed is not attempted — its answer would be meaningless and
|
||
* its failure would be attributed to the wrong cause. The run still continues to the end, so the
|
||
* report says what was established and what was never asked, which are different things.
|
||
*/
|
||
async function step(
|
||
code: string, title: string, needs: string | null, fn: () => Promise<string>,
|
||
): Promise<void> {
|
||
order.push(title);
|
||
if (needs && !steps.get(needs)?.ok) {
|
||
steps.set(title, { code, title, ok: false, seconds: 0, said: "",
|
||
why: `not attempted — ${steps.get(needs)?.code ?? "?"} (${needs}) did not succeed` });
|
||
console.log(`[${code}] SKIP ${title}`);
|
||
return;
|
||
}
|
||
console.log(`\n[${code}] ---- ${title} ----`);
|
||
const began = Date.now();
|
||
const took = () => Math.round((Date.now() - began) / 1000);
|
||
try {
|
||
const said = await fn();
|
||
steps.set(title, { code, title, ok: true, why: "", said, seconds: took() });
|
||
console.log(`[${code}] PASS ${title} (${took()}s)`);
|
||
} catch (err) {
|
||
const why = (err as Error).message;
|
||
steps.set(title, { code, title, ok: false, why, said: "", seconds: took() });
|
||
console.log(`[${code}] FAIL ${title} (${took()}s)\n${why.split("\n").slice(0, 25).join("\n")}`);
|
||
}
|
||
}
|
||
|
||
function report(name: string): string {
|
||
const s = steps.get(name);
|
||
if (!s) return `${name}: never ran`;
|
||
const lines = order.map((n) => {
|
||
const it = steps.get(n);
|
||
return ` ${it?.ok ? "PASS" : "FAIL"} ${n}`;
|
||
});
|
||
return `${s.why}\n\nWhere this bed got to:\n${lines.join("\n")}`;
|
||
}
|
||
|
||
|
||
/**
|
||
* The plan, as data.
|
||
*
|
||
* Stated before any of it is attempted, so a reader knows what the run is trying to establish
|
||
* rather than inferring it from whatever happens to be printed. Codes are stable; titles may be
|
||
* reworded without the step losing its identity.
|
||
*
|
||
* Five phases, and the order is the argument: a mesh is RAISED, then it must be able to PRODUCE,
|
||
* then something is USED on it, then it is asked to describe itself and its networking is
|
||
* VERIFIED, and finally it has to ENDURE — a change following on its own, and coming back after
|
||
* the machine stops.
|
||
*/
|
||
const PLAN: { code: string; title: string }[] = [
|
||
{ code: "R1", title: GENESIS },
|
||
{ code: "R2", title: SUBSTRATE },
|
||
{ code: "R3", title: BUILT_CP },
|
||
{ code: "R4", title: PIVOTED },
|
||
{ code: "R5", title: HAS_REGISTRY },
|
||
{ code: "R6", title: ENROLLED },
|
||
{ code: "R7", title: HAS_BUILDER },
|
||
{ code: "P1", title: BASE_BUILT },
|
||
{ code: "P2", title: STORE_RUNS },
|
||
{ code: "P3", title: CATALOGUE_RUNS },
|
||
{ code: "P4", title: CONTROL_REBUILT },
|
||
{ code: "N1", title: NETWORKED },
|
||
{ code: "N2", title: FILTERED },
|
||
{ code: "U1", title: MODULE_BUILT },
|
||
{ code: "U2", title: NEEDS },
|
||
{ code: "U3", title: ANCHOR_RUNS },
|
||
{ code: "V1", title: DESCRIBES },
|
||
{ code: "V2", title: CATALOGUED },
|
||
{ code: "V3", title: NETWORK },
|
||
{ code: "V4", title: DECLARED },
|
||
{ code: "E1", title: FOLLOWS },
|
||
{ code: "E2", title: SURVIVES },
|
||
];
|
||
|
||
/** What this run intends to establish, said before any of it is attempted. */
|
||
function statePlan(): void {
|
||
console.log(`\n================ THE PLAN ================`);
|
||
for (const s of PLAN) console.log(` ${s.code.padEnd(3)} ${s.title}`);
|
||
console.log(` ${PLAN.length} steps. A step whose dependency fails is not attempted.\n`);
|
||
}
|
||
|
||
/** The run's verdict: a table, and a file something other than a person can read. */
|
||
function stateOutcome(): void {
|
||
console.log(`\n================ WHAT THIS MESH DID FOR ITSELF ================`);
|
||
let established = 0;
|
||
for (const title of order) {
|
||
const s = steps.get(title);
|
||
if (!s) continue;
|
||
if (s.ok) established++;
|
||
const mark = s.ok ? "PASS" : s.why.startsWith("not attempted") ? "SKIP" : "FAIL";
|
||
console.log(` ${s.code.padEnd(3)} ${mark} ${title}${s.seconds ? ` (${s.seconds}s)` : ""}`);
|
||
}
|
||
console.log(` ${established}/${PLAN.length} established.`);
|
||
const where = process.env["MESH_LAB_REPORT"] ?? "one-node-mesh-report.json";
|
||
try {
|
||
writeFileSync(where, JSON.stringify({ scenario: SCENARIO, established, of: PLAN.length,
|
||
steps: PLAN.map(({ code, title }) => {
|
||
const s = steps.get(title);
|
||
return { code, title, status: !s ? "never-ran"
|
||
: s.ok ? "pass" : s.why.startsWith("not attempted") ? "skip" : "fail",
|
||
seconds: s?.seconds ?? 0, why: s?.ok ? "" : s?.why ?? "" };
|
||
}) }, null, 2));
|
||
console.log(` report written to ${where}`);
|
||
} catch { /* a report that cannot be written must not fail a run that passed */ }
|
||
}
|
||
|
||
before(async () => {
|
||
if (skip) return;
|
||
statePlan();
|
||
|
||
const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||
onProgress: (m) => console.log(`raise: ${m}`),
|
||
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
||
});
|
||
instanceId = bed.instanceId;
|
||
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
||
console.log(`NOTHING WAS LOADED: this scenario names no images. Every image on every machine ` +
|
||
`below was pulled from the internet or built by the mesh.`);
|
||
|
||
// ---- 1. GENESIS -----------------------------------------------------------------------------
|
||
//
|
||
// The shared description, the same one `genesis-single` calls. The bundle is the TEMPLATE with
|
||
// nothing held: no image is pre-resolved, because none is here to resolve to.
|
||
await step("R1", GENESIS, null, async () => {
|
||
try {
|
||
raised = await genesis({
|
||
instanceId,
|
||
node: CONTROL,
|
||
installer: installer as string,
|
||
catalogDir,
|
||
// The broker's advertised address, corrected.
|
||
//
|
||
// The template hardcodes 192.0.2.10:5671 — the address of the anchor in the single-machine
|
||
// bed. A token carries this verbatim as the endpoint an enrolling node dials, so on a mesh
|
||
// whose anchor is somewhere else every node, including this one, would enrol against an
|
||
// address nothing answers on. The installer refuses to guess it and says so, which is
|
||
// right: it does not know what this machine is called from outside.
|
||
bundleTemplate: substrateBundle(bundle, []).replaceAll("192.0.2.10:5671", `${ANCHOR}:5671`),
|
||
registry: REGISTRY,
|
||
source,
|
||
sourceRef,
|
||
log: (m) => console.log(m),
|
||
});
|
||
} catch (err) {
|
||
throw new Error(`the installer never ran: ${(err as Error).message}`);
|
||
}
|
||
if (!raised.ok) throw new Error(`${raised.step || "no step named"}: ${raised.why}\n\n${raised.report.join("\n")}`);
|
||
return raised.report.join("\n");
|
||
});
|
||
|
||
// ---- WHAT GENESIS CLAIMED, ASKED OF THE MACHINE ----------------------------------------------
|
||
//
|
||
// **Genesis is twelve steps and was reported as one line.** All the work of raising a mesh
|
||
// happens inside it, so "genesis failed" said nothing about which of its claims broke, and
|
||
// "genesis passed" was one tick standing in for six separate things being true.
|
||
//
|
||
// Each is asked of the machine rather than read from the installer's own output — the installer
|
||
// saying it published an image and the registry serving one are different facts, and it is the
|
||
// second that matters.
|
||
|
||
await step("R2", SUBSTRATE, GENESIS, async () => {
|
||
await waitForContainer(CONTROL, "mesh-store", 120);
|
||
await waitForContainer(CONTROL, "mesh-broker", 120);
|
||
return (await on(CONTROL, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out;
|
||
});
|
||
|
||
// **The whole reason the installer carries a builder.** A carried control-plane image would
|
||
// satisfy "a control plane is running" equally well, which is what makes this worth asserting:
|
||
// the image has to be one this mesh's own registry serves.
|
||
await step("R3", BUILT_CP, GENESIS, async () => {
|
||
const image = (await on(CONTROL,
|
||
`docker inspect -f '{{.Config.Image}}' mesh-control 2>&1`)).out.trim();
|
||
assert.match(image, /@sha256:[0-9a-f]{64}/,
|
||
`the control plane names its image by tag, not by digest: ${image}`);
|
||
assert.ok(image.includes(":5000/"),
|
||
`the control plane runs an image no registry of this mesh served: ${image}`);
|
||
return image;
|
||
});
|
||
|
||
await step("R4", PIVOTED, BUILT_CP, async () => {
|
||
const ps = (await on(CONTROL, `docker ps -a --format '{{.Names}}'`)).out;
|
||
assert.doesNotMatch(ps, /^temp-mesh-control$/m,
|
||
`the temporary control plane is still here, so the pivot did not finish:\n${ps}`);
|
||
return ps;
|
||
});
|
||
|
||
await step("R5", HAS_REGISTRY, SUBSTRATE, async () => {
|
||
await waitForContainer(CONTROL, "mesh-registry", 120);
|
||
const held = (await on(CONTROL,
|
||
`curl -sS --max-time 15 http://127.0.0.1:5000/v2/_catalog`)).out;
|
||
assert.match(held, /mesh-control/,
|
||
`the registry serves no mesh-control, so nothing was published into it:\n${held}`);
|
||
return held.trim();
|
||
});
|
||
|
||
await step("R6", ENROLLED, GENESIS, async () => {
|
||
const nodes = await mesh("node list");
|
||
assert.match(nodes, new RegExp(`^${CONTROL}\\b`, "m"),
|
||
`the mesh has not heard from the machine it is running on:\n${nodes}`);
|
||
// Heard from once is not an agent running, and the difference is the whole of joining.
|
||
const agent = (await on(CONTROL, `pgrep -af '[m]esh-host run' | head -3`)).out.trim();
|
||
assert.ok(agent, `no host agent is running, so nothing would apply what the mesh sends`);
|
||
return `${nodes.trim()}\n${agent}`;
|
||
});
|
||
|
||
await step("R7", HAS_BUILDER, HAS_REGISTRY, async () => {
|
||
await waitForContainer(CONTROL, "mesh-builder", 120);
|
||
const modules = await mesh("module list");
|
||
assert.match(modules, /^builder\b/m,
|
||
`the builder is running but the mesh holds no record of it as a module:\n${modules}`);
|
||
return modules;
|
||
});
|
||
|
||
// ---- 2..6. THE MESH BECOMES ONE --------------------------------------------------------------
|
||
//
|
||
// **Joining used to be here, second, and that was the wrong order.** Three machines were enrolled
|
||
// into a mesh that could not yet produce a single module, and the step was reported as though
|
||
// something had been shown. They do join — reliably — but joining a mesh that can build nothing
|
||
// proves only that enrolment works, which was never the doubtful part.
|
||
//
|
||
// `17-raising-a-mesh` says it plainly: genesis ends with a mesh of one that RUNS, and that is not
|
||
// the same as a mesh that WORKS; what remains after the core modules are built is "adding
|
||
// machines, and deciding what they run". So everything a mesh needs to be a mesh happens first,
|
||
// and machines arrive at the end.
|
||
|
||
// The toolchain and runtime every module with code of its own stands on. It is a module, and it
|
||
// is built like one — cloned from the forge by the builder installing put here, compiled on the
|
||
// machine, published into the mesh's own registry.
|
||
await step("P1", BASE_BUILT, HAS_BUILDER, async () => {
|
||
// Registered from the manifest the builder will also read, so what the mesh holds and what it
|
||
// builds are the same description of the same module.
|
||
//
|
||
// **Not swallowed.** This call used to end in `.catch(() => {})`, on the reasoning that the
|
||
// base might already be known. It hid a real failure — the manifest was being named at a path
|
||
// inside a container that had never seen it — and the step passed anyway, because a base with
|
||
// nothing to stand on builds whether or not the mesh has a record of it.
|
||
await registerModule(BASE.module, baseManifest);
|
||
const built = await mesh(
|
||
`build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000);
|
||
assert.doesNotMatch(built, /failed/i, built);
|
||
return built;
|
||
});
|
||
|
||
// A store of its own. **Not the substrate's.** The installer raises a store for the control
|
||
// plane to keep its own records in, the way it raises a broker — plumbing, not a module the mesh
|
||
// has any record of, so it provides nothing to anything. A module that wants a database wants a
|
||
// provider in the graph, and the catalogue below is the first thing to want one.
|
||
await step("P2", STORE_RUNS, BASE_BUILT, () => bringUp(STORE));
|
||
|
||
// And the catalogue, which was missing from this test altogether.
|
||
//
|
||
// Without it the mesh holds no module graph: it cannot say what it has, what a module is made
|
||
// of, what a change to one reaches, or what must be rebuilt. A mesh in that state still runs,
|
||
// which is exactly how its absence went unnoticed — "the mesh is up" was being read off the
|
||
// installer finishing rather than off the mesh being able to answer anything.
|
||
await step("P3", CATALOGUE_RUNS, STORE_RUNS, () => bringUp(CATALOGUE));
|
||
|
||
// The control plane, rebuilt from its own repository and rolled out.
|
||
//
|
||
// The installer built it once, which is what got the mesh running. Building it again THROUGH THE
|
||
// MODULE PATH — build, notice the version moved, roll it out — is a different claim: it is the
|
||
// moment the mesh stops depending on the installer for anything, and the first time the thing
|
||
// that performs an upgrade performs one on itself.
|
||
await step("P4", CONTROL_REBUILT, CATALOGUE_RUNS, async () => {
|
||
const built = await mesh(
|
||
`build ${forgeUrl(CONTROL_PLANE.repo)} --ref ${sourceRef} --wait 1200s`, 1_500_000);
|
||
assert.doesNotMatch(built, /failed/i, built);
|
||
const rolled = await mesh(`upgrade ${CONTROL_PLANE.module} roll-out`, 900_000);
|
||
// Asked of the machine rather than believed from the command: the control plane that answers
|
||
// afterwards is the one that has to be running for anything below this line to mean anything.
|
||
await waitForContainer(CONTROL, CONTROL_PLANE.container);
|
||
return `${built}\n${rolled}`;
|
||
});
|
||
|
||
// ---- THE MESH'S OWN NETWORKING ---------------------------------------------------------------
|
||
//
|
||
// **Four modules the control plane computes were assigned to nothing, and nothing complained.**
|
||
// `networking`, `mesh-wireguard`, `mesh-names` and `mesh-resolver` all existed as records that
|
||
// had never been placed on a machine — so no names were written, no private network was raised,
|
||
// and `/etc/hosts` held nothing. A module is a definition until it is assigned; being generated
|
||
// by the control plane does not place it.
|
||
//
|
||
// One word, by design: `networking` has no files of its own and is requirements only, so
|
||
// assigning it finds one answer to each and takes them. The day the catalogue holds a second VPN
|
||
// there are two answers, the mesh refuses and names both, and choosing is assigning the one you
|
||
// want.
|
||
await step("N1", NETWORKED, CONTROL_REBUILT, async () => {
|
||
await mesh(`assign ${CONTROL} ${NETWORK_MODULE}`);
|
||
// **Assigned is not placed, and they are two different acts.** Assigning installs the module
|
||
// that answers "how do machines reach each other"; placing says where THIS machine is on the
|
||
// resulting network. Without the second the module runs and writes a names file with no names
|
||
// in it — deliberately, because "a node with no address on the network has no name here", and a
|
||
// name resolving to nothing is worse than no name: a connection to an address that does not
|
||
// answer hangs, where a name that does not resolve fails at once and says so.
|
||
await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site hosting`);
|
||
await mesh(`push ${CONTROL}`, 600_000);
|
||
// What it was assigned for. A machine on a private network with no name on it has had the
|
||
// harder half done and the visible half not.
|
||
const deadline = Date.now() + 120_000;
|
||
let hosts = "";
|
||
while (Date.now() < deadline) {
|
||
hosts = (await on(CONTROL, `cat /etc/hosts`)).out;
|
||
if (/\.internal/.test(hosts)) break;
|
||
await new Promise((r) => setTimeout(r, 5_000));
|
||
}
|
||
assert.match(hosts, /\.internal/,
|
||
`${NETWORK_MODULE} is assigned and no machine has a name:\n${hosts}`);
|
||
const modules = await mesh("module list");
|
||
return `${hosts.trim()}\n\n${modules.trim()}`;
|
||
});
|
||
|
||
// ---- THE PACKET FILTER -------------------------------------------------------------------------
|
||
//
|
||
// Assigned separately from `networking` because they answer different questions: one is how
|
||
// machines reach each other, the other is what may reach this one. Both were assigned to nothing,
|
||
// and the second is the more alarming of the two — every rule the mesh generates from module
|
||
// declarations had never been applied to any machine in this test.
|
||
await step("N2", FILTERED, NETWORKED, async () => {
|
||
// **Registered first: the mesh had never heard of it.** The networking family is computed by
|
||
// the control plane, so the mesh knows those modules exist without anyone saying so. The
|
||
// firewall is an ordinary catalogue module and needs adding like any other — "no module of
|
||
// that name: firewall" — which is a second way for a module to be absent, and less obvious
|
||
// than being present and placed nowhere.
|
||
//
|
||
// No build: its resources are a package and a service, so there is nothing to compile.
|
||
await registerModule(FILTER_MODULE, resolve(catalogDir, FILTER_MODULE, "module.json"));
|
||
await mesh(`assign ${CONTROL} ${FILTER_MODULE}`);
|
||
await mesh(`push ${CONTROL}`, 600_000);
|
||
const deadline = Date.now() + 180_000;
|
||
let ruleset = "";
|
||
while (Date.now() < deadline) {
|
||
ruleset = (await on(CONTROL, `nft list table inet mesh 2>&1`)).out;
|
||
if (/chain input/.test(ruleset)) break;
|
||
await new Promise((r) => setTimeout(r, 5_000));
|
||
}
|
||
assert.match(ruleset, /chain input/,
|
||
`${FILTER_MODULE} is assigned and the machine has no mesh filter:\n${ruleset}`);
|
||
return ruleset;
|
||
});
|
||
|
||
// ---- 7..8. SOMETHING TO RUN ---------------------------------------------------------------
|
||
await step("U1", MODULE_BUILT, FILTERED, async () => {
|
||
await registerModule(MODULE.module, resolve(catalogDir, MODULE.module, "module.json"));
|
||
const built = await mesh(
|
||
`build ${forgeUrl(MODULE.repo)} --path ${MODULE.path} --ref ${refFor(MODULE.repo)} --wait 1200s`,
|
||
1_500_000);
|
||
assert.doesNotMatch(built, /failed/i, built);
|
||
// The point of the whole step: what came out is named by a digest this mesh's registry
|
||
// assigned, not by a placeholder and not by a tag.
|
||
const builds = await mesh(`builds ${MODULE.module}`);
|
||
assert.match(builds, /sha256:[0-9a-f]{12}/,
|
||
`the build recorded no digest — the module is not pinned to anything this registry serves:\n${builds}`);
|
||
return `${built}\n${builds}`;
|
||
});
|
||
|
||
// `amqp-ping` requires the `amqp` provision, and the mesh refused to place it: "nothing provides
|
||
// amqp, wanted by amqp-ping". That refusal is correct and is the reason this step exists rather
|
||
// than the reason to pick an easier module.
|
||
//
|
||
// `lavinmq` is that module. It was first added here on the belief that it needed no building —
|
||
// its broker is an upstream image — and the mesh refused it again: two of its three containers
|
||
// named a placeholder digest, "which is never a real image". Also right. The broker is upstream,
|
||
// but the module is not only the broker: it carries a run-once bootstrap that writes the broker's
|
||
// configuration, a provisioner that grants each consumer its own vhost and user, tools and an
|
||
// event consumer, all of it its own code.
|
||
await step("U2", NEEDS, MODULE_BUILT, () => bringUp(PROVIDER));
|
||
|
||
// The machine that built it. This is the case every earlier proof covered, and it is here as the
|
||
// control for the last step: if this fails, that one's failure says nothing about fetching.
|
||
await step("U3", ANCHOR_RUNS, NEEDS, async () => {
|
||
await mesh(`module issue ${MODULE.module} --node ${CONTROL}`);
|
||
await mesh(`assign ${CONTROL} ${MODULE.module}`);
|
||
await mesh(`push ${CONTROL}`, 600_000);
|
||
return waitForContainer(CONTROL, MODULE.module);
|
||
});
|
||
|
||
// ---- 9. IT CAN DESCRIBE ITSELF ---------------------------------------------------------------
|
||
//
|
||
// **Presence is not function, and this step exists because I kept confusing them.** A container
|
||
// being up was taken as the catalogue working; a name containing "lavinmq" was taken as the
|
||
// module running. The mesh holds a graph — nodes, what each is assigned, what capabilities each
|
||
// has, which claims are occupied, what each module is configured with, which provisions exist and
|
||
// who holds them — and none of it was ever asked a question.
|
||
await step("V1", DESCRIBES, ANCHOR_RUNS, async () => {
|
||
const said: string[] = [];
|
||
|
||
// The mesh's own verdict on itself. Nothing wrong, nothing waiting, nothing behind.
|
||
const state = JSON.parse(await mesh("status --json")) as {
|
||
wrong: { node: string; outcome: string; refused?: string }[];
|
||
waiting: { node: string }[];
|
||
reported: { node: string; outcome: string; current: boolean }[];
|
||
};
|
||
assert.equal(state.wrong.length, 0,
|
||
`the mesh reports something wrong:\n${JSON.stringify(state.wrong, null, 2)}`);
|
||
assert.equal(state.waiting.length, 0,
|
||
`the mesh is waiting on a node:\n${JSON.stringify(state.waiting, null, 2)}`);
|
||
const node = state.reported.find((r) => r.node === CONTROL);
|
||
assert.ok(node, `the mesh does not report the only machine it has:\n${JSON.stringify(state)}`);
|
||
assert.equal(node.outcome, "applied", `${CONTROL} did not apply what it was sent: ${node.outcome}`);
|
||
assert.ok(node.current, `${CONTROL} is not running what the mesh would send it`);
|
||
said.push(` status one node, applied, current, nothing wrong`);
|
||
|
||
// Every module a mesh has to hold, including the three it cannot build for itself.
|
||
const modules = await mesh("module list");
|
||
for (const m of MUST_HOLD) {
|
||
assert.match(modules, new RegExp(`^${m}\\b`, "m"),
|
||
`the mesh holds no ${m}. A mesh without it is not finished:\n${modules}`);
|
||
}
|
||
said.push(` module list ${MUST_HOLD.length} modules, all present`);
|
||
|
||
// What the machine would be sent, and what it names. **No placeholder may survive here** — a
|
||
// digest of all zeroes is never a real image, and a declaration carrying one reaches a machine
|
||
// that will try to fetch it.
|
||
const plan = await mesh(`plan ${CONTROL} --json`);
|
||
assert.doesNotMatch(plan, /sha256:0{64}/,
|
||
`the machine's own plan names a placeholder digest, which is never a real image`);
|
||
assert.match(plan, /sha256:[0-9a-f]{64}/, `the plan pins nothing by digest at all`);
|
||
said.push(` plan every image pinned, no placeholders`);
|
||
|
||
// And the catalogue, ASKED rather than observed. These five questions are what it exists for.
|
||
return said.join("\n");
|
||
});
|
||
|
||
// ---- V2. AND THE CATALOGUE HOLDS WHAT WAS BUILT -----------------------------------------------
|
||
//
|
||
// **Split from the step above, because they are two claims and only one of them fails.** The
|
||
// control plane describing the mesh correctly and the catalogue holding a complete record of it
|
||
// are different things, and bundling them meant one open fault stopped three later steps from
|
||
// ever being attempted.
|
||
await step("V2", CATALOGUED, DESCRIBES, async () => {
|
||
// **Asked as an operator would have to, which turns out to be nobody.**
|
||
//
|
||
// The obvious move — run the invoke inside the catalogue's own container — is refused by the
|
||
// broker: a module's account is scoped to what it declares it emits and consumes, and calling
|
||
// a tool needs a temporary reply queue that scope does not cover. So a module can SERVE tools
|
||
// and cannot CALL them, and nothing issues an account to anyone who wants to ask (novox/hq
|
||
// issue 049). Until that is decided the caller is the substrate's bootstrap admin over the
|
||
// broker's loopback, reached by joining its network namespace.
|
||
const image = (await on(CONTROL,
|
||
`docker inspect -f '{{.Config.Image}}' mesh-catalog`)).out.trim();
|
||
const ask = async (tool: string, args = "{}") =>
|
||
must(CONTROL,
|
||
`docker run --rm --network container:mesh-broker ` +
|
||
`-e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
|
||
`${image} invoke mesh-catalog ${tool} ${quote(args)}`,
|
||
120_000);
|
||
|
||
const held = await ask("catalog_modules");
|
||
// Compared against what the control plane ordered, rather than against a list written here: a
|
||
// catalogue cannot know what it was never told, so it must be measured against something that
|
||
// does. novox/hq issue 050 — on a fresh mesh the modules built before the catalogue existed
|
||
// are exactly the ones it needed in order to exist, so the hole is always the foundation.
|
||
const missing = MUST_HOLD.filter((m) =>
|
||
!["distribution", "builder"].includes(m) && !held.includes(m));
|
||
assert.deepEqual(missing, [],
|
||
`the catalogue does not hold ${missing.join(", ")} — the mesh built them and its own ` +
|
||
`record has no trace of it (novox/hq issue 050):\n${held}`);
|
||
assert.doesNotMatch(held, /sha256:0{64}/, `the catalogue holds a placeholder version`);
|
||
|
||
const provides = await ask("catalog_provides", JSON.stringify({ provision: "amqp" }));
|
||
assert.ok(provides.includes(PROVIDER.module),
|
||
`the catalogue cannot say what provides amqp, which is a question it exists for:\n${provides}`);
|
||
|
||
const stale = await ask("catalog_stale");
|
||
return `${held}\n${provides}\n${stale}`;
|
||
});
|
||
|
||
// ---- V3. AND ITS NETWORKING IS WHAT WAS ASKED FOR ---------------------------------------------
|
||
//
|
||
// **Left out of this test entirely until it was pointed out**, which is hard to defend: the
|
||
// firewall is generated from what modules declare they listen on, and a firewall that opens the
|
||
// wrong set is either a service nobody can reach or a port nobody meant to publish. Neither shows
|
||
// up as a failed container.
|
||
await step("V3", NETWORK, DESCRIBES, async () => {
|
||
const said: string[] = [];
|
||
|
||
const ruleset = (await on(CONTROL, `nft list table inet mesh 2>&1`)).out;
|
||
assert.match(ruleset, /chain input/, `the mesh's own firewall table is not there:\n${ruleset}`);
|
||
// Closed by default, or the rules below decide nothing.
|
||
assert.match(ruleset, /policy drop/, `the firewall does not default to closed:\n${ruleset}`);
|
||
// The floor: a machine that cannot be reached over ssh is a machine nobody can repair.
|
||
assert.match(ruleset, /\b22\b/, `ssh is not allowed anywhere in the ruleset`);
|
||
// What a module actually declared. The registry says it listens on 5000 for the mesh.
|
||
assert.match(ruleset, /\b5000\b/,
|
||
`the registry declares it listens on 5000 and nothing opened it:\n${ruleset}`);
|
||
said.push(` firewall default closed, ssh open, declared ports open`);
|
||
|
||
// The names the mesh writes for itself. A consumer reaching a provider by its `.internal`
|
||
// address depends on this file, and on it reaching inside containers.
|
||
const hosts = (await on(CONTROL, `cat /etc/hosts`)).out;
|
||
assert.match(hosts, /\.internal/, `the mesh wrote no .internal names:\n${hosts}`);
|
||
said.push(` names ${(hosts.match(/[a-z0-9-]+\.internal/g) ?? []).join(" ")}`);
|
||
|
||
// The networks the declarations asked for, rather than whatever the runtime had lying around.
|
||
const networks = (await on(CONTROL, `docker network ls --format '{{.Name}}'`)).out;
|
||
for (const wanted of ["lavinmq", "amqp-ping"]) {
|
||
assert.match(networks, new RegExp(`^${wanted}$`, "m"),
|
||
`the ${wanted} module declares a network and none exists:\n${networks}`);
|
||
}
|
||
said.push(` networks module networks present`);
|
||
return said.join("\n");
|
||
});
|
||
|
||
// ---- V4. THE WHOLE VOCABULARY, NOT THE PART I KEPT LOOKING AT --------------------------------
|
||
//
|
||
// **Every check in this file until now asked about containers.** A container is one resource kind
|
||
// out of ten — directory, file, user, network, access, archive, service, package, container,
|
||
// action — and a module is far more often the others: the firewall is a package and a service,
|
||
// the mesh's names are a file, a run-once step is an action or a container that exits. Asking
|
||
// only about containers is how a module with no container at all went unnoticed for this long.
|
||
//
|
||
// So this takes the declaration the machine was actually sent and verifies each resource in it,
|
||
// by kind, on the machine. Nothing is hand-picked: whatever the installed modules declared is
|
||
// what gets checked, and a kind nothing declared is REPORTED as unexercised rather than quietly
|
||
// counted as working.
|
||
await step("V4", DECLARED, NETWORK, async () => {
|
||
const plan = JSON.parse(await mesh(`plan ${CONTROL} --json`)) as {
|
||
resources: Record<string, unknown>[];
|
||
};
|
||
const seen = new Map<string, number>();
|
||
const wrong: string[] = [];
|
||
const unchecked: string[] = [];
|
||
|
||
for (const r of plan.resources) {
|
||
const kind = String(r["type"] ?? "");
|
||
seen.set(kind, (seen.get(kind) ?? 0) + 1);
|
||
const id = String(r["id"] ?? kind);
|
||
const check = async (command: string, why: string) => {
|
||
if (!(await on(CONTROL, command)).ok) wrong.push(`${kind} ${id}: ${why}`);
|
||
};
|
||
switch (kind) {
|
||
case "directory":
|
||
await check(`test -d ${quote(String(r["path"]))}`, `no directory at ${r["path"]}`);
|
||
break;
|
||
case "file":
|
||
await check(`test -f ${quote(String(r["path"]))}`, `no file at ${r["path"]}`);
|
||
break;
|
||
case "access":
|
||
await check(`test -e ${quote(String(r["path"]))}`, `nothing at ${r["path"]}`);
|
||
break;
|
||
case "archive":
|
||
await check(`test -e ${quote(String(r["path"]))}`,
|
||
`nothing unpacked at ${r["path"]} — the archive was never fetched`);
|
||
break;
|
||
case "package":
|
||
await check(`command -v pacman >/dev/null && pacman -Q ${quote(String(r["package"]))}`,
|
||
`the package ${r["package"]} is not installed`);
|
||
break;
|
||
case "service": {
|
||
// A unit the host put into a state. "running" is the state worth checking; a one-shot
|
||
// that has done its work reports inactive and that is correct (this is the reading that
|
||
// made the firewall module appear broken on every machine for months).
|
||
const unit = String(r["unit"]);
|
||
if (String(r["state"]) === "running") {
|
||
await check(`systemctl is-active ${quote(unit)} >/dev/null || ` +
|
||
`systemctl show -p ExecMainStatus --value ${quote(unit)} | grep -qx 0`,
|
||
`the unit ${unit} is neither active nor a one-shot that succeeded`);
|
||
}
|
||
break;
|
||
}
|
||
case "network":
|
||
await check(`docker network inspect ${quote(String(r["name"]))} >/dev/null 2>&1`,
|
||
`no network named ${r["name"]}`);
|
||
break;
|
||
case "container": {
|
||
const name = String(r["name"]);
|
||
if (r["run-once"] === true || r["schedule"]) {
|
||
// **A run-once step leaves nothing to ask, deliberately.** The host removes the exited
|
||
// container so a later apply is not confused by a stopped one, and keeps the record
|
||
// that it ran in its own store instead. So asserting the container exists asserts the
|
||
// opposite of correct behaviour — which this did, and reported a working mesh as
|
||
// broken on its first run.
|
||
//
|
||
// A scheduled step is the same between fires. Both are counted as unverified here
|
||
// rather than assumed good: what would verify them is the host's own record, and this
|
||
// asks the machine rather than the host.
|
||
unchecked.push(`${kind} ${id} (a step leaves nothing running to ask)`);
|
||
} else {
|
||
await check(`docker ps --format '{{.Names}}' | grep -qx ${quote(name)}`,
|
||
`the container ${name} is not running`);
|
||
}
|
||
break;
|
||
}
|
||
default:
|
||
unchecked.push(`${kind} ${id}`);
|
||
}
|
||
}
|
||
|
||
const kinds = [...seen.entries()].sort().map(([k, n]) => `${k}×${n}`).join(" ");
|
||
const never = ["directory", "file", "user", "network", "access", "archive", "service",
|
||
"package", "container", "action"].filter((k) => !seen.has(k));
|
||
assert.deepEqual(wrong, [],
|
||
`the machine is not what the mesh said it should be:\n ${wrong.join("\n ")}`);
|
||
return [
|
||
` declared ${plan.resources.length} resources — ${kinds}`,
|
||
unchecked.length ? ` not verified here ${unchecked.join(", ")}` : ` every kind present was verified`,
|
||
never.length ? ` NOT EXERCISED ${never.join(", ")} — nothing installed here declares one` : ``,
|
||
].filter(Boolean).join("\n");
|
||
});
|
||
|
||
// ---- 11. A CHANGE REACHES THE MACHINE ON ITS OWN ----------------------------------------------
|
||
//
|
||
// The whole point of the mesh, and the capability the migration depends on: move a module's
|
||
// source and the running copy follows, with nobody driving the steps. Everything above is
|
||
// machinery; this is what the machinery is for.
|
||
await step("E1", FOLLOWS, DECLARED, async () => {
|
||
const before = await mesh(`builds ${MODULE.module}`);
|
||
const was = before.match(/sha256:[0-9a-f]{64}/)?.[0] ?? "";
|
||
assert.ok(was, `nothing is pinned to rebuild from:\n${before}`);
|
||
|
||
// **The source has to actually move, and it cannot be faked.**
|
||
//
|
||
// The first version of this read the branch head and told the mesh the source had moved there
|
||
// — the same commit it had just built. The mesh answered, correctly, that everything was
|
||
// current. Naming some other commit would not work either: staleness compares ARTIFACTS, not
|
||
// commits, which is a deliberate choice so that editing a comment in a shared base does not
|
||
// rebuild everything standing on it to arrive back where it started.
|
||
//
|
||
// So this makes a real change to the module's source and pushes it. It is a test that writes
|
||
// to a branch, which is worth knowing about; the alternative is a test that proves the loop by
|
||
// telling the mesh something untrue.
|
||
const checkout = resolve(catalogDir, "..");
|
||
const marker = `// changed by the one-node test at build ${was.slice(7, 19)}\n`;
|
||
const file = resolve(catalogDir, MODULE.module, "index.ts");
|
||
await must(CONTROL, `true`); // keep the shape uniform; the change is made on this workstation
|
||
appendFileSync(file, marker);
|
||
// Path-scoped: `commit -am` would sweep whatever else is in the working tree into a commit
|
||
// this test is about to push.
|
||
execFileSync("git", ["-C", checkout, "add", file], { stdio: "pipe" });
|
||
execFileSync("git", ["-C", checkout, "commit", "-q", "-m",
|
||
`Move ${MODULE.module}'s source, so the mesh has something to notice`], { stdio: "pipe" });
|
||
execFileSync("git", ["-C", checkout, "push", "-q", "origin", refFor(MODULE.repo)],
|
||
{ stdio: "pipe" });
|
||
const head = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"],
|
||
{ encoding: "utf8" }).trim();
|
||
|
||
// Now the mesh is told. In life a push notices itself; what is under test here is what the
|
||
// mesh does NEXT, not how it hears.
|
||
await mesh(`module moved ${MODULE.module} ${head}`);
|
||
const behind = await mesh(`status`);
|
||
assert.match(behind, /behind|build --behind/,
|
||
`the source moved and the mesh does not report the module behind it:\n${behind}`);
|
||
|
||
await mesh(`build --behind --wait 1200s`, 1_500_000);
|
||
const rolled = await mesh(`upgrade ${MODULE.module} roll-out`, 900_000);
|
||
await waitForContainer(CONTROL, MODULE.container);
|
||
|
||
const after = await mesh(`builds ${MODULE.module}`);
|
||
const now = after.match(/sha256:[0-9a-f]{64}/)?.[0] ?? "";
|
||
assert.notEqual(now, was,
|
||
`the module was rebuilt and came back on the same artifact, so nothing reached the machine:` +
|
||
`\n${after}`);
|
||
return `${behind}\n${rolled}\n${after}`;
|
||
});
|
||
|
||
// ---- 12. AND IT SURVIVES THE MACHINE STOPPING -------------------------------------------------
|
||
//
|
||
// **Never once tested.** A mesh that works until the machine reboots is a demonstration, not
|
||
// something to move real services onto — and the installer is explicit that a host started the
|
||
// way the lab starts it does not survive a reboot, which makes this the check that says whether
|
||
// that matters.
|
||
await step("E2", SURVIVES, FOLLOWS, async () => {
|
||
await restartMachine(CONTROL);
|
||
const missing: string[] = [];
|
||
for (const container of MUST_RUN) {
|
||
try {
|
||
await waitForContainer(CONTROL, container, 240);
|
||
} catch {
|
||
missing.push(container);
|
||
}
|
||
}
|
||
const ps = (await on(CONTROL, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
||
assert.equal(missing.length, 0,
|
||
`after a reboot these are not running: ${missing.join(", ")}\n\ncontainers:\n${ps}`);
|
||
|
||
// **Containers coming back is not the mesh coming back.** The runtime restarts containers on
|
||
// its own; what makes a machine part of a mesh is an agent listening for what it should be. A
|
||
// machine whose containers returned and whose agent did not looks healthy and cannot be told
|
||
// anything.
|
||
//
|
||
// **A failure here is this lab's arrangement, not the mesh's capability.** mesh-host ships
|
||
// `nox-mesh-host.service` and two companions in packaging/; the installer declines to place
|
||
// them because a unit file is a packaging decision, and the lab starts the host with
|
||
// --host-in-background, which says in its own help that it does not survive a reboot. So this
|
||
// check is honest and the thing it catches is the step nobody runs — not a mesh that cannot
|
||
// come back.
|
||
const agent = (await on(CONTROL, `pgrep -af '[m]esh-host run' | head -3`)).out.trim();
|
||
assert.ok(agent,
|
||
`every container came back and no host agent did, so the machine is running the right ` +
|
||
`things and can no longer be told anything:\n${ps}`);
|
||
return `${ps}\n${agent}`;
|
||
});
|
||
|
||
stateOutcome();
|
||
}, { timeout: 7_200_000 });
|
||
|
||
after(async () => {
|
||
if (KEEP) {
|
||
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
|
||
return;
|
||
}
|
||
if (instanceId) await destroy(instanceId);
|
||
await destroyAll(`${SCENARIO}-`);
|
||
}, { timeout: 900_000 });
|
||
|
||
for (const name of [
|
||
GENESIS,
|
||
SUBSTRATE,
|
||
BUILT_CP,
|
||
PIVOTED,
|
||
HAS_REGISTRY,
|
||
ENROLLED,
|
||
HAS_BUILDER,
|
||
BASE_BUILT,
|
||
STORE_RUNS,
|
||
CATALOGUE_RUNS,
|
||
CONTROL_REBUILT,
|
||
NETWORKED,
|
||
FILTERED,
|
||
MODULE_BUILT,
|
||
NEEDS,
|
||
ANCHOR_RUNS,
|
||
DESCRIBES,
|
||
CATALOGUED,
|
||
NETWORK,
|
||
DECLARED,
|
||
FOLLOWS,
|
||
SURVIVES,
|
||
]) {
|
||
test(name, { skip, timeout: 60_000 }, () => {
|
||
assert.ok(steps.get(name)?.ok, report(name));
|
||
});
|
||
}
|